diff --git a/.circleci/config.yml b/.circleci/config.yml index f4ffb5223bdf..9324de943607 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -8,30 +8,33 @@ jobs: docker: # CircleCI maintains a library of pre-built images # documented at https://circleci.com/docs/2.0/circleci-images/ - - image: circleci/python:3.6.6 + - image: circleci/python:3.8.4 working_directory: ~/repo steps: - - checkout + - checkout: + - run: + name: pull changes from merge + command: | + if [[ -v CI_PULL_REQUEST ]] ; then git pull --ff-only origin "refs/pull/${CI_PULL_REQUEST//*pull\//}/merge" ; fi - run: name: create virtual environment, install dependencies command: | - python3 -m venv venv - ln -s $(which python3) venv/bin/python3.6 - . venv/bin/activate sudo apt-get update sudo apt-get install -y graphviz texlive-fonts-recommended texlive-latex-recommended texlive-latex-extra texlive-generic-extra latexmk texlive-xetex + python3.8 -m venv venv + . venv/bin/activate - run: name: build numpy command: | . venv/bin/activate - pip install --upgrade pip 'setuptools<49.2.0' - pip install -r test_requirements.txt + pip install --progress-bar=off --upgrade pip 'setuptools<49.2.0' + pip install --progress-bar=off -r test_requirements.txt pip install . - pip install -r doc_requirements.txt + pip install --progress-bar=off -r doc_requirements.txt - run: name: create release notes @@ -49,11 +52,20 @@ jobs: (cd doc ; git submodule update --init) python tools/refguide_check.py --rst + - run: + name: build devdocs w/ref warnings + command: | + . venv/bin/activate + cd doc + # Don't use -q, show warning summary" + SPHINXOPTS="-n" make -e html || echo "ignoring errors for now, see gh-13114" + - run: name: build devdocs command: | . venv/bin/activate cd doc + make clean SPHINXOPTS=-q make -e html - run: diff --git a/.gitattributes b/.gitattributes index dad6dde37cd0..bce3dbe6daad 100644 --- a/.gitattributes +++ b/.gitattributes @@ -3,3 +3,19 @@ numpy/lib/tests/data/*.npy binary # Release notes, reduce number of conflicts. doc/release/*.rst merge=union + +# Highlight our custom templating language as C, since it's hopefully better +# than nothing. This also affects repo statistics. +*.c.src linguist-language=C +*.inc.src linguist-language=C +*.h.src linguist-language=C + +# Mark some files as vendored +numpy/linalg/lapack_lite/f2c.c linguist-vendored +numpy/linalg/lapack_lite/f2c.h linguist-vendored +tools/npy_tempita/* linguist-vendored + +# Mark some files as generated +numpy/linalg/lapack_lite/f2c_*.c linguist-generated +numpy/linalg/lapack_lite/lapack_lite_names.h linguist-generated + diff --git a/.github/CODE_OF_CONDUCT.md b/.github/CODE_OF_CONDUCT.md index 57c98060eb3b..079098fae68a 100644 --- a/.github/CODE_OF_CONDUCT.md +++ b/.github/CODE_OF_CONDUCT.md @@ -1 +1 @@ -NumPy has a Code of Conduct, please see: https://www.numpy.org/devdocs/dev/conduct/code_of_conduct.html +NumPy has a Code of Conduct, please see: https://numpy.org/code-of-conduct diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE/bug-report.md similarity index 84% rename from .github/ISSUE_TEMPLATE.md rename to .github/ISSUE_TEMPLATE/bug-report.md index 3a25eeb1e9a3..78ffc1063eaf 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE/bug-report.md @@ -1,3 +1,8 @@ +--- +name: "Bug report" +about: Report a bug. Not for security vulnerabilities -- see below. + +--- @@ -11,8 +16,6 @@ import numpy as np << your code here >> ``` - - ### Error message: -### Numpy/Python version information: +### NumPy/Python version information: diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 000000000000..adfff81bd004 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,7 @@ +contact_links: + - name: Question/Help/Support + url: https://numpy.org/gethelp/ + about: "If you have a question, please look at the listed resources available on the website." + - name: Development-related matters + url: https://numpy.org/community/ + about: "If you would like to discuss development-related matters or need help from the NumPy team, see our community's communication channels." diff --git a/.github/ISSUE_TEMPLATE/documentation.md b/.github/ISSUE_TEMPLATE/documentation.md new file mode 100644 index 000000000000..cdb7cde2ee2f --- /dev/null +++ b/.github/ISSUE_TEMPLATE/documentation.md @@ -0,0 +1,20 @@ +--- +name: "Documentation" +about: Report an issue related to the NumPy documentation +labels: 04 - Documentation + +--- + +## Documentation + + + + diff --git a/.github/ISSUE_TEMPLATE/feature-request.md b/.github/ISSUE_TEMPLATE/feature-request.md new file mode 100644 index 000000000000..00c6f59c5faf --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature-request.md @@ -0,0 +1,16 @@ +--- +name: "Feature request" +about: Check instructions for submitting your idea on the mailing list first. + +--- + +## Feature + + diff --git a/.github/ISSUE_TEMPLATE/post-install.md b/.github/ISSUE_TEMPLATE/post-install.md new file mode 100644 index 000000000000..c0ec7896a40d --- /dev/null +++ b/.github/ISSUE_TEMPLATE/post-install.md @@ -0,0 +1,21 @@ +--- +name: "Post-install/importing issue" +about: If you have trouble importing or using NumPy after installation +labels: 32 - Installation + +--- + + + +### Steps to reproduce: + + + +### Error message: + + + + + diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index e12eea7bd763..508c8c034869 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,7 +1,16 @@ - - - + diff --git a/.github/pr-prefix-labeler.yml b/.github/pr-prefix-labeler.yml new file mode 100644 index 000000000000..b50def97e930 --- /dev/null +++ b/.github/pr-prefix-labeler.yml @@ -0,0 +1,12 @@ +"API": "30 - API" +"BENCH": "28 - Benchmark" +"BUG": "00 - Bug" +"DEP": "07 - Deprecation" +"DEV": "16 - Development" +"DOC": "04 - Documentation" +"ENH": "01 - Enhancement" +"MAINT": "03 - Maintenance" +"REV": "34 - Reversion" +"TST": "05 - Testing" +"REL": "14 - Release" +"WIP": "25 - WIP" diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml new file mode 100644 index 000000000000..99db967b383b --- /dev/null +++ b/.github/workflows/labeler.yml @@ -0,0 +1,13 @@ +name: "Pull Request Labeler" +on: + pull_request_target: + types: [opened, synchronize, reopened, edited] + +jobs: + pr-labeler: + runs-on: ubuntu-latest + steps: + - name: Label the PR + uses: gerrymanoim/pr-prefix-labeler@v3 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.gitignore b/.gitignore index c58b0e62feb9..85318467e849 100644 --- a/.gitignore +++ b/.gitignore @@ -182,6 +182,7 @@ benchmarks/results benchmarks/html benchmarks/env benchmarks/numpy +benchmarks/_asv_compare.conf.json # cythonized files cythonize.dat numpy/random/_mtrand/_mtrand.c diff --git a/.gitmodules b/.gitmodules index b1e13c3bc120..e69de29bb2d1 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +0,0 @@ -[submodule "doc/sphinxext"] - path = doc/sphinxext - url = https://github.com/numpy/numpydoc.git diff --git a/INSTALL.rst.txt b/INSTALL.rst.txt index 1c33060a625a..5ee97d7901a0 100644 --- a/INSTALL.rst.txt +++ b/INSTALL.rst.txt @@ -30,8 +30,8 @@ Building NumPy requires the following installed software: This is required for testing NumPy, but not for using it. -Python__ http://www.python.org -pytest__ http://pytest.readthedocs.io +Python__ https://www.python.org/ +pytest__ https://docs.pytest.org/en/stable/ Hypothesis__ https://hypothesis.readthedocs.io/en/latest/ @@ -39,12 +39,12 @@ Hypothesis__ https://hypothesis.readthedocs.io/en/latest/ If you want to build NumPy in order to work on NumPy itself, use ``runtests.py``. For more details, see - https://docs.scipy.org/doc/numpy/dev/development_environment.html + https://numpy.org/devdocs/dev/development_environment.html .. note:: - More extensive information on building NumPy (and SciPy) is maintained at - https://scipy.github.io/devdocs/building/ + More extensive information on building NumPy is maintained at + https://numpy.org/devdocs/user/building.html#building-from-source Basic Installation @@ -69,7 +69,7 @@ NPY_NUM_BUILD_JOBS. Choosing compilers ================== -NumPy needs a C compiler, and for development versions also Cython. A Fortran +NumPy needs a C compiler, and for development versions also needs Cython. A Fortran compiler isn't needed to build NumPy itself; the ``numpy.f2py`` tests will be skipped when running the test suite if no Fortran compiler is available. For building Scipy a Fortran compiler is needed though, so we include some details @@ -87,7 +87,7 @@ Windows ------- On Windows, building from source can be difficult (in particular if you need to -build SciPy as well, because that requires a Fortran compiler). Currently, the +build SciPy as well, because that requires a Fortran compiler). Currently, the most robust option is to use MSVC (for NumPy only). If you also need SciPy, you can either use MSVC + Intel Fortran or the Intel compiler suite. Intel itself maintains a good `application note @@ -151,7 +151,7 @@ Or by preloading a specific BLAS library with:: Build issues ============ -If you run into build issues and need help, the NumPy +If you run into build issues and need help, the NumPy and SciPy `mailing list `_ is the best -place to ask. If the issue is clearly a bug in NumPy, please file an issue (or +place to ask. If the issue is clearly a bug in NumPy, please file an issue (or even better, a pull request) at https://github.com/numpy/numpy. diff --git a/LICENSES_bundled.txt b/LICENSES_bundled.txt index e9c66d1dcfb6..00b7473777ca 100644 --- a/LICENSES_bundled.txt +++ b/LICENSES_bundled.txt @@ -1,16 +1,6 @@ The NumPy repository and source distributions bundle several libraries that are compatibly licensed. We list these here. -Name: Numpydoc -Files: doc/sphinxext/numpydoc/* -License: BSD-2-Clause - For details, see doc/sphinxext/LICENSE.txt - -Name: scipy-sphinx-theme -Files: doc/scipy-sphinx-theme/* -License: BSD-3-Clause AND PSF-2.0 AND Apache-2.0 - For details, see doc/scipy-sphinx-theme/LICENSE.txt - Name: lapack-lite Files: numpy/linalg/lapack_lite/* License: BSD-3-Clause diff --git a/README.md b/README.md index 6b2d327f6ef2..9271a5d28872 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # NumPy -[![Travis](https://img.shields.io/travis/numpy/numpy/master.svg?label=Travis%20CI)]( - https://travis-ci.org/numpy/numpy) +[![Travis CI](https://img.shields.io/travis/com/numpy/numpy/master?label=Travis%20CI)]( + https://travis-ci.com/github/numpy/numpy) [![Azure](https://dev.azure.com/numpy/numpy/_apis/build/status/azure-pipeline%20numpy.numpy)]( https://dev.azure.com/numpy/numpy/_build/latest?definitionId=5) [![codecov](https://codecov.io/gh/numpy/numpy/branch/master/graph/badge.svg)]( diff --git a/azure-pipelines.yml b/azure-pipelines.yml index da57649b884b..e5c0c25569be 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -29,177 +29,12 @@ stages: - stage: ComprehensiveTests jobs: - - job: Linux_Python_38_32bit_full_with_asserts - pool: - vmImage: 'ubuntu-18.04' - steps: - - script: | - docker pull quay.io/pypa/manylinux2010_i686 - docker run -v $(pwd):/numpy -e CFLAGS="-msse2 -std=c99 -UNDEBUG" \ - -e F77=gfortran-5 -e F90=gfortran-5 quay.io/pypa/manylinux2010_i686 \ - /bin/bash -xc "cd numpy && \ - /opt/python/cp38-cp38/bin/python -mvenv venv &&\ - source venv/bin/activate && \ - target=\$(python3 tools/openblas_support.py) && \ - cp -r \$target/lib/* /usr/lib && \ - cp \$target/include/* /usr/include && \ - python3 -m pip install -r test_requirements.txt && \ - echo CFLAGS \$CFLAGS && \ - python3 -m pip install -v . && \ - python3 runtests.py -n --debug-info --mode=full -- -rsx --junitxml=junit/test-results.xml && \ - python3 tools/openblas_support.py --check_version" - displayName: 'Run 32-bit manylinux2010 Docker Build / Tests' - - task: PublishTestResults@2 - condition: succeededOrFailed() - inputs: - testResultsFiles: '**/test-*.xml' - failTaskOnFailedTests: true - testRunTitle: 'Publish test results for Python 3.8-32 bit full Linux' - - job: macOS - pool: - # NOTE: at time of writing, there is a danger - # that using an invalid vmIMage string for macOS - # image silently redirects to a Windows build on Azure; - # for now, use the only image name officially present in - # the docs even though i.e., numba uses another in their - # azure config for mac os -- Microsoft has indicated - # they will patch this issue - vmImage: macOS-10.14 - strategy: - maxParallel: 3 - matrix: - Python36: - PYTHON_VERSION: '3.6' - USE_OPENBLAS: '1' - Python36-ILP64: - PYTHON_VERSION: '3.6' - NPY_USE_BLAS_ILP64: '1' - USE_OPENBLAS: '1' - # Disable this job: the azure images do not create the problematic - # symlink from Accelerate to OpenBLAS. We still have the test - # at import to detect a buggy Accelerate, just cannot easily trigger - # it with azure. - # Accelerate: - # PYTHON_VERSION: '3.6' - # USE_OPENBLAS: '0' - - steps: - # the @0 refers to the (major) version of the *task* on Microsoft's - # end, not the order in the build matrix nor anything to do - # with version of Python selected - - task: UsePythonVersion@0 - inputs: - versionSpec: $(PYTHON_VERSION) - addToPath: true - architecture: 'x64' - - script: | - set -xe - [ -n "$USE_XCODE_10" ] && /bin/bash -c "sudo xcode-select -s /Applications/Xcode_10.app/Contents/Developer" - clang --version - displayName: 'report clang version' - # NOTE: might be better if we could avoid installing - # two C compilers, but with homebrew looks like we're - # now stuck getting the full gcc toolchain instead of - # just pulling in gfortran - - script: | - set -xe - # same version of gfortran as the open-libs and numpy-wheel builds - curl -L https://github.com/MacPython/gfortran-install/raw/master/archives/gfortran-4.9.0-Mavericks.dmg -o gfortran.dmg - GFORTRAN_SHA256=$(shasum -a 256 gfortran.dmg) - KNOWN_SHA256="d2d5ca5ba8332d63bbe23a07201c4a0a5d7e09ee56f0298a96775f928c3c4b30 gfortran.dmg" - if [ "$GFORTRAN_SHA256" != "$KNOWN_SHA256" ]; then - echo sha256 mismatch - exit 1 - fi - hdiutil attach -mountpoint /Volumes/gfortran gfortran.dmg - sudo installer -pkg /Volumes/gfortran/gfortran.pkg -target / - otool -L /usr/local/gfortran/lib/libgfortran.3.dylib - # Manually symlink gfortran-4.9 to plain gfortran for f2py. - # No longer needed after Feb 13 2020 as gfortran is already present - # and the attempted link errors. Keep this for future reference. - # ln -s /usr/local/bin/gfortran-4.9 /usr/local/bin/gfortran - displayName: 'make libgfortran available on mac os for openblas' - # use the pre-built openblas binary that most closely - # matches our MacOS wheel builds -- currently based - # primarily on file size / name details - - script: | - set -xe - target=$(python tools/openblas_support.py) - ls -lR $target - # manually link to appropriate system paths - cp $target/lib/lib* /usr/local/lib/ - cp $target/include/* /usr/local/include/ - otool -L /usr/local/lib/libopenblas* - displayName: 'install pre-built openblas' - condition: eq(variables['USE_OPENBLAS'], '1') - - script: python -m pip install --upgrade pip 'setuptools<49.2.0' wheel - displayName: 'Install tools' - - script: | - python -m pip install -r test_requirements.txt - python -m pip install vulture docutils sphinx==2.2.0 numpydoc - displayName: 'Install dependencies; some are optional to avoid test skips' - - script: /bin/bash -c "! vulture . --min-confidence 100 --exclude doc/,numpy/distutils/ | grep 'unreachable'" - displayName: 'Check for unreachable code paths in Python modules' - # prefer usage of clang over gcc proper - # to match likely scenario on many user mac machines - - script: python setup.py build -j 4 build_src --verbose-cfg install - displayName: 'Build NumPy' - env: - BLAS: None - LAPACK: None - ATLAS: None - CC: /usr/bin/clang - condition: eq(variables['USE_OPENBLAS'], '1') - - script: python setup.py build -j 4 build_ext --inplace install - displayName: 'Build NumPy without OpenBLAS' - env: - BLAS: None - LAPACK: None - ATLAS: None - CC: /usr/bin/clang - condition: eq(variables['USE_OPENBLAS'], '0') - # wait until after dev build of NumPy to pip - # install matplotlib to avoid pip install of older numpy - - script: python -m pip install matplotlib - displayName: 'Install matplotlib before refguide run' - - script: python runtests.py -g --refguide-check - displayName: 'Run Refuide Check' - condition: eq(variables['USE_OPENBLAS'], '1') - - script: python runtests.py -n --mode=full -- -rsx --junitxml=junit/test-results.xml - displayName: 'Run Full NumPy Test Suite' - condition: eq(variables['USE_OPENBLAS'], '1') - - bash: python tools/openblas_support.py --check_version - displayName: 'Verify OpenBLAS version' - condition: eq(variables['USE_OPENBLAS'], '1') - # import doesn't work when in numpy src directory , so do a pip dev install of build lib to test - - script: | - #!/bin/bash -v - set +e - python -c "import numpy as np" > test_output.log 2>&1 - check_output_code=$? - cat test_output.log - grep "buggy Accelerate backend" test_output.log - check_message=$? - if [ $check_output_code == 1 ] && [ $check_message == 0 ]; then exit 0; else exit 1;fi - displayName: "Check if numpy import fails with accelerate" - condition: eq(variables['USE_OPENBLAS'], '0') - - task: PublishTestResults@2 - condition: succeededOrFailed() - inputs: - testResultsFiles: '**/test-*.xml' - failTaskOnFailedTests: true - testRunTitle: 'Publish test results for Python 3.6 64-bit full Mac OS' - job: Windows pool: vmImage: 'VS2017-Win2016' strategy: maxParallel: 6 matrix: - Python38-32bit-fast: - PYTHON_VERSION: '3.8' - PYTHON_ARCH: 'x86' - TEST_MODE: fast - BITS: 32 Python36-64bit-full: PYTHON_VERSION: '3.6' PYTHON_ARCH: 'x64' @@ -217,19 +52,49 @@ stages: BITS: 64 NPY_USE_BLAS_ILP64: '1' OPENBLAS_SUFFIX: '64_' - PyPy36-32bit: - PYTHON_VERSION: 'PyPy3.6' - PYTHON_ARCH: 'x32' - TEST_MODE: fast - BITS: 32 + Python39-64bit-full: + PYTHON_VERSION: '3.9' + PYTHON_ARCH: 'x64' + TEST_MODE: full + BITS: 64 + NPY_USE_BLAS_ILP64: '1' + OPENBLAS_SUFFIX: '64_' + #PyPy36-32bit: + #PYTHON_VERSION: 'PyPy3.6' + #PYTHON_ARCH: 'x32' + #TEST_MODE: fast + #BITS: 32 steps: - template: azure-steps-windows.yml - job: Linux_PyPy3 pool: vmIMage: 'ubuntu-18.04' steps: - - script: source tools/pypy-test.sh - displayName: 'Run PyPy3 Build / Tests' + - task: UsePythonVersion@0 + inputs: + versionSpec: pypy3 + - script: | + python -c "import sys; print(sys.version)" + python -m pip install --upgrade "setuptools<49.2.0" + python -m pip install urllib3 + python -m pip install -r test_requirements.txt + basedir=$(python ./tools/openblas_support.py) + sudo cp -r $basedir/lib/* /usr/local/lib + sudo cp $basedir/include/* /usr/local/include + sudo ldconfig /usr/local/lib + displayName: 'Get OpenBLAS' + - script: | + python setup.py build_src --verbose-cfg bdist_wheel + # do the rest in a subdirectory so 'import numpy' works + pushd dist + python -mpip install ./numpy* + python ../tools/openblas_support.py --check_version + popd + displayName: 'Build, check OpenBLAS version' + - script: | + python runtests.py --debug-info --show-build-log -v -- -rsx \ + --junitxml=junit/test-results.xml --durations 10 + displayName: 'Test' - task: PublishTestResults@2 condition: succeededOrFailed() inputs: @@ -258,4 +123,3 @@ stages: testResultsFiles: '**/test-*.xml' failTaskOnFailedTests: true testRunTitle: 'Publish test results for gcc 4.8' - diff --git a/benchmarks/asv_compare.conf.json.tpl b/benchmarks/asv_compare.conf.json.tpl new file mode 100644 index 000000000000..1f339077c66d --- /dev/null +++ b/benchmarks/asv_compare.conf.json.tpl @@ -0,0 +1,95 @@ +// This config file is almost similar to 'asv.conf.json' except it contains +// custom tokens that can be substituted by 'runtests.py' and ASV, +// due to the necessity to add custom build options when `--bench-compare` +// is used. +{ + // The version of the config file format. Do not change, unless + // you know what you are doing. + "version": 1, + + // The name of the project being benchmarked + "project": "numpy", + + // The project's homepage + "project_url": "https://www.numpy.org/", + + // The URL or local path of the source code repository for the + // project being benchmarked + "repo": "..", + + // List of branches to benchmark. If not provided, defaults to "master" + // (for git) or "tip" (for mercurial). + "branches": ["HEAD"], + + // The DVCS being used. If not set, it will be automatically + // determined from "repo" by looking at the protocol in the URL + // (if remote), or by looking for special directories, such as + // ".git" (if local). + "dvcs": "git", + + // The tool to use to create environments. May be "conda", + // "virtualenv" or other value depending on the plugins in use. + // If missing or the empty string, the tool will be automatically + // determined by looking for tools on the PATH environment + // variable. + "environment_type": "virtualenv", + + // the base URL to show a commit for the project. + "show_commit_url": "https://github.com/numpy/numpy/commit/", + + // The Pythons you'd like to test against. If not provided, defaults + // to the current version of Python used to run `asv`. + "pythons": ["3.7"], + + // The matrix of dependencies to test. Each key is the name of a + // package (in PyPI) and the values are version numbers. An empty + // list indicates to just test against the default (latest) + // version. + "matrix": { + "Cython": [], + }, + + // The directory (relative to the current directory) that benchmarks are + // stored in. If not provided, defaults to "benchmarks" + "benchmark_dir": "benchmarks", + + // The directory (relative to the current directory) to cache the Python + // environments in. If not provided, defaults to "env" + // NOTE: changes dir name will requires update `generate_asv_config()` in + // runtests.py + "env_dir": "env", + + + // The directory (relative to the current directory) that raw benchmark + // results are stored in. If not provided, defaults to "results". + "results_dir": "results", + + // The directory (relative to the current directory) that the html tree + // should be written to. If not provided, defaults to "html". + "html_dir": "html", + + // The number of characters to retain in the commit hashes. + // "hash_length": 8, + + // `asv` will cache wheels of the recent builds in each + // environment, making them faster to install next time. This is + // number of builds to keep, per environment. + "build_cache_size": 8, + + "build_command" : [ + "python setup.py build {numpy_build_options}", + "PIP_NO_BUILD_ISOLATION=false python -mpip wheel --no-deps --no-index -w {build_cache_dir} {build_dir}" + ], + // The commits after which the regression search in `asv publish` + // should start looking for regressions. Dictionary whose keys are + // regexps matching to benchmark names, and values corresponding to + // the commit (exclusive) after which to start looking for + // regressions. The default is to start from the first commit + // with results. If the commit is `null`, regression detection is + // skipped for the matching benchmark. + // + // "regressions_first_commits": { + // "some_benchmark": "352cdf", // Consider regressions only after this commit + // "another_benchmark": null, // Skip regression detection altogether + // } +} diff --git a/doc/DISTUTILS.rst.txt b/doc/DISTUTILS.rst.txt index 01527374d182..c58a423c0bfb 100644 --- a/doc/DISTUTILS.rst.txt +++ b/doc/DISTUTILS.rst.txt @@ -394,37 +394,37 @@ and ``/**end repeat**/`` lines, which may also be nested using consecutively numbered delimiting lines such as ``/**begin repeat1`` and ``/**end repeat1**/``: -1. "/\**begin repeat "on a line by itself marks the beginning of -a segment that should be repeated. +1. ``/**begin repeat`` on a line by itself marks the beginning of + a segment that should be repeated. 2. Named variable expansions are defined using ``#name=item1, item2, item3, -..., itemN#`` and placed on successive lines. These variables are -replaced in each repeat block with corresponding word. All named -variables in the same repeat block must define the same number of -words. + ..., itemN#`` and placed on successive lines. These variables are + replaced in each repeat block with corresponding word. All named + variables in the same repeat block must define the same number of + words. 3. In specifying the repeat rule for a named variable, ``item*N`` is short- -hand for ``item, item, ..., item`` repeated N times. In addition, -parenthesis in combination with \*N can be used for grouping several -items that should be repeated. Thus, #name=(item1, item2)*4# is -equivalent to #name=item1, item2, item1, item2, item1, item2, item1, -item2# + hand for ``item, item, ..., item`` repeated N times. In addition, + parenthesis in combination with ``*N`` can be used for grouping several + items that should be repeated. Thus, ``#name=(item1, item2)*4#`` is + equivalent to ``#name=item1, item2, item1, item2, item1, item2, item1, + item2#``. -4. "\*/ "on a line by itself marks the end of the variable expansion -naming. The next line is the first line that will be repeated using -the named rules. +4. ``*/`` on a line by itself marks the end of the variable expansion + naming. The next line is the first line that will be repeated using + the named rules. 5. Inside the block to be repeated, the variables that should be expanded -are specified as ``@name@`` + are specified as ``@name@``. -6. "/\**end repeat**/ "on a line by itself marks the previous line -as the last line of the block to be repeated. +6. ``/**end repeat**/`` on a line by itself marks the previous line + as the last line of the block to be repeated. 7. A loop in the NumPy C source code may have a ``@TYPE@`` variable, targeted -for string substitution, which is preprocessed to a number of otherwise -identical loops with several strings such as INT, LONG, UINT, ULONG. The -``@TYPE@`` style syntax thus reduces code duplication and maintenance burden by -mimicking languages that have generic type support. + for string substitution, which is preprocessed to a number of otherwise + identical loops with several strings such as ``INT``, ``LONG``, ``UINT``, + ``ULONG``. The ``@TYPE@`` style syntax thus reduces code duplication and + maintenance burden by mimicking languages that have generic type support. The above rules may be clearer in the following template source example: @@ -464,13 +464,13 @@ The above rules may be clearer in the following template source example: /**end repeat**/ -The preprocessing of generically typed C source files (whether in NumPy +The preprocessing of generically-typed C source files (whether in NumPy proper or in any third party package using NumPy Distutils) is performed by `conv_template.py`_. -The type specific C files generated (extension: .c) +The type-specific C files generated (extension: ``.c``) by these modules during the build process are ready to be compiled. This form of generic typing is also supported for C header files (preprocessed -to produce .h files). +to produce ``.h`` files). .. _conv_template.py: https://github.com/numpy/numpy/blob/master/numpy/distutils/conv_template.py @@ -587,10 +587,6 @@ The header of a typical SciPy ``__init__.py`` is:: test = Tester().test bench = Tester().bench -Note that NumPy submodules still use a file named ``info.py`` in which the -module docstring and ``__all__`` dict are defined. These files will be removed -at some point. - Extra features in NumPy Distutils ''''''''''''''''''''''''''''''''' diff --git a/doc/RELEASE_WALKTHROUGH.rst.txt b/doc/RELEASE_WALKTHROUGH.rst.txt index 0ff9ff93352a..733f681af704 100644 --- a/doc/RELEASE_WALKTHROUGH.rst.txt +++ b/doc/RELEASE_WALKTHROUGH.rst.txt @@ -1,41 +1,37 @@ -This file contains a walkthrough of the NumPy 1.14.5 release on Linux, modified +This file contains a walkthrough of the NumPy 1.19.0 release on Linux, modified for building on azure and uploading to anaconda.org The commands can be copied into the command line, but be sure to -replace 1.14.5 by the correct version. +replace 1.19.0 by the correct version. This should be read together with the general directions in `releasing`. -Release Walkthrough -==================== - -Note that in the code snippets below, ``upstream`` refers to the root repository on -github and ``origin`` to a fork in your personal account. You may need to make adjustments -if you have not forked the repository but simply cloned it locally. You can -also edit ``.git/config`` and add ``upstream`` if it isn't already present. - +Release Preparation +=================== Backport Pull Requests ---------------------- Changes that have been marked for this release must be backported to the -maintenance/1.14.x branch. +maintenance/1.19.x branch. Update Release documentation ---------------------------- -The file ``doc/changelog/1.14.5-changelog.rst`` should be updated to reflect +The file ``doc/changelog/1.19.0-changelog.rst`` should be updated to reflect the final list of changes and contributors. This text can be generated by:: - $ python tools/changelog.py $GITHUB v1.14.4..maintenance/1.14.x > doc/changelog/1.14.5-changelog.rst + $ python tools/changelog.py $GITHUB v1.18.0..maintenance/1.19.x > doc/changelog/1.19.0-changelog.rst where ``GITHUB`` contains your github access token. This text may also be -appended to ``doc/release/1.14.5-notes.rst`` for release updates, though not -for new releases like ``1.14.0``, as the changelogs for ``*.0`` releases tend to be -excessively long. The ``doc/source/release.rst`` file should also be -updated with a link to the new release notes. These changes should be committed -to the maintenance branch, and later will be forward ported to master. +appended to ``doc/release/1.19.0-notes.rst`` for patch release, though not for +new releases like ``1.19.0``, as the changelogs for ``*.0`` releases tend to be +excessively long. The ``doc/source/release.rst`` file should also be updated +with a link to the new release notes. These changes should be committed to the +maintenance branch, and later will be forward ported to master. The changelog +should be reviewed for name duplicates or short names and the ``.mailmap`` file +updated if needed. Finish the Release Note @@ -46,25 +42,33 @@ Finish the Release Note This has changed now that we use ``towncrier``. See the instructions for creating the release note in ``doc/release/upcoming_changes/README.rst``. -Fill out the release note ``doc/release/1.14.5-notes.rst`` calling out +Fill out the release note ``doc/release/1.19.0-notes.rst`` calling out significant changes. +Release Walkthrough +==================== + +Note that in the code snippets below, ``upstream`` refers to the root repository on +github and ``origin`` to a fork in your personal account. You may need to make adjustments +if you have not forked the repository but simply cloned it locally. You can +also edit ``.git/config`` and add ``upstream`` if it isn't already present. + Prepare the release commit -------------------------- Checkout the branch for the release, make sure it is up to date, and clean the repository:: - $ git checkout maintenance/1.14.x - $ git pull upstream maintenance/1.14.x + $ git checkout maintenance/1.19.x + $ git pull upstream maintenance/1.19.x $ git submodule update $ git clean -xdfq Edit pavement.py and setup.py as detailed in HOWTO_RELEASE:: - $ gvim pavement.py setup.py - $ git commit -a -m"REL: NumPy 1.14.5 release." + $ gvim pavement.py setup.py # Generally only setup.py needs updating + $ git commit -a -m"REL: NumPy 1.19.0 release." Sanity check:: @@ -73,9 +77,7 @@ Sanity check:: Push this release directly onto the end of the maintenance branch. This requires write permission to the numpy repository:: - $ git push upstream maintenance/1.14.x - -As an example, see the 1.14.3 REL commit: ``_. + $ git push upstream HEAD Build source releases @@ -86,7 +88,7 @@ Paver is used to build the source releases. It will create the ``release`` and source releases in the latter. :: $ python3 -m cython --version # check for correct cython version - $ paver sdist # sdist will do a git clean -xdf, so we omit that + $ paver sdist # sdist will do a git clean -xdfq, so we omit that Build wheels @@ -94,55 +96,46 @@ Build wheels Trigger the wheels build by pointing the numpy-wheels repository at this commit. This can take up to an hour. The numpy-wheels repository is cloned from -``_. Start with a pull as the repo -may have been accessed and changed by someone else and a push will fail:: +``_. If this is the first release in +a series, start with a pull as the repo may have been accessed and changed by +someone else, then create a new branch for the series. If the branch already +exists skip this:: $ cd ../numpy-wheels + $ git co master $ git pull upstream master - $ git branch # only when starting new numpy version - $ git checkout v1.14.x # v1.14.x already existed for the 1.14.4 release + $ git branch v1.19.x -Edit the ``azure/posix.yml`` and ``azure/windows.yml`` files to make sure they -have the correct version, and put in the commit hash for the ``REL`` commit -created above for ``BUILD_COMMIT``, see an _example:: +Checkout the new branch and edit the ``azure-pipelines.yml`` and +``.travis.yml`` files to make sure they have the correct version, and put in +the commit hash for the ``REL`` commit created above for ``BUILD_COMMIT``. The +``azure/posix.yml`` and ``.travis.yml`` files may also need the Cython versions +updated to keep up with Python releases, but generally just do:: - $ gvim azure/posix.yml azure/windows.yml - $ git commit -a + $ git checkout v1.19.x + $ gvim azure-pipelines .travis.yml + $ git commit -a -m"NumPy 1.19.0 release." $ git push upstream HEAD Now wait. If you get nervous at the amount of time taken -- the builds can take a while -- you can check the build progress by following the links -provided at ``_ to check the +provided at ``_ to check the build status. Check if all the needed wheels have been built and -uploaded before proceeding. There should currently be 21 of them at -``_, 3 for Mac, 6 -for Windows, and 12 for Linux. +uploaded to the staging repository before proceeding. -.. example_: https://github.com/MacPython/numpy-wheels/pull/80/commits/cbf4af4 - -Note that sometimes builds, like tests, fail for unrelated reasons and -you will need to restart them. +Note that sometimes builds, like tests, fail for unrelated reasons and you will +need to rerun them. You will need to be logged in under 'numpy' to do this +on azure. Download wheels --------------- -When the wheels have all been successfully built, download them using the ``wheel-uploader`` -in the ``terryfy`` repository. The terryfy repository may be cloned from -``_ if you don't already have it. The -wheels can also be uploaded using the ``wheel-uploader``, but we prefer to -download all the wheels to the ``../numpy/release/installers`` directory and -upload later using ``twine``:: +When the wheels have all been successfully built and staged, download them from the +Anaconda staging directory using the ``tools/download-wheels.py`` script:: - $ cd ../terryfy - $ git pull upstream master - $ CDN_URL=https://anaconda.org/multibuild-wheels-staging/numpy/files - $ NPY_WHLS=../numpy/release/installers - $ ./wheel-uploader -u $CDN_URL -n -v -w $NPY_WHLS -t win numpy 1.14.5 - $ ./wheel-uploader -u $CDN_URL -n -v -w $NPY_WHLS -t manylinux1 numpy 1.14.5 - $ ./wheel-uploader -u $CDN_URL -n -v -w $NPY_WHLS -t macosx numpy 1.14.5 + $ cd ../numpy + $ python3 tools/download-wheels.py 1.19.0 -If you do this often, consider making CDN_URL and NPY_WHLS part of your default -environment. Generate the README files ------------------------- @@ -150,18 +143,16 @@ Generate the README files This needs to be done after all installers are downloaded, but before the pavement file is updated for continued development:: - $ cd ../numpy $ paver write_release Tag the release --------------- -Once the wheels have been built and downloaded without errors, go back to your -numpy repository in the maintenance branch and tag the ``REL`` commit, signing +Once the wheels have been built and downloaded without errors tag the ``REL`` commit, signing it with your gpg key:: - $ git tag -s v1.14.5 + $ git tag -s -m"NumPy 1.19.0 release" v1.19.0 You should upload your public gpg key to github, so that the tag will appear "verified" there. @@ -169,7 +160,7 @@ You should upload your public gpg key to github, so that the tag will appear Check that the files in ``release/installers`` have the correct versions, then push the tag upstream:: - $ git push upstream v1.14.5 + $ git push upstream v1.19.0 We wait until this point to push the tag because it is public and should not be changed after it has been pushed. @@ -185,9 +176,9 @@ Add another ``REL`` commit to the numpy maintenance branch, which resets the Create release notes for next release and edit them to set the version:: - $ cp doc/source/release/template.rst doc/source/release/1.14.6-notes.rst - $ gvim doc/source/release/1.14.6-notes.rst - $ git add doc/source/release/1.14.6-notes.rst + $ cp doc/source/release/template.rst doc/source/release/1.19.1-notes.rst + $ gvim doc/source/release/1.19.1-notes.rst + $ git add doc/source/release/1.19.1-notes.rst Add new release notes to the documentation release list:: @@ -195,26 +186,24 @@ Add new release notes to the documentation release list:: Commit the result:: - $ git commit -a -m"REL: prepare 1.14.x for further development" - $ git push upstream maintenance/1.14.x + $ git commit -a -m"REL: prepare 1.19.x for further development" + $ git push upstream HEAD Upload to PyPI -------------- Upload to PyPI using ``twine``. A recent version of ``twine`` of is needed -after recent PyPI changes, version ``1.11.0`` was used here. - -.. code-block:: sh +after recent PyPI changes, version ``3.1.1`` was used here:: $ cd ../numpy $ twine upload release/installers/*.whl - $ twine upload release/installers/numpy-1.14.5.zip # Upload last. + $ twine upload release/installers/numpy-1.19.0.zip # Upload last. -If one of the commands breaks in the middle, which is not uncommon, you may -need to selectively upload the remaining files because PyPI does not allow the -same file to be uploaded twice. The source file should be uploaded last to -avoid synchronization problems if pip users access the files while this is in +If one of the commands breaks in the middle, you may need to selectively upload +the remaining files because PyPI does not allow the same file to be uploaded +twice. The source file should be uploaded last to avoid synchronization +problems that might occur if pip users access the files while this is in process. Note that PyPI only allows a single source distribution, here we have chosen the zip archive. @@ -222,15 +211,16 @@ chosen the zip archive. Upload files to github ---------------------- -Go to ``_, there should be a ``v1.14.5 +Go to ``_, there should be a ``v1.19.0 tag``, click on it and hit the edit button for that tag. There are two ways to -add files, using an editable text window and as binary uploads. +add files, using an editable text window and as binary uploads. Cut and paste +the ``release/README.md`` file contents into the text window. You will probably +need to make some edits to get it to look right. Then -- Cut and paste the ``release/README.md`` file contents into the text window. -- Upload ``release/installers/numpy-1.14.5.tar.gz`` as a binary file. -- Upload ``release/installers/numpy-1.14.5.zip`` as a binary file. +- Upload ``release/installers/numpy-1.19.0.tar.gz`` as a binary file. +- Upload ``release/installers/numpy-1.19.0.zip`` as a binary file. - Upload ``release/README.rst`` as a binary file. -- Upload ``doc/changelog/1.14.5-changelog.rst`` as a binary file. +- Upload ``doc/changelog/1.19.0-changelog.rst`` as a binary file. - Check the pre-release button if this is a pre-releases. - Hit the ``{Publish,Update} release`` button at the bottom. @@ -255,7 +245,7 @@ If the release series is a new one, you will need to add a new section to the Otherwise, only the ``zip`` and ``pdf`` links should be updated with the new tag name:: - $ gvim doc/build/merge/index.html +/'tag v1.14' + $ gvim doc/build/merge/index.html +/'tag v1.19' You can "test run" the new documentation in a browser to make sure the links work:: @@ -265,7 +255,7 @@ work:: Once everything seems satisfactory, commit and upload the changes:: $ pushd doc/build/merge - $ git commit -am"Add documentation for v1.14.5" + $ git commit -am"Add documentation for v1.19.0" $ git push $ popd @@ -277,7 +267,7 @@ This assumes that you have forked ``_:: $ cd ../scipy.org $ git checkout master $ git pull upstream master - $ git checkout -b numpy-1.14.5 + $ git checkout -b numpy-1.19.0 $ gvim www/index.rst # edit the News section $ git commit -a $ git push origin HEAD @@ -300,13 +290,14 @@ Post-Release Tasks Checkout master and forward port the documentation changes:: - $ git checkout -b update-after-1.14.5-release - $ git checkout maintenance/1.14.x doc/source/release/1.14.5-notes.rst - $ git checkout maintenance/1.14.x doc/changelog/1.14.5-changelog.rst + $ git checkout -b post-1.19.0-release-update + $ git checkout maintenance/1.19.x doc/source/release/1.19.0-notes.rst + $ git checkout maintenance/1.19.x doc/changelog/1.19.0-changelog.rst + $ git checkout maintenance/1.19.x .mailmap # only if updated for release. $ gvim doc/source/release.rst # Add link to new notes - $ git add doc/changelog/1.14.5-changelog.rst doc/source/release/1.14.5-notes.rst + $ git add doc/changelog/1.19.0-changelog.rst doc/source/release/1.19.0-notes.rst $ git status # check status before commit - $ git commit -a -m"REL: Update master after 1.14.5 release." + $ git commit -a -m"REL: Update master after 1.19.0 release." $ git push origin HEAD Go to github and make a PR. diff --git a/doc/changelog/1.19.3-changelog.rst b/doc/changelog/1.19.3-changelog.rst new file mode 100644 index 000000000000..5e8dfa10b6ba --- /dev/null +++ b/doc/changelog/1.19.3-changelog.rst @@ -0,0 +1,31 @@ + +Contributors +============ + +A total of 8 people contributed to this release. People with a "+" by their +names contributed a patch for the first time. + +* Charles Harris +* Chris Brown + +* Daniel Vanzo + +* E. Madison Bray + +* Hugo van Kemenade + +* Ralf Gommers +* Sebastian Berg +* @danbeibei + + +Pull requests merged +==================== + +A total of 10 pull requests were merged for this release. + +* `#17298 `__: BLD: set upper versions for build dependencies +* `#17336 `__: BUG: Set deprecated fields to null in PyArray_InitArrFuncs +* `#17446 `__: ENH: Warn on unsupported Python 3.10+ +* `#17450 `__: MAINT: Update test_requirements.txt. +* `#17522 `__: ENH: Support for the NVIDIA HPC SDK nvfortran compiler +* `#17568 `__: BUG: Cygwin Workaround for #14787 on affected platforms +* `#17647 `__: BUG: Fix memory leak of buffer-info cache due to relaxed strides +* `#17652 `__: MAINT: Backport openblas_support from master. +* `#17653 `__: TST: Add Python 3.9 to the CI testing on Windows, Mac. +* `#17660 `__: TST: Simplify source path names in test_extending. diff --git a/doc/changelog/1.19.4-changelog.rst b/doc/changelog/1.19.4-changelog.rst new file mode 100644 index 000000000000..82632b990855 --- /dev/null +++ b/doc/changelog/1.19.4-changelog.rst @@ -0,0 +1,16 @@ + +Contributors +============ + +A total of 1 people contributed to this release. People with a "+" by their +names contributed a patch for the first time. + +* Charles Harris + +Pull requests merged +==================== + +A total of 2 pull requests were merged for this release. + +* `#17679 `__: MAINT: Add check for Windows 10 version 2004 bug. +* `#17680 `__: REV: Revert OpenBLAS to 1.19.2 version for 1.19.4 diff --git a/doc/neps/_static/nep-0041-type-sketch-no-fonts.svg b/doc/neps/_static/nep-0041-type-sketch-no-fonts.svg new file mode 100644 index 000000000000..3250396c530a --- /dev/null +++ b/doc/neps/_static/nep-0041-type-sketch-no-fonts.svg @@ -0,0 +1,1110 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + image/svg+xml + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/doc/neps/_static/nep-0041-type-sketch.svg b/doc/neps/_static/nep-0041-type-sketch.svg new file mode 100644 index 000000000000..9e597db9d9b2 --- /dev/null +++ b/doc/neps/_static/nep-0041-type-sketch.svg @@ -0,0 +1,523 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + image/svg+xml + + + + + + + + + + + + + Value Storage + Parameters andStorage options + Value Space andBehaviour + type + instance + + ABC + instance + + type + + DType + + + base dtype + element + + dtype + + element + + dtype + + Python type + + Python typewith ABC + NEP 41 Proposal + Alternative + + + diff --git a/doc/neps/_static/nep43-sketch-with-text.svg b/doc/neps/_static/nep43-sketch-with-text.svg new file mode 100644 index 000000000000..212cfe89cc5b --- /dev/null +++ b/doc/neps/_static/nep43-sketch-with-text.svg @@ -0,0 +1,1304 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + image/svg+xml + + + + + + + + + + + + + + + + + + + Loopdescriptors + + ResolverInput + + + + + → + + + + + + → + + >U5 + <U8 + + set descriptorsfor inner-loop… + <U5 + <U8 + <U13 + + User Input + + + + + + Input/OutputOperands + + + + + + → + + DType classesof the ArrayMethod + + + + + → + Promotion (if necessary) + + If provided + + + + + + → + + >U5 + S8 + >U5 + <S8 + Unicode + Unicode + Unicode + If not provided + Cast descriptorsto Loop DTypes + <U13 + + + + + + + + Registered + + resolve_descriptors + + + Perform operation with these descriptors(setup, inner-loop function, teardown) + NumPy + Registered or default + ArrayMethod + + Casting, Result Allocation and Outer Iterationdone by UFunc Machinery (within ArrayMethod) + Promoter + + Inputs + Output + … including correctoutput descriptor + + + ArrayMethod lookup + + diff --git a/doc/neps/_static/nep43-sketch.svg b/doc/neps/_static/nep43-sketch.svg new file mode 100644 index 000000000000..372c0ee46fc0 --- /dev/null +++ b/doc/neps/_static/nep43-sketch.svg @@ -0,0 +1,3009 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + image/svg+xml + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/doc/neps/nep-0000.rst b/doc/neps/nep-0000.rst index dcc7f4cf8b68..590976081723 100644 --- a/doc/neps/nep-0000.rst +++ b/doc/neps/nep-0000.rst @@ -234,7 +234,9 @@ Header Preamble Each NEP must begin with a header preamble. The headers must appear in the following order. Headers marked with ``*`` are -optional. All other headers are required. :: +optional. All other headers are required. + +.. code-block:: rst :Author: :Status: if the email address is included, and just +.. code-block:: rst + Random J. User if the address is not given. If there are multiple authors, each should be on diff --git a/doc/neps/nep-0002-warnfix.rst b/doc/neps/nep-0002-warnfix.rst index dfccd5ab802d..a1138b2f1b83 100644 --- a/doc/neps/nep-0002-warnfix.rst +++ b/doc/neps/nep-0002-warnfix.rst @@ -9,6 +9,8 @@ NEP 2 — A proposal to build numpy without warning with a big set of warning fl :Date: 2008-09-04 :Status: Deferred +.. highlight:: c + Executive summary ================= @@ -23,7 +25,9 @@ Warning flags ============= Each compiler detects a different set of potential errors. The baseline will -be gcc -Wall -W -Wextra. Ideally, a complete set would be nice:: +be gcc -Wall -W -Wextra. Ideally, a complete set would be nice: + +.. code-block:: bash -W -Wall -Wextra -Wstrict-prototypes -Wmissing-prototypes -Waggregate-return -Wcast-align -Wcast-qual -Wnested-externs -Wshadow -Wbad-function-cast @@ -67,9 +71,7 @@ When applied to a variable, one would get:: int foo(int * NPY_UNUSED(dummy)) -expanded to - -:: +expanded to:: int foo(int * __NPY_UNUSED_TAGGEDdummy __COMP_NPY_UNUSED) diff --git a/doc/neps/nep-0010-new-iterator-ufunc.rst b/doc/neps/nep-0010-new-iterator-ufunc.rst index 87617f414ea0..358018c46417 100644 --- a/doc/neps/nep-0010-new-iterator-ufunc.rst +++ b/doc/neps/nep-0010-new-iterator-ufunc.rst @@ -393,7 +393,9 @@ The proposed ‘order=’ flags become as follows: ‘K’ a layout equivalent to ‘C’ followed by some permutation of the axes, as close to the layout of the input(s) as possible (“Keep Layout”) === ===================================================================================== -Or as an enum:: +Or as an enum: + +.. code-block:: c /* For specifying array memory layout or iteration order */ typedef enum { @@ -416,7 +418,9 @@ parameter to control the layout of their output(s). The iterator can do automatic casting, and I have created a sequence of progressively more permissive casting rules. Perhaps for 2.0, NumPy -could adopt this enum as its preferred way of dealing with casting.:: +could adopt this enum as its preferred way of dealing with casting. + +.. code-block:: c /* For specifying allowed casting in operations which support it */ typedef enum { @@ -496,7 +500,9 @@ Proposed Iterator Memory Layout The following struct describes the iterator memory. All items are packed together, which means that different values of the flags, -ndim, and niter will produce slightly different layouts. :: +ndim, and niter will produce slightly different layouts. + +.. code-block:: c struct { /* Flags indicate what optimizations have been applied, and @@ -710,13 +716,17 @@ Construction and Destruction Returns NULL if there is an error, otherwise returns the allocated iterator. - To make an iterator similar to the old iterator, this should work.:: + To make an iterator similar to the old iterator, this should work. + + .. code-block:: c iter = NpyIter_New(op, NPY_ITER_READWRITE, NPY_CORDER, NPY_NO_CASTING, NULL, 0, NULL); If you want to edit an array with aligned ``double`` code, - but the order doesn't matter, you would use this.:: + but the order doesn't matter, you would use this. + + .. code-block:: c dtype = PyArray_DescrFromType(NPY_DOUBLE); iter = NpyIter_New(op, NPY_ITER_READWRITE | @@ -764,7 +774,9 @@ Construction and Destruction In ``op_axes[j][i]`` is stored either a valid axis of ``op[j]``, or -1 which means ``newaxis``. Within each ``op_axes[j]`` array, axes may not be repeated. The following example is how normal broadcasting - applies to a 3-D array, a 2-D array, a 1-D array and a scalar.:: + applies to a 3-D array, a 2-D array, a 1-D array and a scalar. + + .. code-block:: c npy_intp oa_ndim = 3; /* # iteration axes */ npy_intp op0_axes[] = {0, 1, 2}; /* 3-D operand */ @@ -1139,7 +1151,9 @@ Construction and Destruction If you want to reset both the ``iterindex`` range and the base pointers at the same time, you can do the following to avoid extra buffer copying (be sure to add the return code error checks - when you copy this code).:: + when you copy this code). + + .. code-block:: c /* Set to a trivial empty range */ NpyIter_ResetToIterIndexRange(iter, 0, 0); @@ -1190,7 +1204,9 @@ Construction and Destruction is used as the source for ``baseptrs``, it will point into a small buffer instead of the array and the inner iteration will be invalid. - The pattern for using nested iterators is as follows.:: + The pattern for using nested iterators is as follows: + + .. code-block:: c NpyIter *iter1, *iter1; NpyIter_IterNext_Fn iternext1, iternext2; @@ -1412,7 +1428,9 @@ Functions For Iteration non-NULL, the function may be safely called without holding the Python GIL. - The typical looping construct is as follows.:: + The typical looping construct is as follows: + + .. code-block:: c NpyIter_IterNext_Fn iternext = NpyIter_GetIterNext(iter, NULL); char **dataptr = NpyIter_GetDataPtrArray(iter); @@ -1422,7 +1440,9 @@ Functions For Iteration } while(iternext(iter)); When ``NPY_ITER_NO_INNER_ITERATION`` is specified, the typical - inner loop construct is as follows.:: + inner loop construct is as follows: + + .. code-block:: c NpyIter_IterNext_Fn iternext = NpyIter_GetIterNext(iter, NULL); char **dataptr = NpyIter_GetDataPtrArray(iter); @@ -1451,7 +1471,9 @@ Functions For Iteration to become zero when ``iternext()`` returns false, enabling the following loop construct. Note that if you use this construct, you should not pass ``NPY_ITER_GROWINNER`` as a flag, because it - will cause larger sizes under some circumstances.:: + will cause larger sizes under some circumstances: + + .. code-block:: c /* The constructor should have buffersize passed as this value */ #define FIXED_BUFFER_SIZE 1024 @@ -1571,7 +1593,9 @@ result. If the input is a reference type, this function will fail. To fix this, the code must be changed to specially handle writeable -references, and add ``NPY_ITER_WRITEABLE_REFERENCES`` to the flags.:: +references, and add ``NPY_ITER_WRITEABLE_REFERENCES`` to the flags: + +.. code-block:: c /* NOTE: This code has not been compiled/tested */ PyObject *CopyArray(PyObject *arr, NPY_ORDER order) diff --git a/doc/neps/nep-0011-deferred-ufunc-evaluation.rst b/doc/neps/nep-0011-deferred-ufunc-evaluation.rst index c40ca56d78bf..866a774d15b0 100644 --- a/doc/neps/nep-0011-deferred-ufunc-evaluation.rst +++ b/doc/neps/nep-0011-deferred-ufunc-evaluation.rst @@ -41,11 +41,15 @@ For an idea of how to get this kind of behavior in NumPy without changing the Python code, consider the C++ technique of expression templates. These can be used to quite arbitrarily rearrange expressions using -vectors or other data structures, example,:: +vectors or other data structures, example: + +.. code-block:: cpp A = B + C + D; -can be transformed into something equivalent to:: +can be transformed into something equivalent to: + +.. code-block:: cpp for(i = 0; i < A.size; ++i) { A[i] = B[i] + C[i] + D[i]; diff --git a/doc/neps/nep-0012-missing-data.rst b/doc/neps/nep-0012-missing-data.rst index 0adcea69ab2f..f47feadbd031 100644 --- a/doc/neps/nep-0012-missing-data.rst +++ b/doc/neps/nep-0012-missing-data.rst @@ -957,6 +957,8 @@ so the later code will raise exceptions as desired. C Implementation Details ************************ +.. highlight:: c + The first version to implement is the array masks, because it is the more general approach. The mask itself is an array, but since it is intended to never be directly accessible from Python, it won't @@ -1159,32 +1161,32 @@ Acknowledgments In addition to feedback from Travis Oliphant and others at Enthought, this NEP has been revised based on a great deal of feedback from the NumPy-Discussion mailing list. The people participating in -the discussion are:: - - Nathaniel Smith - Robert Kern - Charles Harris - Gael Varoquaux - Eric Firing - Keith Goodman - Pierre GM - Christopher Barker - Josef Perktold - Ben Root - Laurent Gautier - Neal Becker - Bruce Southey - Matthew Brett - Wes McKinney - Lluís - Olivier Delalleau - Alan G Isaac - E. Antero Tammi - Jason Grout - Dag Sverre Seljebotn - Joe Harrington - Gary Strangman - Chris Jordan-Squire - Peter +the discussion are: + +- Nathaniel Smith +- Robert Kern +- Charles Harris +- Gael Varoquaux +- Eric Firing +- Keith Goodman +- Pierre GM +- Christopher Barker +- Josef Perktold +- Ben Root +- Laurent Gautier +- Neal Becker +- Bruce Southey +- Matthew Brett +- Wes McKinney +- Lluís +- Olivier Delalleau +- Alan G Isaac +- E. Antero Tammi +- Jason Grout +- Dag Sverre Seljebotn +- Joe Harrington +- Gary Strangman +- Chris Jordan-Squire +- Peter I apologize if I missed anyone. diff --git a/doc/neps/nep-0013-ufunc-overrides.rst b/doc/neps/nep-0013-ufunc-overrides.rst index 5c28e9446370..698e45738708 100644 --- a/doc/neps/nep-0013-ufunc-overrides.rst +++ b/doc/neps/nep-0013-ufunc-overrides.rst @@ -102,7 +102,7 @@ Take this example of ufuncs interoperability with sparse matrices.:: [ 4, 1, 4]], dtype=int64) In [5]: np.multiply(a, bsp) # Returns NotImplemented to user, bad! - Out[5]: NotImplemted + Out[5]: NotImplemented Returning :obj:`NotImplemented` to user should not happen. Moreover:: diff --git a/doc/neps/nep-0025-missing-data-3.rst b/doc/neps/nep-0025-missing-data-3.rst index 19be3cf1e591..ffe208c98070 100644 --- a/doc/neps/nep-0025-missing-data-3.rst +++ b/doc/neps/nep-0025-missing-data-3.rst @@ -132,6 +132,8 @@ Some example use cases: dtype C-level API extensions ============================ +.. highlight:: c + The `PyArray_Descr`_ struct gains the following new fields:: void * NA_value; @@ -357,6 +359,8 @@ so convenient as that for NaN values now, but then, NaN values don't have their own global singleton.) So for now we stick to scalar indexing just returning ``np.NA``, but this can be revisited if anyone objects. +.. highlight:: python + Python API for generic NA support ================================= diff --git a/doc/neps/nep-0029-deprecation_policy.rst b/doc/neps/nep-0029-deprecation_policy.rst index dbead1b9b550..957674ee6e20 100644 --- a/doc/neps/nep-0029-deprecation_policy.rst +++ b/doc/neps/nep-0029-deprecation_policy.rst @@ -77,7 +77,7 @@ release in November 2020 should support Python 3.7 and newer. The current Python release cadence is 18 months so a 42 month window ensures that there will always be at least two minor versions of Python in the window. The window is extended 6 months beyond the anticipated two-release -interval for Python to provides resilience against small fluctuations / +interval for Python to provide resilience against small fluctuations / delays in its release schedule. Because Python minor version support is based only on historical @@ -111,8 +111,10 @@ Jun 23, 2020 3.7+ 1.15+ Jul 23, 2020 3.7+ 1.16+ Jan 13, 2021 3.7+ 1.17+ Jul 26, 2021 3.7+ 1.18+ -Dec 26, 2021 3.8+ 1.18+ -Apr 14, 2023 3.9+ 1.18+ +Dec 22, 2021 3.7+ 1.19+ +Dec 26, 2021 3.8+ 1.19+ +Jun 21, 2022 3.8+ 1.20+ +Apr 14, 2023 3.9+ 1.20+ ============ ====== ===== @@ -127,7 +129,9 @@ Drop Schedule On Jul 23, 2020 drop support for Numpy 1.15 (initially released on Jul 23, 2018) On Jan 13, 2021 drop support for Numpy 1.16 (initially released on Jan 13, 2019) On Jul 26, 2021 drop support for Numpy 1.17 (initially released on Jul 26, 2019) + On Dec 22, 2021 drop support for Numpy 1.18 (initially released on Dec 22, 2019) On Dec 26, 2021 drop support for Python 3.7 (initially released on Jun 27, 2018) + On Jun 21, 2022 drop support for Numpy 1.19 (initially released on Jun 20, 2020) On Apr 14, 2023 drop support for Python 3.8 (initially released on Oct 14, 2019) @@ -255,6 +259,8 @@ Code to generate support and drop schedule tables :: Jan 13, 2019: Numpy 1.16 Jul 26, 2019: Numpy 1.17 Oct 14, 2019: Python 3.8 + Dec 22, 2019: Numpy 1.18 + Jun 20, 2020: Numpy 1.19 """ releases = [] @@ -274,8 +280,12 @@ Code to generate support and drop schedule tables :: releases = sorted(releases, key=lambda x: x[0]) - minpy = '3.9+' - minnum = '1.18+' + + py_major,py_minor = sorted([int(x) for x in r[2].split('.')] for r in releases if r[1] == 'Python')[-1] + minpy = f"{py_major}.{py_minor+1}+" + + num_major,num_minor = sorted([int(x) for x in r[2].split('.')] for r in releases if r[1] == 'Numpy')[-1] + minnum = f"{num_major}.{num_minor+1}+" toprint_drop_dates = [''] toprint_support_table = [] @@ -289,14 +299,14 @@ Code to generate support and drop schedule tables :: minnum = v+'+' else: minpy = v+'+' - - for e in toprint_drop_dates[::-1]: + print("On next release, drop support for Python 3.5 (initially released on Sep 13, 2015)") + for e in toprint_drop_dates[-4::-1]: print(e) print('============ ====== =====') print('Date Python NumPy') print('------------ ------ -----') - for e in toprint_support_table[::-1]: + for e in toprint_support_table[-4::-1]: print(e) print('============ ====== =====') diff --git a/doc/neps/nep-0036-fair-play.rst b/doc/neps/nep-0036-fair-play.rst new file mode 100644 index 000000000000..32e5e1e75c03 --- /dev/null +++ b/doc/neps/nep-0036-fair-play.rst @@ -0,0 +1,177 @@ +================== +NEP 36 — Fair play +================== + +:Author: Stéfan van der Walt +:Status: Draft +:Type: Informational +:Created: 2019-10-24 +:Resolution: Draft + + +Abstract +-------- + +This document sets out Rules of Play for companies and outside +developers that engage with the NumPy project. It covers: + +- Restrictions on use of the NumPy name +- How and whether to publish a modified distribution +- How to make us aware of patched versions + +Companies and developers will know after reading this NEP what kinds +of behavior the community would like to see, and which we consider +troublesome, bothersome, and unacceptable. + +Motivation +---------- + +We sometimes learn of NumPy versions modified and circulated by outsiders. +These patched versions can cause problems for the NumPy community. + +- In December 2018, a `bug report + `__ was filed against + `np.erf` -- a function that didn't exist in the NumPy distribution. + It came to light that a company had published a NumPy version with + an extended API footprint. After several months of discussion, the + company agreed to make its patches public, and we added a label to + the NumPy issue tracker to identify issues pertaining to that + distribution. + +- In another case, after a security issue (CVE-2019-6446) was filed + against NumPy, distributions put in their own fixes, most often by + changing a default keyword value. As a result the NumPy API was + inconsistent across distributions. + +When issues arise in cases like these, our developers waste time +identifying the problematic release, locating alterations, +and determining an appropriate course of action. + +During a community call on `October 16th, 2019 +`__ +the community resolved to draft guidelines on the distribution of +modified NumPy versions. + +Scope +----- + +This document aims to define a minimal set of rules that, when +followed, will be considered good-faith efforts in line with the +expectations of the NumPy developers. + +Our hope is that developers who feel they need to modify NumPy will +first consider contributing to the project, or use one of several existing +mechanisms for extending our APIs and for operating on +externally defined array objects. + +When in doubt, please `talk to us first +`__. We may suggest an alternative; at +minimum, we'll be prepared. + +Fair play rules +--------------- + +1. Do not reuse the NumPy name for projects not developed by the NumPy + community. + + At time of writing, there are only a handful of ``numpy``-named + packages developed by the community, including ``numpy``, + ``numpy-financial``, and ``unumpy``. We ask that external packages not + include the phrase ``numpy``, i.e., avoid names such as + ``mycompany_numpy``. + + To be clear, this rule only applies to modules (package names); it + is perfectly acceptable to have a *submodule* of your own library + named ``mylibrary.numpy``. + + NumPy is a trademark owned by NumFOCUS. + +2. Do not republish modified versions of NumPy. + + Modified versions of NumPy make it very difficult for the + developers to address bug reports, since we typically do not know + which parts of NumPy have been modified. + + If you have to break this rule (and we implore you not + to!), then make it clear in the ``__version__`` tag that + you have modified NumPy, e.g.:: + + >>> print(np.__version__) + '1.17.2+mycompany.15` + + We understand that minor patches are often required to make a + library work under a certain distribution. E.g., Debian may patch + NumPy so that it searches for optimized BLAS libraries in the + correct locations. But we ask that no substantive changes are + made. + +3. Do not extend NumPy's API footprint. + + If you absolutely have to break rule two, please do not add + additional functions to the namespace. NumPy's API is already + quite large, and we are working hard to reduce it where feasible. + Having additional functions exposed in distributed versions is + confusing for users and developers alike. + +4. *DO* use official mechanism to engage with the API. + + Protocols such as `__array_ufunc__ + `__ and + `__array_function__ + `__ + were designed to help external packages interact more easily with + NumPy. E.g., the latter allows objects from foreign libraries to + pass through NumPy unharmed. We actively encourage using any of + these "officialy sanctioned" mechanisms for overriding or + interacting with NumPy. + + If these mechanisms are deemed insufficient, please start a + discussion on the mailing list before monkeypatching NumPy. + +Questions and answers +------------------- + +**Q:** We would like to distribute an optimized version of NumPy that +utilizes special instructions for our company's CPU. You recommend +against that, so what are we to do? + +**A:** Please consider including the patches required in the official +NumPy repository. Not only do we encourage such contributions, but we +already have optimized loops for some platforms available. + +**Q:** We would like to ship a much faster version of FFT than NumPy +provides, but NumPy has no mechanism for overriding its FFT routines. +How do we proceed? + +**A:** There are two solutions that we approve of: let the users +install your optimizations using a piece of code, such as:: + + from my_company_accel import patch_numpy_fft + patch_numpy_fft() + +or have your distribution automatically perform the above, but print a +message to the terminal clearly stating what is happening:: + + We are now patching NumPy for optimal performance under MyComp + Special Platform. Please direct all bug reports to + https://mycomp.com/numpy-bugs + +If you require additional mechanisms for overriding code, please +discuss this with the development team on the mailing list. + +**Q:** We would like to distribute NumPy with faster linear algebra +routines. Are we allowed to do this? + +**A:** Yes, this is explicitly supported by linking to a different +version of BLAS. + +Discussion +---------- + +References and footnotes +------------------------ + +Copyright +--------- + +This document has been placed in the public domain. diff --git a/doc/neps/nep-0040-legacy-datatype-impl.rst b/doc/neps/nep-0040-legacy-datatype-impl.rst index c247e3d62d10..39889109de40 100644 --- a/doc/neps/nep-0040-legacy-datatype-impl.rst +++ b/doc/neps/nep-0040-legacy-datatype-impl.rst @@ -13,15 +13,15 @@ NEP 40 — Legacy Datatype Implementation in NumPy .. note:: - This NEP is part of a series of NEPs encompassing first information - about the previous dtype implementation and issues with it in NEP 40 - (this document). - :ref:`NEP 41 ` then provides an overview and generic design choices - for the refactor. - Further NEPs 42 and 43 go into the technical details of the datatype - and universal function related internal and external API changes. - In some cases it may be necessary to consult the other NEPs for a full - picture of the desired changes and why these changes are necessary. + This NEP is first in a series: + + - NEP 40 (this document) explains the shortcomings of NumPy's dtype implementation. + + - :ref:`NEP 41 ` gives an overview of our proposed replacement. + + - :ref:`NEP 42 ` describes the new design's datatype-related APIs. + + - NEP 43 describes the new design's API for universal functions. @@ -44,6 +44,8 @@ of the current implementation of dtypes as well as a discussion. In many cases subsections will be split roughly to first describe the current implementation and then follow with an "Issues and Discussion" section. +.. _parametric-datatype-discussion: + Parametric Datatypes ^^^^^^^^^^^^^^^^^^^^ @@ -253,6 +255,8 @@ types such as ``np.inexact`` (see figure below). In fact, some control flow within NumPy currently uses ``issubclass(a.dtype.type, np.inexact)``. +.. _nep-0040_dtype-hierarchy: + .. figure:: _static/nep-0040_dtype-hierarchy.png **Figure:** Hierarchy of NumPy scalar types reproduced from the reference @@ -335,7 +339,7 @@ Each of these signatures is associated with a single inner-loop function defined in C, which does the actual calculation, and may be called multiple times. The main step in finding the correct inner-loop function is to call a -:c:type:`PyUFunc_TypeResolutionFunc` which retrieves the input dtypes from +:c:type:`PyUFunc_TypeResolutionFunc` which retrieves the input dtypes from the provided input arrays and will determine the full type signature (including output dtype) to be executed. @@ -366,7 +370,7 @@ It is currently only possible for user defined functions to be found/resolved if any of the inputs (or the outputs) has the user datatype, since it uses the `OO->O` signature. For example, given that a ufunc loop to implement ``fraction_divide(int, int) --> Fraction`` has been implemented, +-> Fraction`` has been implemented, the call ``fraction_divide(4, 5)`` (with no specific output dtype) will fail because the loop that includes the user datatype ``Fraction`` (as output) can only be found if any of @@ -572,7 +576,7 @@ Related Work ------------ * Julia types are an interesting blueprint for a type hierarchy, and define - abstract and concrete types [julia-types]_. + abstract and concrete types [julia-types]_. * In Julia promotion can occur based on abstract types. If a promoter is defined, it will cast the inputs and then Julia can then retry to find @@ -607,7 +611,7 @@ the following provides a subset for more recent ones: * https://hackmd.io/ok21UoAQQmOtSVk6keaJhw and https://hackmd.io/s/ryTFaOPHE (2019-04-30) Proposals for subclassing implementation approach. - + * Discussion about the calling convention of ufuncs and need for more powerful UFuncs: https://github.com/numpy/numpy/issues/12518 diff --git a/doc/neps/nep-0041-improved-dtype-support.rst b/doc/neps/nep-0041-improved-dtype-support.rst index 6dc4ea50c6fd..d7a08562d9c4 100644 --- a/doc/neps/nep-0041-improved-dtype-support.rst +++ b/doc/neps/nep-0041-improved-dtype-support.rst @@ -15,15 +15,15 @@ NEP 41 — First step towards a new Datatype System .. note:: - This NEP is part of a series of NEPs encompassing first information - about the previous dtype implementation and issues with it in - :ref:`NEP 40 `. - NEP 41 (this document) then provides an overview and generic design - choices for the refactor. - Further NEPs 42 and 43 go into the technical details of the datatype - and universal function related internal and external API changes. - In some cases it may be necessary to consult the other NEPs for a full - picture of the desired changes and why these changes are necessary. + This NEP is second in a series: + + - :ref:`NEP 40 ` explains the shortcomings of NumPy's dtype implementation. + + - NEP 41 (this document) gives an overview of our proposed replacement. + + - :ref:`NEP 42 ` describes the new design's datatype-related APIs. + + - NEP 43 describes the new design's API for universal functions. Abstract @@ -412,27 +412,28 @@ multiple development stages are required: * Phase II: Incrementally define or rework API - * Create a new and easily extensible API for defining new datatypes - and related functionality. (NEP 42) - - * Incrementally define all necessary functionality through the new API (NEP 42): - - * Defining operations such as ``np.common_type``. - * Allowing to define casting between datatypes. - * Add functionality necessary to create a numpy array from Python scalars - (i.e. ``np.array(...)``). - * … - - * Restructure how universal functions work (NEP 43), in order to: - - * make it possible to allow a `~numpy.ufunc` such as ``np.add`` to be - extended by user-defined datatypes such as Units. - - * allow efficient lookup for the correct implementation for user-defined - datatypes. - - * enable reuse of existing code. Units should be able to use the - normal math loops and add additional logic to determine output type. + * Incrementally define all necessary functionality through methods and + properties on the DType (NEP 42): + + * The properties of the class hierarchy and DType class itself, + including methods not covered by the following, most central, points. + * The functionality that will support dtype casting using ``arr.astype()`` + and casting related operations such as ``np.common_type``. + * The implementation of item access and storage, and the way shape and + dtype are determined when creating an array with ``np.array()`` + * Create a public C-API to define new DTypes. + + * Restructure how universal functions work (NEP 43), to allow extending + a `~numpy.ufunc` such as ``np.add`` for user-defined datatypes + such as Units: + + * Refactor how the low-level C functions are organized to make it + extensible and flexible enough for complicated DTypes such as Units. + * Implement registration and efficient lookup for these low-level C + functions as defined by the user. + * Define how promotion will be used to implement behaviour when casting + is required. For example ``np.float64(3) + np.int32(3)`` promotes the + ``int32`` to a ``float64``. * Phase III: Growth of NumPy and Scientific Python Ecosystem capabilities: @@ -583,7 +584,7 @@ special methods move from the dtype instances to methods on the new DType class. This is the typical design pattern used in Python. Organizing these methods and information in a more Pythonic way provides a solid foundation for refining and extending the API in the future. -The current API cannot be extended due to how it is exposed publically. +The current API cannot be extended due to how it is exposed publicly. This means for example that the methods currently stored in ``PyArray_ArrFuncs`` on each datatype (see :ref:`NEP 40 `) will be defined differently in the future and @@ -620,6 +621,49 @@ While DType and Scalar describe the same concept/type (e.g. an `int64`), it seems practical to split out the information and functionality necessary for numpy into the DType class. +The dtype instances provide parameters and storage options +"""""""""""""""""""""""""""""""""""""""""""""""""""""""""" + +From a computer science point of view a type defines the *value space* +(all possible values its instances can take) and their *behaviour*. +As proposed in this NEP, the DType class defines value space and behaviour. +The ``dtype`` instance can be seen as part of the value, so that the typical +Python ``instance`` corresponds to ``dtype + element`` (where *element* is the +data stored in the array). +An alternative view would be to define value space and behaviour on the +``dtype`` instances directly. +These two options are presented in the following figure and compared to +similar Python implementation patterns: + +.. image:: _static/nep-0041-type-sketch-no-fonts.svg + +The difference is in how parameters, such as string length or the datetime +units (``ms``, ``ns``, ...), and storage options, such as byte-order, are handled. +When implementing a Python (scalar) ``type`` parameters, for example the datetimes +unit, will be stored in the instance. +This is the design NEP 42 tries to mimic, however, the parameters are now part +of the dtype instance, meaning that part of the data stored in the instance +is shared by all array elements. +As mentioned previously, this means that the Python ``instance`` corresponds +to the ``dtype + element`` stored in a NumPy array. + +An more advanced approach in Python is to use a class factory and an abstract +base class (ABC). +This allows moving the parameter into the dynamically created ``type`` and +behaviour implementation may be specific to those parameters. +An alternative approach might use this model and implemented behaviour +directly on the ``dtype`` instance. + +We believe that the version as proposed here is easier to work with and understand. +Python class factories are not commonly used and NumPy does not use code +specialized for dtype parameters or byte-orders. +Making such specialization easier to implement such specialization does not +seem to be a priority. +One result of this choice is that some DTypes may only have a singleton instance +if they have no parameters or storage variation. +However, all of the NumPy dtypes require dynamically created instances due +to allowing metadata to be attached. + Scalars should not be instances of the datatypes (2) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/doc/neps/nep-0042-new-dtypes.rst b/doc/neps/nep-0042-new-dtypes.rst index b37555892b62..ff92e1612b09 100644 --- a/doc/neps/nep-0042-new-dtypes.rst +++ b/doc/neps/nep-0042-new-dtypes.rst @@ -1,9 +1,12 @@ -======================================== -NEP 42 — Implementation of New DataTypes -======================================== +.. _NEP42: -:title: Extensible Datatypes for NumPy +============================================================================== +NEP 42 — New and extensible DTypes +============================================================================== + +:title: New and extensible DTypes :Author: Sebastian Berg +:Author: Ben Nathanson :Author: Marten van Kerkwijk :Status: Draft :Type: Standard @@ -12,720 +15,498 @@ NEP 42 — Implementation of New DataTypes .. note:: - This NEP is part of a series of NEPs encompassing first information - about the previous dtype implementation and issues with it in - :ref:`NEP 40 `. - :ref:`NEP 41 ` then provides an overview and generic design - choices for the refactor. NEPs 42 (this document) - and 43 go into the technical details of the internal and external - API changes related to datatypes and universal functions, respectively. - In some cases it may be necessary to consult the other NEPs for a full - picture of the desired changes and why these changes are necessary. + This NEP is third in a series: + + - :ref:`NEP40` explains the shortcomings of NumPy's dtype implementation. + + - :ref:`NEP41` gives an overview of our proposed replacement. + - NEP 42 (this document) describes the new design's datatype-related APIs. + - :ref:`NEP43` describes the new design's API for universal functions. + + +****************************************************************************** Abstract --------- - -NEP 40 and 41 detailed the need for the creation of a new datatype system within -NumPy to better serve downstream use-cases and improve the maintainability -and the extensibility of NumPy. -A main issue with the current dtype API is that datatypes are written as -a single Python class with special instances for each of the actual datatypes. -While this certainly has been a practical approach in implementing numerical -datatypes, it does not allow to naturally split up logic. For example, -functions such as ``can_cast`` have explicit logic for each datatype. -Because of this monolithic code structure user-defined datatypes do not have -the same capabilities as NumPy datatypes have. -The current structure also makes understanding and modifying datatypes harder. -The current datatypes are not well encapsulated, so modifications targeting -a single datatype inevitably touch code involving others. -As detailed in NEP 41, the desired general design is to create classes for -each of the NumPy-provided datatypes, meaning that ``np.dtype("float64")`` -returns an instance of a ``Float64`` class which is a subclass of ``np.dtype``. -``np.dtype[float64]`` will also be used to denote this class. -This will allow moving all logic into special methods on the ``np.dtype`` -subclasses. This ``DType`` class would then serve as the central -extension point for adding new dtypes to NumPy. - -This document proposes the new API for the datatypes itself. -A second proposal NEP 43 details proposed changes to the universal -functions. -Note that only the implementation of both NEPs will provide the desired full -functionality. +****************************************************************************** +NumPy's dtype architecture is monolithic -- each dtype is an instance of a +single class. There's no principled way to expand it for new dtypes, and the +code is difficult to read and maintain. -.. note:: +As :ref:`NEP 41 ` explains, we are proposing a new architecture that is +modular and open to user additions. dtypes will derive from a new ``DType`` +class serving as the extension point for new types. ``np.dtype("float64")`` +will return an instance of a ``Float64`` class, a subclass of root class +``np.dtype``. - At this time this NEP is in a preliminary state. Both internal and - external API may be adapted based on user input or implementation needs. - The general design principles and choices, while provisional, should not - be expected to change dramatically. +This NEP is one of two that lay out the design and API of this new +architecture. This NEP addresses dtype implementation; :ref:`NEP 43 ` addresses +universal functions. +.. note:: -Detailed Description --------------------- + Details of the private and external APIs may change to reflect user + comments and implementation constraints. The underlying principles and + choices should not change significantly. -NEP 41 layed out the creation of a class hierarchy for datatypes using the -new DType classes to provide all necessary implementations. -This NEP defines the specific choice of API necessary to define new DTypes. -Here, these are suggested as C-API slots; however, conceptually these -translate identically to Python methods. -Additionally, the NEP proposes to implement the notion of *abstract* DTypes. -Further, we detail – in part – how the proposed methods (C-API slots) -enable all necessary use cases. +****************************************************************************** +Motivation and scope +****************************************************************************** -Each section will begin with a short motivation of the issue or what -problem is addressed. This is followed by a description of the proposed -design choice, and then may list alternatives. +Our goal is to allow user code to create fully featured dtypes for a broad +variety of uses, from physical units (such as meters) to domain-specific +representations of geometric objects. :ref:`NEP 41 ` describes a number +of these new dtypes and their benefits. +Any design supporting dtypes must consider: -Nomenclature -"""""""""""" +- How shape and dtype are determined when an array is created +- How array elements are stored and accessed +- The rules for casting dtypes to other dtypes -As a brief note on nomenclature, it should be noted that ``dtype`` normally -denotes the dtype *instance*, which is the object attached to a numpy array. -On the other hand the ``DType`` class is the subclass of ``np.dtype``. -On the C-level we currently use the name ``descriptor`` or ``descr`` -interchangeably with *dtype instance*. ``descriptor`` or ``descr`` will be -used in proposed C-API names to differentiate dtype instances from DType -classes more clearly. -Note that the notion of dtype class is currently represented mainly as -the ``dtype.num`` and ``dtype.char``. -Please see the :ref:`dtype hierarchy figure ` for an -illustration of this distinction. +In addition: -There are currently classes in NumPy for numeric types e.g. -``np.float64``; however, -these are not DTypes but the corresponding scalar classes -(see also NEP 40 and 41 for discussion on why these are largely unrelated to -the proposed changes). +- We want dtypes to comprise a class hierarchy open to new types and to + subhierarchies, as motivated in :ref:`NEP 41 `. +And to provide this, -Proposed access to DType class -"""""""""""""""""""""""""""""" +- We need to define a user API. -**Motivation:** +All these are the subjects of this NEP. -Currently we often call ``np.dtype`` to create the dtype instance -corresponding to a given scalar type (e.g. ``np.dtype(np.int64)``). -Adding the DType classes may require a way to access the classes conveniently. +- The class hierarchy, its relation to the Python scalar types, and its + important attributes are described in `nep42_DType class`_. -**Description:** +- The functionality that will support dtype casting is described in `Casting`_. -To avoid duplication, but also to expose the classes conveniently to users -we propose the addition of:: +- The implementation of item access and storage, and the way shape and dtype + are determined when creating an array, are described in :ref:`nep42_array_coercion`. - np.dtype[np.int64] +- The functionality for users to define their own DTypes is described in + `Public C-API`_. -as a class getter. This can work both for user and NumPy DTypes, -although, in many cases a library may choose to provide a more direct -way to access the specific DType class. -This method may initially be limited to concrete DTypes. -The main reason for this choice is to provide a single -clear and future-proof way to find the DType class given the -Python (scalar) class. +The API here and in :ref:`NEP 43 ` is entirely on the C side. A Python-side version +will be proposed in a future NEP. A future Python API is expected to be +similar, but provide a more convenient API to reuse the functionality of +existing DTypes. It could also provide shorthands to create structured DTypes +similar to Python's +`dataclasses `_. -This should not be a common operation, so providing this class getter reduces -the pressure of adding the new DType classes into the namespace. -*Note: This is currently a possible extension and not yet decided.* +****************************************************************************** +Backward compatibility +****************************************************************************** +The disruption is expected to be no greater than that of a typical NumPy +release. -Hierarchy of DTypes and Abstract DTypes -""""""""""""""""""""""""""""""""""""""" +- The main issues are noted in :ref:`NEP 41 ` and will mostly affect + heavy users of the NumPy C-API. -**Motivation:** -The creation of DType classes has already been decided in NEP 41. -Here we discuss the notion of **abstract** DTypes. -There are multiple reasons for this: +- Eventually we will want to deprecate the API currently used for creating + user-defined dtypes. -1. It allows the definition of a class hierarchy, in principle allowing checks like - ``isinstance(np.dtype("float64"), np.inexact)``. - **This hierarchy may be a prerequisite to implementing dispatching - for universal functions (NEP 43)** -2. Abstract DTypes can enable code such as - ``arr.astype(Complex)`` to express the desire to cast to a - complex data type of unspecified precision. -3. It anticipates the creation of families of DTypes by users. - For example allowing the creation of an abstract ``Unit`` class with a concrete - ``Float64Unit``. In which case ``Unit(np.float64, "m")`` could be - identical to ``Float64Unit("m")``. - -A very concrete example is the current Pandas ``Categorical`` DType, -which may benefit from abstraction to allow the differentiation of -a categorical of integer values and one of general object values. -The reason for this is that we may want to reject -``common_dtype(CategoricalInt64, String)``, but accept -``common_dtype(CategoricalObject, String)`` to be the ``object`` DType. -The current Pandas ``Categorical`` DType combines both and must remain -representable. The solution is thus to make ``Categorical`` abstract with -the two subclasses ``CategoricalInt64`` and ``CategoricalObject`` -distinguishing the two. - - -**Description:** - -The figure below shows the proposed datatype hierarchy. -It should be noted that abstract DTypes are distinct in two ways: - -1. They do not have instances. Instantiating an abstract DType has to return - a concrete subclass or raise an error (default, and possibly enforced - initially). -2. Unlike concrete DTypes, abstract DTypes can be superclasses, they may also - serve like Python's abstract base classes (ABC). - (It may be possible to simply use/inherit from Python ABCs.) - -These two rules are identical to the type choices made for example in the -`Julia language `_. -It allows for the creation of a datatype hierarchy, but avoids issues with -subclassing concrete DTypes directly. -For example, logic such as ``can_cast`` does not cleanly inherit from a -``Int64`` to a ``Datetime64`` even though the ``Datetime64`` could be seen -as an integer with only a unit attached (and thus implemented as a subclass). - -The main consequence for the DType implementer is that concrete DTypes can -never be subclasses of existing concrete DTypes. -End-users would not notice or need to know about this distinction. -However, subclassing may be a possible mechanism to extend the datatypes -in the future to allow specialized implementations for existing dtypes -such as a GPU float64 subclassing a NumPy float64. - -The combination of (initially) rejecting subclassing of concrete DTypes -while allowing it for abstract ones allows the transparent definition of -a class hierarchy, while avoiding potential issues with subclassing and -especially inheritance. - -As a technical implementation detail: the DType class will require C-side -storage of methods and additional information. -This requires the creation of a ``DTypeMeta`` class. -Each ``DType`` class is thus an instance of ``DTypeMeta`` with a well-defined -and extensible interface. -The end-user will not need to be aware of this. - -.. _hierarchy_figure: -.. figure:: _static/dtype_hierarchy.svg - :figclass: align-center - - -Methods/Slots defined for each DType -"""""""""""""""""""""""""""""""""""" - -NEP 41 detailed that all logic should be defined through special methods -on the DTypes. -This section will list a specific set of such methods (in the form of -Python methods). -The C-side equivalent slot signature will be summarized below after proposing -the general C-API for defining new Datatypes. -Note that while the slots are defined as special Python methods here, this is -for the readers convenience and *not* meant to imply the identical exposure -as a Python API. -This will need to be proposed in a separate, later, NEP. - -Some of the methods may be similar or even reuse existing Python slots. -User-defined DType classes are discouraged from defining or using Python's -special slots without consulting the NumPy developers, in order to allow -defining them later. -For example ``dtype1 & dtype2`` could be a shorthand for -``np.common_dtype(dtype1, dtype2)``, and comparisons should be defined mainly -through casting logic. - - -Additional Information -^^^^^^^^^^^^^^^^^^^^^^ - -In addition to the more detailed methods below, the following general -information is currently provided and will be defined on the class: - -* ``cls.parametric`` (see also `NEP 40 `_): - - * Parametric will be a flag in the (private) C-API. However, the - Python API will instead use a ``ParametricDType`` class from - which to inherit. (This is similar to Python's type flags, which include - flags for some basic subclasses such as subclasses of ``float`` or ``tuple``) - * This flag is mainly to simplify DType creation and casting and - allow for performance tweaks. - * DTypes which are not parametric must define a canonical dtype instance - which should be a singleton. - * Parametric dtypes require some additional methods (below). - -* ``self.canonical`` method (Alternative: new instance attribute) - - * Instead of byteorder, we may want a ``canonical`` flag (reusing the - ISNBO flag – "is native byte order" seems possible here). - This flag signals that the data are stored in the default/canonical way. - In practice this is always an NBO check, but generalization should be possible. - A potential use-case is a complex-conjugated instance of Complex which - stores ``real - imag`` instead of ``real + imag`` and is thus not - the canonical storage. - -* ``ensure_canonical(self) -> dtype`` return a new dtype (or ``self``). - The returned dtype must have the ``canonical`` flag set. - -* ``DType.type`` is the associated scalar type. ``dtype.type`` will be a - class attribute and the current ``dtype.type`` field will be considered - deprecated. This may be relaxed if a use-case arises. - -Additionally, existing methods (and C-side fields) will be provided. -However, the fields ``kind`` and ``char`` will be set to ``\0`` -(NULL character) on the C-side. -While discouraged, except for NumPy builtin types, ``kind`` both will return -the ``__qualname__`` of the object to ensure uniqueness for all DTypes. -(the replacement for ``kind`` will be to use ``isinstance`` checks). - -Another example of methods that should be moved to the DType class are the -various sorting functions, which shall be implemented by defining a method: - -* ``dtype_get_sort_function(self, sortkind="stable") -> sortfunction`` - -which must return ``NotImplemented`` if the given ``sortkind`` is not known. -Similarly, any function implemented previously which cannot be removed will -be implemented as a special method. -Since these methods can be deprecated and new (renamed) replacements added, -the API is not defined here and it is acceptable if it changes over time. - -For some of the current "methods" defined on the dtype, including sorting, -a long term solution may be to instead create generalized ufuncs to provide -the functionality. +- Small, rarely noticed inconsistencies are likely to change. Examples: -**Alternatives:** + - ``np.array(np.nan, dtype=np.int64)`` behaves differently from + ``np.array([np.nan], dtype=np.int64)`` with the latter raising an error. + This may require identical results (either both error or both succeed). + - ``np.array([array_like])`` sometimes behaves differently from + ``np.array([np.array(array_like)])`` + - array operations may or may not preserve dtype metadata -Some of these flags could be implemented by inheriting -for example from a ``ParametricDType`` class. However, on the C-side as -an implementation detail it seems simpler to provide a flag. -This does not preclude the possibility of creating a ``ParametricDType`` -to Python to represent the same thing. +- Documentation that describes the internal structure of dtypes will need + to be updated. -**Example:** +The new code must pass NumPy's regular test suite, giving some assurance that +the changes are compatible with existing code. -The ``datetime64`` DType is considered parametric, due to its unit, and -unlike a float64 has no canonical representation. The associated ``type`` -is the ``np.datetime64`` scalar. +****************************************************************************** +Usage and impact +****************************************************************************** +We believe the few structures in this section are sufficient to consolidate +NumPy's present functionality and also to support complex user-defined DTypes. -**Issues and Details:** +The rest of the NEP fills in details and provides support for the claim. -A potential DType such as ``Categorical`` will not be required to have a clear type -associated with it. Instead, the ``type`` may be ``object`` and the -categorical's values are arbitrary objects. -Unlike with well-defined scalars, this ``type`` cannot -not be used for the dtype discovery necessary for coercion -(compare section `DType Discovery during Array Coercion`_). +Again, though Python is used for illustration, the implementation is a C API only; a +future NEP will tackle the Python API. +After implementing this NEP, creating a DType will be possible by implementing +the following outlined DType base class, +that is further described in `nep42_DType class`_: -Coercion to and from Python Objects -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.. code-block:: python + :dedent: 0 -**Motivation:** + class DType(np.dtype): + type : type # Python scalar type + parametric : bool # (may be indicated by superclass) -When storing a single value in an array or taking it out of the array, -it is necessary to coerce (convert) it to and from the low-level -representation inside the array. + @property + def canonical(self) -> bool: + raise NotImplementedError -**Description:** + def ensure_canonical(self : DType) -> DType: + raise NotImplementedError -Coercing to and from Python scalars requires two to three methods: +For casting, a large part of the functionality is provided by the "methods" stored +in ``_castingimpl`` -1. ``__dtype_setitem__(self, item_pointer, value)`` -2. ``__dtype_getitem__(self, item_pointer, base_obj) -> object`` - The ``base_obj`` should be ignored normally, it is provided *only* for - memory management purposes, pointing to an object owning the data. - It exists only to allow support of structured datatypes with subarrays - within NumPy, which (currently) return views into the array. - The function returns an equivalent Python scalar (i.e. typically a NumPy - scalar). -3. ``__dtype_get_pyitem__(self, item_pointer, base_obj) -> object`` - (initially hidden for new-style user-defined datatypes, may be exposed - on user request). This corresponds to the ``arr.item()`` method which - is also used by ``arr.tolist()`` and returns e.g. Python floats instead of - NumPy floats. +.. code-block:: python + :dedent: 0 -(The above is meant for C-API. A Python-side API would have to use byte -buffers or similar to implement this, which may be useful for prototyping.) + @classmethod + def common_dtype(cls : DTypeMeta, other : DTypeMeta) -> DTypeMeta: + raise NotImplementedError -These largely correspond to the current definitions. When a certain scalar -has a known (different) dtype, NumPy may in the future use casting instead -of ``__dtype_setitem__``. -A user datatype is (initially) expected to implement ``__dtype_setitem__`` -for its own ``DType.type`` and all basic Python scalars it wishes to support -(e.g. integers, floats, datetime). -In the future a function "``known_scalartype``" may be added to allow a user -dtype to signal which Python scalars it can store directly. + def common_instance(self : DType, other : DType) -> DType: + raise NotImplementedError + # A mapping of "methods" each detailing how to cast to another DType + # (further specified at the end of the section) + _castingimpl = {} -**Implementation:** +For array-coercion, also part of casting: -The pseudo-code implementation for setting a single item in an array -from an arbitrary Python object ``value`` is (note that some of the -functions are only defined below):: +.. code-block:: python + :dedent: 0 - def PyArray_Pack(dtype, item_pointer, value): - DType = type(dtype) - if DType.type is type(value) or DType.known_scalartype(type(value)): - return dtype.__dtype_setitem__(item_pointer, value) + def __dtype_setitem__(self, item_pointer, value): + raise NotImplementedError - # The dtype cannot handle the value, so try casting: - arr = np.array(value) - if arr.dtype is object or arr.ndim != 0: - # not a numpy or user scalar; try using the dtype after all: - return dtype.__dtype_setitem__(item_pointer, value) + def __dtype_getitem__(self, item_pointer, base_obj) -> object: + raise NotImplementedError - arr.astype(dtype) - item_pointer.write(arr[()]) + @classmethod + def __discover_descr_from_pyobject__(cls, obj : object) -> DType: + raise NotImplementedError -where the call to ``np.array()`` represents the dtype discovery and is -not actually performed. + # initially private: + @classmethod + def _known_scalar_type(cls, obj : object) -> bool: + raise NotImplementedError -**Example:** -Current ``datetime64`` returns ``np.datetime64`` scalars and can be assigned -from ``np.datetime64``. -However, the datetime ``__dtype_setitem__`` also allows assignment from -date strings ("2016-05-01") or Python integers. -Additionally the datetime ``__dtype_get_pyitem__`` function actually returns -Python ``datetime.datetime`` object (most of the time). +Other elements of the casting implementation is the ``CastingImpl``: +.. code-block:: python + :dedent: 0 -**Alternatives:** + casting = Union["safe", "same_kind", "unsafe"] -This may be seen as simply a cast to and from the ``object`` dtype. -However, it seems slightly more complicated. This is because -in general a Python object could itself be a zero-dimensional array or -scalar with an associated DType. -Thus, representing it as a normal cast would either require that: + class CastingImpl: + # Object describing and performing the cast + casting : casting -* The implementor handles all Python classes, even those for which - ``np.array(scalar).astype(UserDType)`` already works because - ``np.array(scalar)`` returns, say, a datetime64 array. -* The cast is actually added between a typed-object to dtype. And even - in this case a generic fallback (for example ``float64`` can use - ``float(scalar)`` to do the cast) is also necessary. + def resolve_descriptors(self, Tuple[DType] : input) -> (casting, Tuple[DType]): + raise NotImplementedError -It is certainly possible to describe the coercion to and from Python objects -using the general casting machinery. However, it seems special enough to -handle specifically. + # initially private: + def _get_loop(...) -> lowlevel_C_loop: + raise NotImplementedError +which describes the casting from one DType to another. In +:ref:`NEP 43 ` this ``CastingImpl`` object is used unchanged to +support universal functions. -**Further Issues and Discussion:** -The setitem function currently duplicates some code, such as coercion -from a string. ``datetime64`` allows assignment from string, but the same -conversion also occurs for casts from the string dtype to ``datetime64``. -In the future, we may expose a way to signal whether a conversion is known, -and otherwise a normal cast is made so that the item is effectively set to ``np.array(scalar).astype(requested_dtype)``. +****************************************************************************** +Definitions +****************************************************************************** +.. glossary:: -There is a general issue about the handling of subclasses. We anticipate to not -automatically detect the dtype for ``np.array(float64_subclass)`` to be -float64. The user can still provide ``dtype=np.float64``. However, the above -"assign by casting" using ``np.array(scalar_subclass).astype(requested_dtype)`` -will fail. + dtype + The dtype *instance*; this is the object attached to a numpy array. -.. note:: + DType + Any subclass of the base type ``np.dtype``. - This means that ``np.complex256`` should not use ``__float__`` in its - ``__dtype_setitem__`` method in the future unless it is a known floating - point type. If the scalar is a subclass of a different high precision - floating point type (e.g. ``np.float128``) then this will lose precision. + coercion + Conversion of Python types to NumPy arrays and values stored in a NumPy + array. + cast + Conversion of an array to a different dtype. -DType Discovery during Array Coercion -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + parametric type + A dtype whose representation can change based on a parameter value, + like a string dtype with a length parameter. All members of the current + ``flexible`` dtype class are parametric. See + :ref:`NEP 40 `. -An important step in the usage of NumPy arrays is the creation of the array -itself from collections of generic Python objects. + promotion + Finding a dtype that can perform an operation on a mix of dtypes without + loss of information. -**Motivation:** + safe cast + A cast is safe if no information is lost when changing type. -Although the distinction is not clear currently, there are two main needs:: +On the C level we use ``descriptor`` or ``descr`` to mean +*dtype instance*. In the proposed C-API, these terms will distinguish +dtype instances from DType classes. - np.array([1, 2, 3, 4.]) +.. note:: + NumPy has an existing class hierarchy for scalar types, as + seen :ref:`in the figure ` of + :ref:`NEP 40 `, and the new DType hierarchy will resemble it. The + types are used as an attribute of the single dtype class in the current + NumPy; they're not dtype classes. They neither harm nor help this work. -needs to guess the correct dtype based on the Python objects inside. -Such an array may include a mix of datatypes, as long as they can be clearly -promoted. -Currently not clearly distinct (but partially existing for strings) is the -use case of:: +.. _nep42_DType class: - # np.dtype[np.str_] can also be spelled np.str_ or "S" (which works today) - np.array([object(), None], dtype=np.dtype[np.str_]) +****************************************************************************** +The DType class +****************************************************************************** -which forces each object to be interpreted as string. This is anticipated -to be useful for example for categorical datatypes:: +This section reviews the structure underlying the proposed DType class, +including the type hierarchy and the use of abstract DTypes. - np.array([1, 2, 1, 1, 2], dtype=Categorical) +Class getter +============================================================================== -to allow the discovery the of all unique values. -(For NumPy ``datetime64`` this is also currently used to allow string input.) +To create a DType instance from a scalar type users now call +``np.dtype`` (for instance, ``np.dtype(np.int64)``). Sometimes it is +also necessary to access the underlying DType class; this comes up in +particular with type hinting because the "type" of a DType instance is +the DType class. Taking inspiration from type hinting, we propose the +following getter syntax:: -There are three further issues to consider: + np.dtype[np.int64] -1. It may be desirable that datatypes can be created which are associated - to normal Python scalars (such as ``datetime.datetime``), which do not - have a ``dtype`` attribute already. -2. In general, a datatype could represent a sequence, however, NumPy currently - assumes that sequences are always collections of elements (the sequence cannot be an - element itself). An example for this is would be a ``vector`` DType. -3. An array may itself contain arrays with a specific dtype (even - general Python objects). For example: - ``np.array([np.array(None, dtype=object)], dtype=np.String)`` - poses the issue of how to handle the included array. +to get the DType class corresponding to a scalar type. The notation +works equally well with built-in and user-defined DTypes. -Some of these difficulties arise due to the fact that finding the correct shape -of the output array and finding the correct datatype are closely related. +This getter eliminates the need to create an explicit name for every +DType, crowding the ``np`` namespace; the getter itself signifies the +type. It also opens the possibility of making ``np.ndarray`` generic +over DType class using annotations like:: -**Implementation:** + np.ndarray[np.dtype[np.float64]] -There are two distinct cases given above: First, when the user has provided no -dtype information, and second when the user provided a DType class – -a notion that is currently represented e.g. by the parametric instance ``"S"`` -representing a string of any length. +The above is fairly verbose, so it is possible that we will include +aliases like:: -In the first case, it is necessary to establish a mapping from the Python type(s) -of the constituent elements to the DType class. -When the DType class is known, the correct dtype instance still needs to be found. -This shall be implemented by leveraging two pieces of information: + Float64 = np.dtype[np.float64] -1. ``DType.type``: The current type attribute to indicate which Python scalar - type is associated with the DType class (this is a *class* attribute that always - exists for any datatype and is not limited to array coercion). -2. The reverse lookup will remain hardcoded for the basic Python types initially. - Otherwise the ``type`` attribute will be used, and at least initially may - enforce deriving the scalar from a NumPy-provided scalar base class. - This method may be expanded later (see alternatives). -3. ``__discover_descr_from_pyobject__(cls, obj) -> dtype``: A classmethod that - returns the correct descriptor given the input object. - *Note that only parametric DTypes have to implement this*, most datatypes - can simply use a default (singleton) dtype instance which is found only - based on the ``type(obj)`` of the Python object. +in ``numpy.typing``, thus keeping annotations concise but still +avoiding crowding the ``np`` namespace as discussed above. For a +user-defined DType:: -The Python type which is already associated with a DType through the -``DType.type`` attribute maps from the DType to the Python type. -A DType may choose to automatically discover from this Python type. -This will be achieved using a global a mapping (dictionary-like) of:: + class UserDtype(dtype): ... - known_python_types[type] = DType +one can do ``np.ndarray[UserDtype]``, keeping annotations concise in +that case without introducing boilerplate in NumPy itself. For a user +user-defined scalar type:: -To anticipate the possibility of creating both a Python type (``pytype``) -and ``DType`` dynamically, and thus the potential desire to delete them again, -this mapping should generally be weak. -This requires that the ``pytype`` holds on to the ``DType`` explicitly. -Thus, in addition to building the global mapping, NumPy will store -the ``DType`` as ``pytype.__associated_array_dtype__`` in the Python type. -This does *not* define the mapping and should *not* be accessed directly. -In particular potential inheritance of the attribute does not mean that -NumPy will use the superclasses ``DType`` automatically. -A new ``DType`` must be created for the subclass. + class UserScalar(generic): ... -.. note:: +we would need to add a typing overload to ``dtype``:: - Python integers do not have a clear/concrete NumPy type associated with - them right now. This is because during array coercion NumPy currently - finds the first type capable of representing their value in the list - of `long`, `unsigned long`, `int64`, `unsigned int64`, and `object` - (on many machines `long` is 64 bit). - - Instead they will need to be implemented using an - ``AbstractPyInt``. This DType class can then provide - ``__discover_descr_from_pyobject__`` and return the actual dtype which - is e.g. ``np.dtype("int64")``. - For dispatching/promotion in ufuncs, it will also be necessary - to dynamically create ``AbstractPyInt[value]`` classes (creation can be - cached), so that they can provide the current value based promotion - functionality provided by ``np.result_type(python_integer, array)`` [1]_. - -To allow for a DType to accept specific inputs as known scalars, we will -initially use a ``known_scalar_type`` method. -This allows discovery of a ``vector`` as a scalar (element) instead of -a sequence (for the command ``np.array(vector, dtype=VectorDType)``) -even when ``vector`` is itself a sequence or even an array subclass. -This will *not* be public API initially, but may be made public at a later -time. - -This will work similar to the following pseudo-code:: + @overload + __new__(cls, dtype: Type[UserScalar], ...) -> UserDtype - def find_dtype(array_like): - common_dtype = None - for element in array_like: - # default to object dtype, if unknown - DType = known_python_types.get(type(element), np.dtype[object]) - dtype = DType.__discover_descr_from_pyobject__(element) +to allow ``np.dtype[UserScalar]``. - if common_dtype is None: - common_dtype = dtype - else: - common_dtype = np.promote_types(common_dtype, dtype) +The initial implementation probably will return only concrete (not abstract) +DTypes. -In practice, we have to find out whether an element is actually a sequence. -This means that instead of using the ``object`` dtype directly, we have to -check whether or not it is a sequence. +*This item is still under review.* -The full algorithm (without user provided dtype) thus looks more like:: - def find_dtype_recursive(array_like, dtype=None): - """ - Recursively find the dtype for a nested sequences (arrays are not - supported here). - """ - DType = known_python_types.get(type(element), None) +Hierarchy and abstract classes +============================================================================== - if DType is None and is_array_like(array_like): - # Code for a sequence, an array_like may have a DType we - # can use directly: - for element in array_like: - dtype = find_dtype_recursive(element, dtype=dtype) - return dtype +We will use abstract classes as building blocks of our extensible DType class +hierarchy. - elif DType is None: - DType = np.dtype[object] +1. Abstract classes are inherited cleanly, in principle allowing checks like + ``isinstance(np.dtype("float64"), np.inexact)``. - # Same as above +2. Abstract classes allow a single piece of code to handle a multiplicity of + input types. Code written to accept Complex objects can work with numbers + of any precision; the precision of the results is determined by the + precision of the arguments. + +3. There's room for user-created families of DTypes. We can envision an + abstract ``Unit`` class for physical units, with a concrete subclass like + ``Float64Unit``. Calling ``Unit(np.float64, "m")`` (``m`` for meters) would + be equivalent to ``Float64Unit("m")``. + +4. The implementation of universal functions in :ref:`NEP 43 ` may require + a class hierarchy. + +**Example:** A NumPy ``Categorical`` class would be a match for pandas +``Categorical`` objects, which can contain integers or general Python objects. +NumPy needs a DType that it can assign a Categorical to, but it also needs +DTypes like ``CategoricalInt64`` and ``CategoricalObject`` such that +``common_dtype(CategoricalInt64, String)`` raises an error, but +``common_dtype(CategoricalObject, String)`` returns an ``object`` DType. In +our scheme, ``Categorical`` is an abstract type with ``CategoricalInt64`` and +``CategoricalObject`` subclasses. + + +Rules for the class structure, illustrated :ref:`below `: + +1. Abstract DTypes cannot be instantiated. Instantiating an abstract DType + raises an error, or perhaps returns an instance of a concrete subclass. + Raising an error will be the default behavior and may be required initially. + +2. While abstract DTypes may be superclasses, they may also act like Python's + abstract base classes (ABC) allowing registration instead of subclassing. + It may be possible to simply use or inherit from Python ABCs. + +3. Concrete DTypes may not be subclassed. In the future this might be relaxed + to allow specialized implementations such as a GPU float64 subclassing a + NumPy float64. + +The +`Julia language `_ +has a similar prohibition against subclassing concrete types. +For example methods such as the later ``__common_instance__`` or +``__common_dtype__`` cannot work for a subclass unless they were designed +very carefully. +It helps avoid unintended vulnerabilities to implementation changes that +result from subclassing types that were not written to be subclassed. +We believe that the DType API should rather be extended to simplify wrapping +of existing functionality. + +The DType class requires C-side storage of methods and additional information, +to be implemented by a ``DTypeMeta`` class. Each ``DType`` class is an +instance of ``DTypeMeta`` with a well-defined and extensible interface; +end users ignore it. + +.. _nep42_hierarchy_figure: +.. figure:: _static/dtype_hierarchy.svg + :figclass: align-center -If the user provides ``DType``, then this DType will be tried first, and the -``dtype`` may need to be cast before the promotion is performed. -**Limitations:** - -The above issue 3. is currently (sometimes) supported by NumPy so that -the values of an included array are inspected. -Support in those cases may be kept for compatibility, however, -it will not be exposed to user datatypes. -This means that if e.g. an array with a parametric string dtype is coerced above -(or cast) to an array of a fixed length string dtype (with unknown length), -this will result in an error. -Such a conversion will require passing the correct DType (fixed length of the -string) or providing a utility function to the user. - -The use of a global type map means that an error or warning has to be given -if two DTypes wish to map to the same Python type. In most cases user -DTypes should only be implemented for types defined within the same library to -avoid the potential for conflicts. -It will be the DType implementor's responsibility to be careful about this and use -the flag to disable registration when in doubt. +Miscellaneous methods and attributes +============================================================================== + +This section collects definitions in the DType class that are not used in +casting and array coercion, which are described in detail below. + +* Existing dtype methods (:class:`numpy.dtype`) and C-side fields are preserved. + +* ``DType.type`` replaces ``dtype.type``. Unless a use case arises, + ``dtype.type`` will be deprecated. + This indicates a Python scalar type which represents the same values as + the DType. This is the same type as used in the proposed `Class getter`_ + and for `DType discovery during array coercion`_. + (This can may also be set for abstract DTypes, this is necessary + for array coercion.) + +* A new ``self.canonical`` property generalizes the notion of byte order to + indicate whether data has been stored in a default/canonical way. For + existing code, "canonical" will just signify native byte order, but it can + take on new meanings in new DTypes -- for instance, to distinguish a + complex-conjugated instance of Complex which stores ``real - imag`` instead + of ``real + imag``. The ISNBO ("is + native byte order") flag might be repurposed as the canonical flag. + +* Support is included for parametric DTypes. A DType will be deemed parametric + if it inherits from ParametricDType. + +* DType methods may resemble or even reuse existing Python slots. Thus Python + special slots are off-limits for user-defined DTypes (for instance, defining + ``Unit("m") > Unit("cm")``), since we may want to develop a meaning for these + operators that is common to all DTypes. + +* Sorting functions are moved to the DType class. They may be implemented by + defining a method ``dtype_get_sort_function(self, sortkind="stable") -> + sortfunction`` that must return ``NotImplemented`` if the given ``sortkind`` + is not known. + +* Functions that cannot be removed are implemented as special methods. + Many of these were previously defined part of the :c:type:`PyArray_ArrFuncs` + slot of the dtype instance (``PyArray_Descr *``) and include functions + such as ``nonzero``, ``fill`` (used for ``np.arange``), and + ``fromstr`` (used to parse text files). + These old methods will be deprecated and replacements + following the new design principles added. + The API is not defined here. Since these methods can be deprecated and renamed + replacements added, it is acceptable if these new methods have to be modified. + +* Use of ``kind`` for non-built-in types is discouraged in favor of + ``isinstance`` checks. ``kind`` will return the ``__qualname__`` of the + object to ensure uniqueness for all DTypes. On the C side, ``kind`` and + ``char`` are set to ``\0`` (NULL character). + While ``kind`` will be discouraged, the current ``np.issubdtype`` + may remain the preferred method for this type of check. + +* A method ``ensure_canonical(self) -> dtype`` returns a new dtype (or + ``self``) with the ``canonical`` flag set. + +* Since NumPy's approach is to provide functionality through unfuncs, + functions like sorting that will be implemented in DTypes might eventually be + reimplemented as generalized ufuncs. + +.. _nep_42_casting: + +****************************************************************************** +Casting +****************************************************************************** -**Alternatives:** +We review here the operations related to casting arrays: -The above proposes to add a global mapping, however, initially limiting it -to types deriving from a NumPy subclass (and a fixed set of Python types). -This could be relaxed in the future. -Alternatively, we could rely on the scalar belonging to the user dtype to -implement ``scalar.__associated_array_dtype__`` or similar. - -Initially, the exact implementation shall be *undefined*, if -scalars will have to derive from a NumPy scalar, they will also have -a ``.__associated_array_dtype__`` attribute. -At this time, a future update may to use this instead of a global mapping, -however, it makes NumPy a hard dependency for the scalar class. - -An initial alternative suggestion was to use a two-pass approach instead. -The first pass would only find the correct DType class, and the second pass -would then find correct dtype instance (the second pass is often not necessary). -The advantage of this is that the DType class information is vital for universal -functions to decide which loop to execute. -The first pass would provide the full information necessary for value-based -casting currently implemented for scalars, giving even the possibility of -expanding it to e.g. list inputs ``np.add(np.array([8], dtype="uint8"), [4])`` -giving a ``uint8`` result. -This is mainly related to the question to how the common dtype is found above. -It seems unlikely that this is useful, and similar to a global, could be -added later if deemed necessary. - -**Further Issues and Discussion:** - -While it is possible to create e.g. a DType such as Categorical, array, -or vector which can only be used if `dtype=DType` is provided, if this -is necessary these will not roundtrip correctly when converted back -and forth:: +- Finding the "common dtype," returned by :func:`numpy.promote_types` and + :func:`numpy.result_type` - np.array(np.array(1, dtype=Categorical)[()]) +- The result of calling :func:`numpy.can_cast` -requires to pass the original ``dtype=Categorical`` or returns an array -with dtype ``object``. -While a general limitation, the round-tripping shall always be possible if -``dtype=old_dtype`` is provided. +We show how casting arrays with ``astype(new_dtype)`` will be implemented. -**Example:** +`Common DType` operations +============================================================================== -The current datetime DType requires a ``__discover_descr_from_pyobject__`` -which returns the correct unit for string inputs. This allows it to support -the current:: +When input types are mixed, a first step is to find a DType that can hold +the result without loss of information -- a "common DType." - np.array(["2020-01-02", "2020-01-02 11:24"], dtype="M8") +Array coercion and concatenation both return a common dtype instance. Most +universal functions use the common DType for dispatching, though they might +not use it for a result (for instance, the result of a comparison is always +bool). -By inspecting the date strings. Together with the below common dtype -operation, this allows it to automatically find that the datetime64 unit -should be "minutes". +We propose the following implementation: +- For two DType classes:: -Common DType Operations -^^^^^^^^^^^^^^^^^^^^^^^ + __common_dtype__(cls, other : DTypeMeta) -> DTypeMeta -NumPy currently provides functions like ``np.result_type`` and -``np.promote_types`` for determining common types. -These differ in that ``np.result_type`` can take arrays and scalars as input -and implements value based promotion [1]_. + Returns a new DType, often one of the inputs, which can represent values + of both input DTypes. This should usually be minimal: + the common DType of ``Int16`` and ``Uint16`` is ``Int32`` and not ``Int64``. + ``__common_dtype__`` may return NotImplemented to defer to other and, + like Python operators, subclasses take precedence (their + ``__common_dtype__`` method is tried first). -To distinguish between the promotion occurring during universal function application, -we will call it "common type" operation here. +- For two instances of the same DType:: -**Motivation:** -Common type operations are vital for array coercion when different -input types are mixed. -They also provide the logic currently used to decide the output dtype of -``np.concatenate()`` and on their own are quite useful. + __common_instance__(self: SelfT, other : SelfT) -> SelfT -Furthermore, common type operations may be used to find the correct dtype -to use for functions with different inputs (including universal functions). -This includes an interesting distinction: + For nonparametric built-in dtypes, this returns a canonicalized copy of + ``self``, preserving metadata. For nonparametric user types, this provides + a default implementation. -1. Universal functions use the DType classes for dispatching, they thus - require the common DType class (as a first step). - While this can help with finding the correct loop to execute, the loop - may not need the actual common dtype instance. - (Hypothetical example: - ``float_arr + string_arr -> string``, but the output string length is - not the same as ``np.concatenate(float_arr, string_arr)).dtype``.) -2. Array coercion and concatenation require the common dtype *instance*. +- For instances of different DTypes, for example ``>float64`` and ``S8``, + the operation is done in three steps: -**Implementation:** -The implementation of the common dtype (instance) determination -has some overlap with casting. -Casting from a specific dtype (Float64) to a String needs to find -the correct string length (a step that is mainly necessary for parametric dtypes). + 1. ``Float64.__common_dtype__(type(>float64), type(S8))`` + returns ``String`` (or defers to ``String.__common_dtype__``). -We propose the following implementation: + 2. The casting machinery (explained in detail below) provides the + information that ``">float64"`` casts to ``"S32"`` + + 3. ``String.__common_instance__("S8", "S32")`` returns the final ``"S32"``. -1. ``__common_dtype__(cls, other : DTypeMeta) -> DTypeMeta`` answers what the common - DType class is given two DType class objects. - It may return ``NotImplemented`` to defer to ``other``. - (For abstract DTypes, subclasses get precedence, concrete types are always - leaves, so always get preference or are tried from left to right). -2. ``__common_instance__(self: SelfT, other : SelfT) -> SelfT`` is used when - two instances of the same DType are given. - For builtin dtypes (that are not parametric), this - currently always returns ``self`` (but ensures native byte order). - This is to preserve metadata. We can thus provide a default implementation - for non-parametric user dtypes. - -These two cases do *not* cover the case where two different dtype instances -need to be promoted. For example `">float64"` and `"S8"`. -The solution is partially "outsourced" to the casting machinery by -splitting the operation up into three steps: - -1. ``Float64.__common_dtype__(type(>float64), type(S8))`` - returns `String` (or defers to ``String.__common_dtype__``). -2. The casting machinery provides the information that `">float64"` casts - to `"S32"` (see below for how casting will be defined). -3. ``String.__common_instance__("S8", "S32")`` returns the final `"S32"`. - -The main reason for this is to avoid the need to implement -identical functionality multiple times. -The design (together with casting) naturally separates the concerns of -different Datatypes. -In the above example, Float64 does not need to know about the cast. -While the casting machinery (``CastingImpl[Float64, String]``) -could include the third step, it is not required to do so and the string -can always be extended (e.g. with new encodings) without extending the -``CastingImpl[Float64, String]``. +The benefit of this handoff is to reduce duplicated code and keep concerns +separate. DType implementations don't need to know how to cast, and the +results of casting can be extended to new types, such as a new string encoding. This means the implementation will work like this:: @@ -737,14 +518,14 @@ This means the implementation will work like this:: raise TypeError("no common dtype") return common_dtype - def promote_types(dtype1, dtyp2): + def promote_types(dtype1, dtype2): common = common_dtype(type(dtype1), type(dtype2)) if type(dtype1) is not common: # Find what dtype1 is cast to when cast to the common DType # by using the CastingImpl as described below: castingimpl = get_castingimpl(type(dtype1), common) - safety, (_, dtype1) = castingimpl.adjust_descriptors((dtype1, None)) + safety, (_, dtype1) = castingimpl.resolve_descriptors((dtype1, None)) assert safety == "safe" # promotion should normally be a safe cast if type(dtype2) is not common: @@ -753,305 +534,248 @@ This means the implementation will work like this:: if dtype1 is not dtype2: return common.__common_instance__(dtype1, dtype2) -Some of these steps may be optimized for non-parametric DTypes. +Some of these steps may be optimized for nonparametric DTypes. -**Note:** - -A currently implemented fallback for the ``__common_dtype__`` operation -is to use the "safe" casting logic. -Since ``int16`` can safely cast to ``int64``, it is clear that -``np.promote_types(int16, int64)`` should be ``int64``. - -However, this cannot define all such operations, and will fail for example for:: +Since the type returned by ``__common_dtype__`` is not necessarily one of the +two arguments, it's not equivalent to NumPy's "safe" casting. +Safe casting works for ``np.promote_types(int16, int64)``, which returns +``int64``, but fails for:: np.promote_types("int64", "float32") -> np.dtype("float64") -In this design, it is the responsibility of the DType author to ensure that -in most cases a safe-cast implies that this will be the result of the -``__common_dtype__`` method. +It is the responsibility of the DType author to ensure that the inputs +can be safely cast to the ``__common_dtype__``. -Note that some exceptions may apply. For example casting ``int32`` to -a (long enough) string is – at least at this time – considered "safe". +Exceptions may apply. For example, casting ``int32`` to +a (long enough) string is at least at this time considered "safe". However ``np.promote_types(int32, String)`` will *not* be defined. -**Alternatives:** +**Example:** -The use of casting for common dtype (instance) determination neatly separates -the concerns and allows for a minimal set of duplicate functionality -being implemented. -In cases of mixed DType (classes), it also adds an additional step -to finding the common dtype. -The common dtype (of two instances) could thus be implemented explicitly to avoid -this indirection, potentially only as a fast-path. -The above suggestion assumes that this is, however, not a speed relevant path, -since in most cases, e.g. in array coercion, only a single Python type (and thus -dtype) is involved. -The proposed design hinges in the implementation of casting to be -separated into its own ufunc-like object as described below. - -In principle common DType could be defined only based on "safe casting" rules, -if we order all DTypes and find the first one both can cast to safely. -However, the issue with this approach is that a newly added DType can change -the behaviour of an existing program. For example, a new ``int24`` would be -the first valid common type for ``int16`` and ``uint16``, demoting the currently -defined behaviour of ``int32``. -This API extension could be allowed in the future, while adding it may be -more involved, the current proposal for defining casts is fully opaque in -this regard and thus extensible. +``object`` always chooses ``object`` as the common DType. For +``datetime64`` type promotion is defined with no other datatype, but if +someone were to implement a new higher precision datetime, then:: -**Example:** + HighPrecisionDatetime.__common_dtype__(np.dtype[np.datetime64]) -``object`` always chooses ``object`` as the common DType. For ``datetime64`` -type promotion is defined with no other datatype, but if someone were to -implement a new higher precision datetime, then:: +would return ``HighPrecisionDatetime``, and the casting implementation, +as described below, may need to decide how to handle the datetime unit. - HighPrecisionDatetime.__common_dtype__(np.dtype[np.datetime64]) -would return ``HighPrecisionDatetime``, and the below casting may need to -decide how to handle the datetime unit. +**Alternatives:** +- We're pushing the decision on common DTypes to the DType classes. Suppose + instead we could turn to a universal algorithm based on safe casting, + imposing a total order on DTypes and returning the first type that both + arguments could cast to safely. -Casting -^^^^^^^ + It would be difficult to devise a reasonable total order, and it would have + to accept new entries. Beyond that, the approach is flawed because + importing a type can change the behavior of a program. For example, a + program requiring the common DType of ``int16`` and ``uint16`` would + ordinarily get the built-in type ``int32`` as the first match; if the + program adds ``import int24``, the first match becomes ``int24`` and the + smaller type might make the program overflow for the first time. [1]_ + +- A more flexible common DType could be implemented in the future where + ``__common_dtype__`` relies on information from the casting logic. + Since ``__commond_dtype__`` is a method a such a default implementation + could be added at a later time. + +- The three-step handling of differing dtypes could, of course, be coalesced. + It would lose the value of splitting in return for a possibly faster + execution. But few cases would benefit. Most cases, such as array coercion, + involve a single Python type (and thus dtype). + + +The cast operation +============================================================================== -Maybe the most complex and interesting operation which is provided -by DTypes is the ability to cast from one dtype to another. -The casting operation is much like a typical function (universal function) on -arrays converting one input to a new output. -There are mainly two distinctions: +Casting is perhaps the most complex and interesting DType operation. It +is much like a typical universal function on arrays, converting one input to a +new output, with two distinctions: -1. Casting always requires an explicit output datatype to be given. -2. The NumPy iterator API requires access to lower-level functions than - is currently necessary for universal functions. +- Casting always requires an explicit output datatype. +- The NumPy iterator API requires access to functions that are lower-level + than what universal functions currently need. -Casting from one dtype to another can be complex, and generally a casting -function may not implement all details of each input datatype (such as -non-native byte order or unaligned access). -Thus casting naturally is performed in up to three steps: +Casting can be complex and may not implement all details of each input +datatype (such as non-native byte order or unaligned access). So a complex +type conversion might entail 3 steps: -1. The input datatype is normalized and prepared for the actual cast. +1. The input datatype is normalized and prepared for the cast. 2. The cast is performed. -3. The cast result, which is in a normalized form, is cast to the requested +3. The result, which is in a normalized form, is cast to the requested form (non-native byte order). -although often only step 2. is required. - Further, NumPy provides different casting kinds or safety specifiers: -* "safe" -* "same_kind" -* "unsafe" +* ``equivalent``, allowing only byte-order changes +* ``safe``, requiring a type large enough to preserve value +* ``same_kind``, requiring a safe cast or one within a kind, like float64 to float32 +* ``unsafe``, allowing any data conversion -and in some cases a cast may even be represented as a simple view. +and in some cases a cast may be just a view. +We need to support the two current signatures of ``arr.astype``: -**Motivation:** +- For DTypes: ``arr.astype(np.String)`` -Similar to the common dtype/DType operation above, we again have two use cases: + - current spelling ``arr.astype("S")`` + - ``np.String`` can be an abstract DType -1. ``arr.astype(np.String)`` (current spelling ``arr.astype("S")``) -2. ``arr.astype(np.dtype("S8"))``. +- For dtypes: ``arr.astype(np.dtype("S8"))`` -Where the first case is also noted in NEP 40 and 41 as a design goal, since -``np.String`` could also be an abstract DType as mentioned above. -The implementation of casting should also come with as little duplicate -implementation as necessary, i.e. to avoid unnecessary methods on the -DTypes. -Furthermore, it is desirable that casting is implemented similar to universal -functions. +We also have two signatures of ``np.can_cast``: + +- Instance to class: ``np.can_cast(dtype, DType, "safe")`` +- Instance to instance: ``np.can_cast(dtype, other_dtype, "safe")`` + +On the Python level ``dtype`` is overloaded to mean class or instance. + +A third ``can_cast`` signature, ``np.can_cast(DType, OtherDType, "safe")``,may be used +internally but need not be exposed to Python. -Analogous to the above, the following also need to be defined: +During DType creation, DTypes will be able to pass a list of ``CastingImpl`` +objects, which can define casting to and from the DType. -1. ``np.can_cast(dtype, DType, "safe")`` (instance to class) -2. ``np.can_cast(dtype, other_dtype, "safe")`` (casting an instance to another instance) +One of them should define the cast between instances of that DType. It can be +omitted if the DType has only a single implementation and is nonparametric. -overloading the meaning of ``dtype`` to mean either class or instance -(on the Python level). -The question of ``np.can_cast(DType, OtherDType, "safe")`` is also a -possibility and may be used internally. -However, it is initially not necessary to expose to Python. +Each ``CastingImpl`` has a distinct DType signature: + ``CastingImpl[InputDtype, RequestedDtype]`` -**Implementation:** +and implements the following methods and attributes: -During DType creation, DTypes will have the ability to pass a list of -``CastingImpl`` objects, which can define casting to and from the DType. -One of these ``CastingImpl`` objects is special because it should define -the cast within the same DType (from one instance to another). -A DType which does not define this, must have only a single implementation -and not be parametric. -Each ``CastingImpl`` has a specific DType signature: -``CastingImpl[InputDtype, RequestedDtype]``. -And implements the following methods and attributes: +* To report safeness, -* ``adjust_descriptors(self, Tuple[DType] : input) -> casting, Tuple[DType]``. - Here ``casting`` signals the casting safeness (safe, unsafe, or same-kind) - and the output dtype tuple is used for more multi-step casting (see below). -* ``get_transferfunction(...) -> function handling cast`` (signature to be decided). - This function returns a low-level implementation of a strided casting function - ("transfer function"). -* ``cast_kind`` attribute with one of safe, unsafe, or same-kind. Used to - quickly decide casting safety when this is relevant. + ``resolve_descriptors(self, Tuple[DType] : input) -> casting, Tuple[DType]``. -``adjust_descriptors`` provides information about whether or -not a cast is safe and is of importance mainly for parametric DTypes. -``get_transferfunction`` provides NumPy with a function capable of performing -the actual cast. Initially the implementation of ``get_transferfunction`` -will be *private*, and users will only be able to provide contiguous loops -with the signature. + The ``casting`` output reports safeness (safe, unsafe, or same-kind), and + the tuple is used for more multistep casting, as in the example below. -**Performing the Cast:** +* To get a casting function, + + ``get_loop(...) -> function_to_handle_cast (signature to be decided)`` + + returns a low-level implementation of a strided casting function + ("transfer function") capable of performing the + cast. + + Initially the implementation will be *private*, and users will only be + able to provide strided loops with the signature. + +* For performance, a ``casting`` attribute taking a value of ``equivalent``, ``safe``, + ``unsafe``, or ``same-kind``. + + +**Performing a cast** + +.. _nep42_cast_figure: -.. _cast_figure: .. figure:: _static/casting_flow.svg :figclass: align-center -`The above figure `_ illustrates the multi-step logic necessary to -cast for example an ``int24`` with a value of ``42`` to a string of length 20 +The above figure illustrates a multistep +cast of an ``int24`` with a value of ``42`` to a string of length 20 (``"S20"``). -In this example, the implementer only provided the functionality of casting -an ``int24`` to an ``S8`` string (which can hold all 24bit integers). -Due to this limited implementation, the full cast has to do multiple -conversions. The full process is: -1. Call ``CastingImpl[Int24, String].adjust_descriptors((int24, "S20"))``. +We've picked an example where the implementer has only provided limited +functionality: a function to cast an ``int24`` to an ``S8`` string (which can +hold all 24-bit integers). This means multiple conversions are needed. + +The full process is: + +1. Call + + ``CastingImpl[Int24, String].resolve_descriptors((int24, "S20"))``. + This provides the information that ``CastingImpl[Int24, String]`` only - implements the cast of ``int24`` to ``"S8``. + implements the cast of ``int24`` to ``"S8"``. + 2. Since ``"S8"`` does not match ``"S20"``, use - ``CastingImpl[String, String].get_transferfunction()`` + + ``CastingImpl[String, String].get_loop()`` + to find the transfer (casting) function to convert an ``"S8"`` into an ``"S20"`` + 3. Fetch the transfer function to convert an ``int24`` to an ``"S8"`` using - ``CastingImpl[Int24, String].get_transferfunction()`` -4. Perform the actual cast using the two transfer functions: - ``int24(42) -> S8("42") -> S20("42")``. -Note that in this example the ``adjust_descriptors`` function plays a less -central role. It becomes more important for ``np.can_cast``. + ``CastingImpl[Int24, String].get_loop()`` -Further, ``adjust_descriptors`` allows the implementation for -``np.array(42, dtype=int24).astype(String)`` to call -``CastingImpl[Int24, String].adjust_descriptors((int24, None))``. -In this case the result of ``(int24, "S8")`` defines the correct cast: -``np.array(42, dtype=int24),astype(String) == np.array("42", dtype="S8")``. +4. Perform the actual cast using the two transfer functions: -**Casting Safety:** + ``int24(42) -> S8("42") -> S20("42")``. -To answer the question of casting safety -``np.can_cast(int24, "S20", casting="safe")``, only the ``adjust_descriptors`` -function is required and called is in the same way as in -`the figure describing a cast `_. -In this case, the calls to ``adjust_descriptors``, will also provide the -information that ``int24 -> "S8"`` as well as ``"S8" -> "S20"`` are safe casts, -and thus also the ``int24 -> "S20"`` is a safe cast. + ``resolve_descriptors`` allows the implementation for -The casting safety can currently be "equivalent" when a cast is both safe -and can be performed using only a view. -The information that a cast is a simple "view" will instead be handled by -an additional flag. Thus the ``casting`` can have the 6 values in total: -safe, unsafe, same-kind as well as safe+view, unsafe+view, same-kind+view. -Where the current "equivalent" is the same as safe+view. + ``np.array(42, dtype=int24).astype(String)`` -(For more information on the ``adjust_descriptor`` signature see the -C-API section below.) + to call + ``CastingImpl[Int24, String].resolve_descriptors((int24, None))``. -**Casting between instances of the same DType:** + In this case the result of ``(int24, "S8")`` defines the correct cast: -In general one of the casting implementations define by the DType implementor -must be ``CastingImpl[DType, DType]`` (unless there is only a singleton -instance). -To keep the casting to as few steps as possible, this implementation must -be capable any conversions between all instances of this DType. + ``np.array(42, dtype=int24).astype(String) == np.array("42", dtype="S8")``. +**Casting safety** -**General Multi-Step Casting** +To compute ``np.can_cast(int24, "S20", casting="safe")``, only the +``resolve_descriptors`` function is required and +is called in the same way as in :ref:`the figure describing a cast `. -In general we could implement certain casts, such as ``int8`` to ``int24`` -even if the user only provides an ``int16 -> int24`` cast. -This proposal currently does not provide this functionality. However, -it could be extended in the future to either find such casts dynamically, -or at least allow ``adjust_descriptors`` to return arbitrary ``dtypes``. -If ``CastingImpl[Int8, Int24].adjust_descriptors((int8, int24))`` returns -``(int16, int24)``, the actual casting process could be extended to include -the ``int8 -> int16`` cast. Unlike the above example, which is limited -to at most three steps. +In this case, the calls to ``resolve_descriptors``, will also provide the +information that ``int24 -> "S8"`` as well as ``"S8" -> "S20"`` are safe +casts, and thus also the ``int24 -> "S20"`` is a safe cast. +In some cases, no cast is necessary. For example, on most Linux systems +``np.dtype("long")`` and ``np.dtype("longlong")`` are different dtypes but are +both 64-bit integers. In this case, the cast can be performed using +``long_arr.view("longlong")``. The information that a cast is a view will be +handled by an additional flag. Thus the ``casting`` can have the 8 values in +total: the original 4 of ``equivalent``, ``safe``, ``unsafe``, and ``same-kind``, +plus ``equivalent+view``, ``safe+view``, ``unsafe+view``, and +``same-kind+view``. NumPy currently defines ``dtype1 == dtype2`` to be True +only if byte order matches. This functionality can be replaced with the +combination of "equivalent" casting and the "view" flag. -**Alternatives:** +(For more information on the ``resolve_descriptors`` signature see the +:ref:`nep42_C-API` section below and :ref:`NEP 43 `.) -The choice of using only the DType classes in the first step of finding the -correct ``CastingImpl`` means that the default implementation of -``__common_dtype__`` has a reasonable definition of "safe casting" between -DTypes classes (although e.g. the concatenate operation using it may still -fail when attempting to find the actual common instance or cast). - -The split into multiple steps may seem to add complexity -rather than reduce it, however, it consolidates that we have the two distinct -signatures of ``np.can_cast(dtype, DTypeClass)`` and ``np.can_cast(dtype, other_dtype)``. -Further, the above API guarantees the separation of concerns for user DTypes. -The user ``Int24`` dtype does not have to handle all string lengths if it -does not wish to do so. Further, if an encoding was added to the ``String`` -DType, this does not affect the overall cast. -The ``adjust_descriptor`` function can keep returning the default encoding -and the ``CastingImpl[String, String]`` can take care of any necessary encoding -changes. - -The main alternative to the proposed design is to move most of the information -which is here pushed into the ``CastingImpl`` directly into methods -on the DTypes. This, however, will not allow the close similarity between casting -and universal functions. On the up side, it reduces the necessary indirection -as noted below. - -An initial proposal defined two methods ``__can_cast_to__(self, other)`` -to dynamically return ``CastingImpl``. -The advantage of this addition is that it removes the requirement to know all -possible casts at DType creation time (of one of the involved DTypes). -Such API could be added at a later time. It should be noted, however, -that it would be mainly useful for inheritance-like logic, which can be -problematic. As an example two different ``Float64WithUnit`` implementations -both could infer that they can unsafely cast between one another when in fact -some combinations should cast safely or preserve the Unit (both of which the -"base" ``Float64`` would discard). -In the proposed implementation this is not possible, since the two implementations -are not aware of each other. +**Casting between instances of the same DType** -**Notes:** +To keep down the number of casting +steps, CastingImpl must be capable of any conversion between all instances +of this DType. -The proposed ``CastingImpl`` is designed to be compatible with the -``UFuncImpl`` proposed in NEP 43. -While initially it will be a distinct object or C-struct, the aim is that -``CastingImpl`` can be a subclass or extension of ``UFuncImpl``. -Once this happens, this may naturally allow the use of a ``CastingImpl`` to -pass around a specialized casting function directly. +In general the DType implementer must include ``CastingImpl[DType, DType]`` +unless there is only a singleton instance. -In the future, we may consider adding a way to spell out that specific -casts are known to be *not* possible. +**General multistep casting** -In the above text ``CastingImpl`` is described as a Python object. In practice, -the current plan is to implement it as a C-side structure stored on the ``from`` -datatype. -A Python side API to get an equivalent ``CastingImpl`` object will be created, -but storing it (similar to the current implementation) on the ``from`` datatype -avoids the creation of cyclic reference counts. +We could implement certain casts, such as ``int8`` to ``int24``, +even if the user provides only an ``int16 -> int24`` cast. This proposal does +not provide that, but future work might find such casts dynamically, or at least +allow ``resolve_descriptors`` to return arbitrary ``dtypes``. -The way dispatching works for ``CastingImpl`` is planned to be limited initially -and fully opaque. -In the future, it may or may not be moved into a special UFunc, or behave -more like a universal function. +If ``CastingImpl[Int8, Int24].resolve_descriptors((int8, int24))`` returns +``(int16, int24)``, the actual casting process could be extended to include +the ``int8 -> int16`` cast. This adds a step. **Example:** -The implementation for casting integers to datetime would currently generally -say that this cast is unsafe (it is always an unsafe cast). -Its ``adjust_descriptors`` functions may look like:: - - def adjust_descriptors(input): - from_dtype, to_dtype = input +The implementation for casting integers to datetime would generally +say that this cast is unsafe (because it is always an unsafe cast). +Its ``resolve_descriptors`` function may look like:: + def resolve_descriptors(self, given_dtypes): + from_dtype, to_dtype = given_dtypes from_dtype = from_dtype.ensure_canonical() # ensure not byte-swapped if to_dtype is None: raise TypeError("Cannot convert to a NumPy datetime without a unit") @@ -1065,26 +789,440 @@ Its ``adjust_descriptors`` functions may look like:: .. note:: - While NumPy currently defines some of these casts, with the possible - exception of the unit-less ``timedelta64`` it may be better to not - define these cast at all. In general we expect that user defined - DTypes will be using other methods such as ``unit.drop_unit(arr)`` - or ``arr * unit.seconds``. + While NumPy currently defines integer-to-datetime casts, with the possible + exception of the unit-less ``timedelta64`` it may be better to not define + these casts at all. In general we expect that user defined DTypes will be + using custom methods such as ``unit.drop_unit(arr)`` or ``arr * + unit.seconds``. + + +**Alternatives:** + +- Our design objectives are: + - Minimize the number of DType methods and avoid code duplication. + - Mirror the implementation of universal functions. + +- The decision to use only the DType classes in the first step of finding the + correct ``CastingImpl`` in addition to defining ``CastingImpl.casting``, + allows to retain the current default implementation of + ``__common_dtype__`` for existing user defined dtypes, which could be + expanded in the future. + +- The split into multiple steps may seem to add complexity rather than reduce + it, but it consolidates the signatures of ``np.can_cast(dtype, DTypeClass)`` + and ``np.can_cast(dtype, other_dtype)``. + + Further, the API guarantees separation of concerns for user DTypes. The user + ``Int24`` dtype does not have to handle all string lengths if it does not + wish to do so. Further, an encoding added to the ``String`` DType would + not affect the overall cast. The ``resolve_descriptors`` function + can keep returning the default encoding and the ``CastingImpl[String, + String]`` can take care of any necessary encoding changes. + +- The main alternative is moving most of the information that is here pushed + into the ``CastingImpl`` directly into methods on the DTypes. But this + obscures the similarity between casting and universal functions. It does + reduce indirection, as noted below. + +- An earlier proposal defined two methods ``__can_cast_to__(self, other)`` to + dynamically return ``CastingImpl``. This + removes the requirement to define all possible casts at DType creation + (of one of the involved DTypes). + Such an API could be added later. It resembles Python's ``__getattr__`` in + providing additional control over attribute lookup. -C-Side API -^^^^^^^^^^ -A Python side API shall not be defined here. This is a general side approach. +**Notes:** + +The proposed ``CastingImpl`` is designed to be identical to the +``PyArrayMethod`` proposed in NEP43 as part of restructuring ufuncs to handle +new DTypes. + +The way dispatching works for ``CastingImpl`` is planned to be limited +initially and fully opaque. In the future, it may or may not be moved into a +special UFunc, or behave more like a universal function. + + +.. _nep42_array_coercion: + + +Coercion to and from Python objects +============================================================================== + +When storing a single value in an array or taking it out, it is necessary to +coerce it -- that is, convert it -- to and from the low-level representation +inside the array. + +Coercion is slightly more complex than typical casts. One reason is that a +Python object could itself be a 0-dimensional array or scalar with an +associated DType. + +Coercing to and from Python scalars requires two to three +methods that largely correspond to the current definitions: + +1. ``__dtype_setitem__(self, item_pointer, value)`` + +2. ``__dtype_getitem__(self, item_pointer, base_obj) -> object``; + ``base_obj`` is for memory management and usually ignored; it points to + an object owning the data. Its only role is to support structured datatypes + with subarrays within NumPy, which currently return views into the array. + The function returns an equivalent Python scalar (i.e. typically a NumPy + scalar). + +3. ``__dtype_get_pyitem__(self, item_pointer, base_obj) -> object`` (initially + hidden for new-style user-defined datatypes, may be exposed on user + request). This corresponds to the ``arr.item()`` method also used by + ``arr.tolist()`` and returns Python floats, for example, instead of NumPy + floats. + +(The above is meant for C-API. A Python-side API would have to use byte +buffers or similar to implement this, which may be useful for prototyping.) + +When a certain scalar +has a known (different) dtype, NumPy may in the future use casting instead of +``__dtype_setitem__``. +A user datatype is (initially) expected to implement +``__dtype_setitem__`` for its own ``DType.type`` and all basic Python scalars +it wishes to support (e.g. ``int`` and ``float``). In the future a +function ``known_scalar_type`` may be made public to allow a user dtype to signal +which Python scalars it can store directly. + + +**Implementation:** The pseudocode implementation for setting a single item in +an array from an arbitrary Python object ``value`` is (some +functions here are defined later):: + + def PyArray_Pack(dtype, item_pointer, value): + DType = type(dtype) + if DType.type is type(value) or DType.known_scalartype(type(value)): + return dtype.__dtype_setitem__(item_pointer, value) + + # The dtype cannot handle the value, so try casting: + arr = np.array(value) + if arr.dtype is object or arr.ndim != 0: + # not a numpy or user scalar; try using the dtype after all: + return dtype.__dtype_setitem__(item_pointer, value) + + arr.astype(dtype) + item_pointer.write(arr[()]) + +where the call to ``np.array()`` represents the dtype discovery and is +not actually performed. + +**Example:** Current ``datetime64`` returns ``np.datetime64`` scalars and can +be assigned from ``np.datetime64``. However, the datetime +``__dtype_setitem__`` also allows assignment from date strings ("2016-05-01") +or Python integers. Additionally the datetime ``__dtype_get_pyitem__`` +function actually returns a Python ``datetime.datetime`` object (most of the +time). + + +**Alternatives:** This functionality could also be implemented as a cast to and +from the ``object`` dtype. +However, coercion is slightly more complex than typical casts. +One reason is that in general a Python object could itself be a +zero-dimensional array or scalar with an associated DType. +Such an object has a DType, and the correct cast to another DType is already +defined:: + + np.array(np.float32(4), dtype=object).astype(np.float64) + +is identical to:: + + np.array(4, dtype=np.float32).astype(np.float64) + +Implementing the first ``object`` to ``np.float64`` cast explicitly, +would require the user to take to duplicate or fall back to existing +casting functionality. + +It is certainly possible to describe the coercion to and from Python objects +using the general casting machinery, but the ``object`` dtype is special and +important enough to be handled by NumPy using the presented methods. + +**Further issues and discussion:** + +- The ``__dtype_setitem__`` function duplicates some code, such as coercion + from a string. + + ``datetime64`` allows assignment from string, but the same conversion also + occurs for casting from the string dtype to ``datetime64``. + + We may in the future expose the ``known_scalartype`` function to allow the + user to implement such duplication. + + For example, NumPy would normally use + + ``np.array(np.string_("2019")).astype(datetime64)`` + + but ``datetime64`` could choose to use its ``__dtype_setitem__`` instead + for performance reasons. + +- There is an issue about how subclasses of scalars should be handled. We + anticipate to stop automatically detecting the dtype for + ``np.array(float64_subclass)`` to be float64. The user can still provide + ``dtype=np.float64``. However, the above automatic casting using + ``np.array(scalar_subclass).astype(requested_dtype)`` will fail. In many + cases, this is not an issue, since the Python ``__float__`` protocol can be + used instead. But in some cases, this will mean that subclasses of Python + scalars will behave differently. + +.. note:: + + *Example:* ``np.complex256`` should not use ``__float__`` in its + ``__dtype_setitem__`` method in the future unless it is a known floating + point type. If the scalar is a subclass of a different high precision + floating point type (e.g. ``np.float128``) then this currently loses + precision without notifying the user. + In that case ``np.array(float128_subclass(3), dtype=np.complex256)`` + may fail unless the ``float128_subclass`` is first converted to the + ``np.float128`` base class. + + +DType discovery during array coercion +============================================================================== + +An important step in the use of NumPy arrays is creation of the array from +collections of generic Python objects. + +**Motivation:** Although the distinction is not clear currently, there are two main needs:: + + np.array([1, 2, 3, 4.]) + +needs to guess the correct dtype based on the Python objects inside. +Such an array may include a mix of datatypes, as long as they can be +promoted. +A second use case is when users provide the output DType class, but not the +specific DType instance:: + + np.array([object(), None], dtype=np.dtype[np.string_]) # (or `dtype="S"`) + +In this case the user indicates that ``object()`` and ``None`` should be +interpreted as strings. +The need to consider the user provided DType also arises for a future +``Categorical``:: + + np.array([1, 2, 1, 1, 2], dtype=Categorical) + +which must interpret the numbers as unique categorical values rather than +integers. + +There are three further issues to consider: + +1. It may be desirable to create datatypes associated + with normal Python scalars (such as ``datetime.datetime``) that do not + have a ``dtype`` attribute already. + +2. In general, a datatype could represent a sequence, however, NumPy currently + assumes that sequences are always collections of elements + (the sequence cannot be an element itself). + An example would be a ``vector`` DType. + +3. An array may itself contain arrays with a specific dtype (even + general Python objects). For example: + ``np.array([np.array(None, dtype=object)], dtype=np.String)`` + poses the issue of how to handle the included array. + +Some of these difficulties arise because finding the correct shape +of the output array and finding the correct datatype are closely related. + +**Implementation:** There are two distinct cases above: + +1. The user has provided no dtype information. + +2. The user provided a DType class -- as represented, for example, by ``"S"`` + representing a string of any length. + +In the first case, it is necessary to establish a mapping from the Python type(s) +of the constituent elements to the DType class. +Once the DType class is known, the correct dtype instance needs to be found. +In the case of strings, this requires to find the string length. + +These two cases shall be implemented by leveraging two pieces of information: + +1. ``DType.type``: The current type attribute to indicate which Python scalar + type is associated with the DType class (this is a *class* attribute that always + exists for any datatype and is not limited to array coercion). + +2. ``__discover_descr_from_pyobject__(cls, obj) -> dtype``: A classmethod that + returns the correct descriptor given the input object. + Note that only parametric DTypes have to implement this. + For nonparametric DTypes using the default instance will always be acceptable. + +The Python scalar type which is already associated with a DType through the +``DType.type`` attribute maps from the DType to the Python scalar type. +At registration time, a DType may choose to allow automatically discover for +this Python scalar type. +This requires a lookup in the opposite direction, which will be implemented +using global a mapping (dictionary-like) of:: + + known_python_types[type] = DType + +Correct garbage collection requires additional care. +If both the Python scalar type (``pytype``) and ``DType`` are created dynamically, +they will potentially be deleted again. +To allow this, it must be possible to make the above mapping weak. +This requires that the ``pytype`` holds a reference of ``DType`` explicitly. +Thus, in addition to building the global mapping, NumPy will store the ``DType`` as +``pytype.__associated_array_dtype__`` in the Python type. +This does *not* define the mapping and should *not* be accessed directly. +In particular potential inheritance of the attribute does not mean that NumPy will use the +superclasses ``DType`` automatically. A new ``DType`` must be created for the +subclass. + +.. note:: + + Python integers do not have a clear/concrete NumPy type associated right + now. This is because during array coercion NumPy currently finds the first + type capable of representing their value in the list of `long`, `unsigned + long`, `int64`, `unsigned int64`, and `object` (on many machines `long` is + 64 bit). + + Instead they will need to be implemented using an ``AbstractPyInt``. This + DType class can then provide ``__discover_descr_from_pyobject__`` and + return the actual dtype which is e.g. ``np.dtype("int64")``. For + dispatching/promotion in ufuncs, it will also be necessary to dynamically + create ``AbstractPyInt[value]`` classes (creation can be cached), so that + they can provide the current value based promotion functionality provided + by ``np.result_type(python_integer, array)`` [2]_ . + +To allow for a DType to accept inputs as scalars that are not basic Python +types or instances of ``DType.type``, we use ``known_scalar_type`` method. +This can allow discovery of a ``vector`` as a scalar (element) instead of a sequence +(for the command ``np.array(vector, dtype=VectorDType)``) even when ``vector`` is itself a +sequence or even an array subclass. This will *not* be public API initially, +but may be made public at a later time. + +**Example:** The current datetime DType requires a +``__discover_descr_from_pyobject__`` which returns the correct unit for string +inputs. This allows it to support:: + + np.array(["2020-01-02", "2020-01-02 11:24"], dtype="M8") + +By inspecting the date strings. Together with the common dtype +operation, this allows it to automatically find that the datetime64 unit +should be "minutes". + + +**NumPy internal implementation:** The implementation to find the correct dtype +will work similar to the following pseudocode:: + + def find_dtype(array_like): + common_dtype = None + for element in array_like: + # default to object dtype, if unknown + DType = known_python_types.get(type(element), np.dtype[object]) + dtype = DType.__discover_descr_from_pyobject__(element) + + if common_dtype is None: + common_dtype = dtype + else: + common_dtype = np.promote_types(common_dtype, dtype) + +In practice, the input to ``np.array()`` is a mix of sequences and array-like +objects, so that deciding what is an element requires to check whether it +is a sequence. +The full algorithm (without user provided dtypes) thus looks more like:: + + def find_dtype_recursive(array_like, dtype=None): + """ + Recursively find the dtype for a nested sequences (arrays are not + supported here). + """ + DType = known_python_types.get(type(element), None) + + if DType is None and is_array_like(array_like): + # Code for a sequence, an array_like may have a DType we + # can use directly: + for element in array_like: + dtype = find_dtype_recursive(element, dtype=dtype) + return dtype + + elif DType is None: + DType = np.dtype[object] + + # dtype discovery and promotion as in `find_dtype` above + +If the user provides ``DType``, then this DType will be tried first, and the +``dtype`` may need to be cast before the promotion is performed. + +**Limitations:** The motivational point 3. of a nested array +``np.array([np.array(None, dtype=object)], dtype=np.String)`` is currently +(sometimes) supported by inspecting all elements of the nested array. +User DTypes will implicitly handle these correctly if the nested array +is of ``object`` dtype. +In some other cases NumPy will retain backward compatibility for existing +functionality only. +NumPy uses such functionality to allow code such as:: + + >>> np.array([np.array(["2020-05-05"], dtype="S")], dtype=np.datetime64) + array([['2020-05-05']], dtype='datetime64[D]') + +which discovers the datetime unit ``D`` (days). +This possibility will not be accessible to user DTypes without an +intermediate cast to ``object`` or a custom function. + +The use of a global type map means that an error or warning has to be given if +two DTypes wish to map to the same Python type. In most cases user DTypes +should only be implemented for types defined within the same library to avoid +the potential for conflicts. It will be the DType implementor's responsibility +to be careful about this and use avoid registration when in doubt. + +**Alternatives:** + +- Instead of a global mapping, we could rely on the scalar attribute + ``scalar.__associated_array_dtype__``. This only creates a difference in + behavior for subclasses, and the exact implementation can be undefined + initially. Scalars will be expected to derive from a NumPy scalar. In + principle NumPy could, for a time, still choose to rely on the attribute. + +- An earlier proposal for the ``dtype`` discovery algorithm used a two-pass + approach, first finding the correct ``DType`` class and only then + discovering the parametric ``dtype`` instance. It was rejected as + needlessly complex. But it would have enabled value-based promotion + in universal functions, allowing:: + + np.add(np.array([8], dtype="uint8"), [4]) + + to return a ``uint8`` result (instead of ``int16``), which currently happens for:: + + np.add(np.array([8], dtype="uint8"), 4) + + (note the list ``[4]`` instead of scalar ``4``). + This is not a feature NumPy currently has or desires to support. + +**Further issues and discussion:** It is possible to create a DType +such as Categorical, array, or vector which can only be used if ``dtype=DType`` +is provided. Such DTypes cannot roundtrip correctly. For example:: + + np.array(np.array(1, dtype=Categorical)[()]) + +will result in an integer array. To get the original ``Categorical`` array +``dtype=Categorical`` will need to be passed explicitly. +This is a general limitation, but round-tripping is always possible if +``dtype=original_arr.dtype`` is passed. + + +.. _nep42_c-api: + +****************************************************************************** +Public C-API +****************************************************************************** DType creation -"""""""""""""" +============================================================================== + +To create a new DType the user will need to define the methods and attributes +outlined in the `Usage and impact`_ section and detailed throughout this +proposal. + +In addition, some methods similar to those in :c:type:`PyArray_ArrFuncs` will +be needed for the slots struct below. -As already mentioned in NEP 41, the interface to define new DTypes in C -is modeled after the limited API in Python: the above-mentioned slots -and some additional necessary information will thus be passed within a slots -struct and identified by ``ssize_t`` integers:: +As mentioned in :ref:`NEP 41 `, the interface to define this DType +class in C is modeled after :PEP:`384`: Slots and some additional information +will be passed in a slots struct and identified by ``ssize_t`` integers:: static struct PyArrayMethodDef slots[] = { {NPY_dt_method, method_implementation}, @@ -1093,10 +1231,8 @@ struct and identified by ``ssize_t`` integers:: } typedef struct{ - PyTypeObject *typeobj; /* type of python scalar */ - int is_parametric; /* Is the dtype parametric? */ - int is_abstract; /* Is the dtype abstract? */ - int flags /* flags (to be discussed) */ + PyTypeObject *typeobj; /* type of python scalar or NULL */ + int flags /* flags, including parametric and abstract */ /* NULL terminated CastingImpl; is copied and references are stolen */ CastingImpl *castingimpls[]; PyType_Slot *slots; @@ -1105,20 +1241,18 @@ struct and identified by ``ssize_t`` integers:: PyObject* PyArray_InitDTypeMetaFromSpec(PyArrayDTypeMeta_Spec *dtype_spec); -all of this information will be copied during instantiation. +All of this is passed by copying. -**TODO:** The DType author should be able to at define new methods for -their DType, up to defining a full type object and in the future possibly even -extending the ``PyArrayDTypeMeta_Type`` struct. -We have to decide on how (and what) to make available to the user initially. -A proposed initial solution may be to simply allow inheriting from an existing -class. -Further this prevents overriding some slots, such as `==` which may not be -desirable. +**TODO:** The DType author should be able to define new methods for the +DType, up to defining a full object, and, in the future, possibly even +extending the ``PyArrayDTypeMeta_Type`` struct. We have to decide what to make +available initially. A solution may be to allow inheriting only from an +existing class: ``class MyDType(np.dtype, MyBaseclass)``. If ``np.dtype`` is +first in the method resolution order, this also prevents an undesirable +override of slots like ``==``. - -The proposed method slots are (prepended with ``NPY_dt_``), these are -detailed above and given here for summary: +The ``slots`` will be identified by names which are prefixed with ``NPY_dt_`` +and are: * ``is_canonical(self) -> {0, 1}`` * ``ensure_canonical(self) -> dtype`` @@ -1129,216 +1263,155 @@ detailed above and given here for summary: * ``common_dtype(cls, other) -> DType, NotImplemented, or NULL`` * ``common_instance(self, other) -> dtype or NULL`` -If not set, most slots are filled with slots which either error or defer automatically. -Non-parametric dtypes do not have to implement: +Where possible, a default implementation will be provided if the slot is +omitted or set to ``NULL``. Nonparametric dtypes do not have to implement: * ``discover_descr_from_pyobject`` (uses ``default_descr`` instead) * ``common_instance`` (uses ``default_descr`` instead) -* ``ensure_canonical`` (uses ``default_descr`` instead) - -Which will be correct for most dtypes *which do not store metadata*. - -Other slots may be replaced by convenience versions, e.g. sorting methods -can be defined by providing: +* ``ensure_canonical`` (uses ``default_descr`` instead). -* ``compare(self, char *item_ptr1, char *item_ptr2, int *res) -> {-1, 0}`` - *TODO: We would like an error return, is this reasonable? (similar to old - python compare)* - -which uses generic sorting functionality. In general, we could add a -functions such as: +Sorting is expected to be implemented using: * ``get_sort_function(self, NPY_SORTKIND sort_kind) -> {out_sortfunction, NotImplemented, NULL}``. - If the sortkind is not understood it may be allowed to return ``NotImplemented``. -in the future. However, for example sorting is likely better solved by the -implementation of multiple generalized ufuncs which are called internally. +For convenience, it will be sufficient if the user implements only: + +* ``compare(self, char *item_ptr1, char *item_ptr2, int *res) -> {-1, 0, 1}`` -**Limitations:** -Using the above ``PyArrayDTypeMeta_Spec`` struct, the structure itself can -only be extended clumsily (e.g. by adding a version tag to the ``slots`` -to indicate a new, longer version of the struct). -We could also provide the struct using a function, which however will require -memory management but would allow ABI-compatible extension -(the struct is freed again when the DType is created). +**Limitations:** The ``PyArrayDTypeMeta_Spec`` struct is clumsy to extend (for +instance, by adding a version tag to the ``slots`` to indicate a new, longer +version). We could use a function to provide the struct; it would require +memory management but would allow ABI-compatible extension (the struct is +freed again when the DType is created). CastingImpl -""""""""""" +============================================================================== The external API for ``CastingImpl`` will be limited initially to defining: -* ``cast_kind`` attribute, which can be one of the supported casting kinds. - This is the safest cast possible. For example casting between two NumPy +* ``casting`` attribute, which can be one of the supported casting kinds. + This is the safest cast possible. For example, casting between two NumPy strings is of course "safe" in general, but may be "same kind" in a specific instance if the second string is shorter. If neither type is parametric the - ``adjust_descriptors`` must use it. -* ``adjust_descriptors(dtypes_in[2], dtypes_out[2], casting_out) -> int {0, -1}`` - The out dtypes must be set correctly to dtypes which the strided loop - (transfer function) can handle. Initially the result must have be instances - of the same DType class as the ``CastingImpl`` is defined for. - The ``casting_out`` will be set to ``NPY_SAFE_CASTING``, ``NPY_UNSAFE_CASTING``, - or ``NPY_SAME_KIND_CASTING``. With a new, additional, flag ``NPY_CAST_IS_VIEW`` - which can be set to indicate that no cast is necessary, but a simple view - is sufficient to perform the cast. - The cast should return ``-1`` when a custom error message is set and - ``NPY_NO_CASTING`` to indicate that a generic casting error should be - set (this is in most cases preferable). -* ``strided_loop(char **args, npy_intp *dimensions, npy_intp *strides, dtypes[2]) -> int {0, nonzero}`` (must currently succeed) - -This is identical to the proposed API for ufuncs. By default the two dtypes -are passed in as the last argument. On error return (if no error is set) a -generic error will be given. -More optimized loops are in use internally, and will be made available to users -in the future (see notes) -The iterator API does not currently support casting errors: this is -a bug that needs to be fixed. Until it is fixed the loop should always -succeed (return 0). - -Although verbose, the API shall mimic the one for creating a new DType. -The ``PyArrayCastingImpl_Spec`` will include a field for ``dtypes`` and -identical to a ``PyArrayUFuncImpl_Spec``:: + ``resolve_descriptors`` must use it. + +* ``resolve_descriptors(dtypes_in[2], dtypes_out[2], casting_out) -> int {0, + -1}`` The out + dtypes must be set correctly to dtypes which the strided loop + (transfer function) can handle. Initially the result must have instances + of the same DType class as the ``CastingImpl`` is defined for. The + ``casting`` will be set to ``NPY_EQUIV_CASTING``, ``NPY_SAFE_CASTING``, + ``NPY_UNSAFE_CASTING``, or ``NPY_SAME_KIND_CASTING``. + A new, additional flag, + ``NPY_CAST_IS_VIEW``, can be set to indicate that no cast is necessary and a + view is sufficient to perform the cast. The cast should return + ``-1`` when a custom error is set and ``NPY_NO_CASTING`` to indicate + that a generic casting error should be set (this is in most cases + preferable). + +* ``strided_loop(char **args, npy_intp *dimensions, npy_intp *strides, + ...) -> int {0, -1}`` (signature will be fully defined in :ref:`NEP 43 `) + +This is identical to the proposed API for ufuncs. The additional ``...`` +part of the signature will include information such as the two ``dtype``\s. +More optimized loops are in use internally, and +will be made available to users in the future (see notes). + +Although verbose, the API will mimic the one for creating a new DType: + +.. code-block:: C typedef struct{ - int needs_api; /* whether the cast requires the API */ - PyArray_DTypeMeta *in_dtype; /* input DType class */ - PyArray_DTypeMeta *out_dtype; /* output DType class */ + int flags; /* e.g. whether the cast requires the API */ + int nin, nout; /* Number of Input and outputs (always 1) */ + NPY_CASTING casting; /* The default casting level */ + PyArray_DTypeMeta *dtypes; /* input and output DType class */ /* NULL terminated slots defining the methods */ PyType_Slot *slots; - } PyArrayUFuncImpl_Spec; + } PyArrayMethod_Spec; -The actual creation function ``PyArrayCastingImpl_FromSpec()`` will additionally -require a ``casting`` parameter to define the default (maximum) casting safety. -The internal representation of ufuncs and casting implementations may differ -initially if it makes implementation simpler, but should be kept opaque to -allow future merging. +The focus differs between casting and general ufuncs. For example, for casts +``nin == nout == 1`` is always correct, while for ufuncs ``casting`` is +expected to be usually `"safe"`. -**TODO:** It may be possible to make this more close to the ufuncs or even -use a single FromSpec. This API shall only be finalized after/when NEP 43 -is finalized. +**Notes:** We may initially allow users to define only a single loop. +Internally NumPy optimizes far more, and this should be made public +incrementally in one of two ways: -**Notes:** +* Allow multiple versions, such as: -We may initially allow users to define only a single loop. -However, internally NumPy optimizes far more, and this should be made -public incrementally, by either allowing to provide multiple versions, such -as: + * contiguous inner loop + * strided inner loop + * scalar inner loop -* contiguous inner loop -* strided inner loop -* scalar inner loop +* Or, more likely, expose the ``get_loop`` function which is passed additional + information, such as the fixed strides (similar to our internal API). -or more likely through an additional ``get_inner_loop`` function which has -additional information, such as the fixed strides (similar to our internal API). +The example does not yet include setup and error handling. Since these are +similar to the UFunc machinery, they will be defined in :ref:`NEP 43 ` and then +incorporated identically into casting. -The above example does not yet include the definition of setup/teardown -functionality, which may overlap with ``get_inner_loop``. -Since these are similar to the UFunc machinery, this should be defined in -detail in NEP 43 and then incorporated identically into casting. +The slots/methods used will be prefixed with ``NPY_meth_``. -Also the ``needs_api`` decision may actually be moved into a setup function, -and removed or mainly provided as a convenience flag. -The slots/methods used will be prefixed ``NPY_uf_`` for similarity to the ufunc -machinery. +**Alternatives:** +- Aside from name changes and signature tweaks, there seem to be few + alternatives to the above structure. The proposed API using ``*_FromSpec`` + function is a good way to achieve a stable and extensible API. The slots + design is extensible and can be changed without breaking binary + compatibility. Convenience functions can still be provided to allow creation + with less code. - -Alternatives -"""""""""""" - -Aside from name changes, and possible signature tweaks, there seem to -be few alternatives to the above structure. -Keeping the creation process close the Python limited API has some advantage. -Convenience functions could still be provided to allow creation with less -code. -The central point in the above design is that the enumerated slots design -is extensible and can be changed without breaking binary compatibility. -A downside is the possible need to pass in e.g. integer flags using a void -pointer inside this structure. - -A downside of this is that compilers cannot warn about function -pointer incompatibilities. There is currently no proposed solution to this. - - -Issues -^^^^^^ - -Any possible design decision will have issues. - -The above split into Python objects has the disadvantage that reference cycles -naturally occur. For example a potential ``CastingImpl`` object needs to -hold on to both ``DTypes``. Further, a scalar type may want to own a -strong reference to the corresponding ``DType`` while the ``DType`` *must* -hold a strong reference to the scalar. -We do not believe that these reference cycles are an issue. The may -require implementation of of cyclic reference counting at some point, but -cyclic reference resolution is very common in Python and dtypes (especially -classes) are only a small number of objects. - -In some cases, the new split will add additional indirections to the code, -since methods on the DType have to be looked up and called. -This should not have serious performance impact and seems necessary to -achieve the desired flexibility. - -From a user-perspective, a more serious downside is that handling certain -functionality in the ``DType`` rather than directly can mean that error -messages need to be raised from places where less context is available. -This may mean that error messages can be less specific. -This will be alleviated by exception chaining. Also decisions such as -returning the casting safety (even when it is impossible to cast) allow -most exceptions to be set at a point where more context is available -and ensures uniform errors messages. +- One downside is that compilers cannot warn about function-pointer + incompatibilities. +****************************************************************************** Implementation --------------- - -Internally a few implementation details have to be decided. These will be -fully opaque to the user and can be changed at a later time. +****************************************************************************** -This includes: +Steps for implementation are outlined in the Implementation section of +:ref:`NEP 41 `. In brief, we first will rewrite the internals of +casting and array coercion. After that, the new public API will be added +incrementally. We plan to expose it in a preliminary state initially to gain +experience. All functionality currently implemented on the dtypes will be +replaced systematically as new features are added. -* How ``CastingImpl`` lookup, and thus the decision whether a cast is possible, - is defined. (This is speed relevant, although mainly during a transition - phase where UFuncs where NEP 43 is not yet implemented). - Thus, it is not very relevant to the NEP. It is only necessary to ensure fast - lookup during the transition phase for the current builtin Numerical types. -* How the mapping from a python scalar (e.g. ``3.``) to the DType is - implemented. - -The main steps for implementation are outlined in :ref:`NEP 41 `. -This includes the internal restructure for how casting and array-coercion -works. -After this the new public API will be added incrementally. -This includes replacements for certain slots which are occasionally -directly used on the dtype (e.g. ``dtype->f->setitem``). - - -Discussion ----------- +****************************************************************************** +Alternatives +****************************************************************************** -There is a large space of possible implementations with many discussions -in various places, as well as initial thoughts and design documents. -These are listed in the discussion of NEP 40 and not repeated here for -brevity. +The space of possible implementations is large, so there have been many +discussions, conceptions, and design documents. These are listed in +:ref:`NEP 40 `. Alternatives were also been discussed in the +relevant sections above. +****************************************************************************** References ----------- +****************************************************************************** + +.. [1] To be clear, the program is broken: It should not have stored a value + in the common DType that was below the lowest int16 or above the highest + uint16. It avoided overflow earlier by an accident of implementation. + Nonetheless, we insist that program behavior not be altered just by + importing a type. -.. [1] NumPy currently inspects the value to allow the operations:: +.. [2] NumPy currently inspects the value to allow the operations:: np.array([1], dtype=np.uint8) + 1 np.array([1.2], dtype=np.float32) + 1. to return a ``uint8`` or ``float32`` array respectively. This is - further described in the documentation of `numpy.result_type`. + further described in the documentation for :func:`numpy.result_type`. +****************************************************************************** Copyright ---------- +****************************************************************************** This document has been placed in the public domain. diff --git a/doc/neps/nep-0043-extensible-ufuncs.rst b/doc/neps/nep-0043-extensible-ufuncs.rst new file mode 100644 index 000000000000..96d4794f3774 --- /dev/null +++ b/doc/neps/nep-0043-extensible-ufuncs.rst @@ -0,0 +1,1309 @@ +.. _NEP43: + +============================================================================== +NEP 43 — Enhancing the Extensibility of UFuncs +============================================================================== + +:title: Enhancing the Extensibility of UFuncs +:Author: Sebastian Berg +:Status: Draft +:Type: Standard +:Created: 2020-06-20 + + +.. note:: + + This NEP is fourth in a series: + + - :ref:`NEP 40 ` explains the shortcomings of NumPy's dtype implementation. + + - :ref:`NEP 41 ` gives an overview of our proposed replacement. + + - :ref:`NEP 42 ` describes the new design's datatype-related APIs. + + - NEP 43 (this document) describes the new design's API for universal functions. + + +****************************************************************************** +Abstract +****************************************************************************** + +The previous NEP 42 proposes the creation of new DTypes which can +be defined by users outside of NumPy itself. +The implementation of NEP 42 will enable users to create arrays with a custom dtype +and stored values. +This NEP outlines how NumPy will operate on arrays with custom dtypes in the future. +The most important functions operating on NumPy arrays are the so called +"universal functions" (ufunc) which include all math functions, such as +``np.add``, ``np.multiply``, and even ``np.matmul``. +These ufuncs must operate efficiently on multiple arrays with +different datatypes. + +This NEP proposes to expand the design of ufuncs. +It makes a new distinction between the ufunc which can operate +on many different dtypes such as floats or integers, +and a new ``ArrayMethod`` which defines the efficient operation for +specific dtypes. + +.. note:: + + Details of the private and external APIs may change to reflect user + comments and implementation constraints. The underlying principles and + choices should not change significantly. + + +****************************************************************************** +Motivation and scope +****************************************************************************** + +The goal of this NEP is to extend universal +functions support the new DType system detailed in NEPs 41 and 42. +While the main motivation is enabling new user-defined DTypes, this will +also significantly simplify defining universal functions for NumPy strings or +structured DTypes. +Until now, these DTypes are not supported by any of NumPy's functions +(such as ``np.add`` or ``np.equal``), due to difficulties arising from +their parametric nature (compare NEP 41 and 42), e.g. the string length. + +Functions on arrays must handle a number of distinct steps which are +described in more detail in section "`Steps involved in a UFunc call`_". +The most important ones are: + +- Organizing all functionality required to define a ufunc call for specific + DTypes. This is often called the "inner-loop". +- Deal with input for which no exact matching implementation is found. + For example when ``int32`` and ``float64`` are added, the ``int32`` + is cast to ``float64``. This requires a distinct "promotion" step. + +After organizing and defining these, we need to: + +- Define the user API for customizing both of the above points. +- Allow convenient reuse of existing functionality. + For example a DType representing physical units, such as meters, + should be able to fall back to NumPy's existing math implementations. + +This NEP details how these requirements will be achieved in NumPy: + +- All DTyper-specific functionality currently part of the ufunc + definition will be defined as part of a new `ArrayMethod`_ object. + This ``ArrayMethod`` object will be the new, preferred, way to describe any + function operating on arrays. + +- Ufuncs will dispatch to the ``ArrayMethod`` and potentially use promotion + to find the correct ``ArrayMethod`` to use. + This will be described in the `Promotion and dispatching`_ section. + +A new C-API will be outlined in each section. A future Python API is +expected to be very similar and the C-API is presented in terms of Python +code for readability. + +The NEP proposes a large, but necessary, refactor of the NumPy ufunc internals. +This modernization will not affect end users directly and is not only a necessary +step for new DTypes, but in itself a maintenance effort which is expected to +help with future improvements to the ufunc machinery. + +While the most important restructure proposed is the new ``ArrayMethod`` +object, the largest long-term consideration is the API choice for +promotion and dispatching. + + +*********************** +Backwards Compatibility +*********************** + +The general backwards compatibility issues have also been listed +previously in NEP 41. + +The vast majority of users should not see any changes beyond those typical +for NumPy releases. +There are three main users or use-cases impacted by the proposed changes: + +1. The Numba package uses direct access to the NumPy C-loops and modifies + the NumPy ufunc struct directly for its own purposes. +2. Astropy uses its own "type resolver", meaning that a default switch over + from the existing type resolution to a new default Promoter requires care. +3. It is currently possible to register loops for dtype *instances*. + This is theoretically useful for structured dtypes and is a resolution + step happening *after* the DType resolution step proposed here. + +This NEP will try hard to maintain backward compatibility as much as +possible. However, both of these projects have signaled willingness to adapt +to breaking changes. + +The main reason why NumPy will be able to provide backward compatibility +is that: + +* Existing inner-loops can be wrapped, adding an indirection to the call but + maintaining full backwards compatibility. + The ``get_loop`` function can, in this case, search the existing + inner-loop functions (which are stored on the ufunc directly) in order + to maintain full compatibility even with potential direct structure access. +* Legacy type resolvers can be called as a fallback (potentially caching + the result). The resolver may need to be called twice (once for the DType + resolution and once for the ``resolve_descriptor`` implementation). +* The fallback to the legacy type resolver should in most cases handle loops + defined for such structured dtype instances. This is because if there is no + other ``np.Void`` implementation, the legacy fallback will retain the old + behaviour at least initially. + +The masked type resolvers specifically will *not* remain supported, but +has no known users (including NumPy itself, which only uses the default +version). + +While the above changes potentially break some workflows, +we believe that the long-term improvements vastly outweigh this. +Further, packages such as astropy and Numba are capable of adapting so that +end-users may need to update their libraries but not their code. + + +****************************************************************************** +Usage and impact +****************************************************************************** + +This NEP restructures how operations on NumPy arrays are defined both +within NumPy and for external implementers. +The NEP mainly concerns those who either extend ufuncs for custom DTypes +or create custom ufuncs. It does not aim to finalize all +potential use-cases, but rather restructure NumPy to be extensible and allow +addressing new issues or feature requests as they arise. + + +Overview and end user API +========================= + +To give an overview of how this NEP proposes to structure ufuncs, +the following describes the potential exposure of the proposed restructure +to the end user. + +Universal functions are much like a Python method defined on the DType of +the array when considering a ufunc with only a single input:: + + res = np.positive(arr) + +could be implemented (conceptually) as:: + + positive_impl = arr.dtype.positive + res = positive_impl(arr) + +However, unlike methods, ``positive_impl`` is not stored on the dtype itself. +It is rather the implementation of ``np.positive`` for a specific DType. +Current NumPy partially exposes this "choice of implementation" using +the ``dtype`` (or more exact ``signature``) attribute in universal functions, +although these are rarely used:: + + np.positive(arr, dtype=np.float64) + +forces NumPy to use the ``positive_impl`` written specifically for the Float64 +DType. + +This NEP makes the distinction more explicit, by creating a new object to +represent ``positive_impl``:: + + positive_impl = np.positive.resolve_impl((type(arr.dtype), None)) + # The `None` represents the output DType which is automatically chosen. + +While the creation of a ``positive_impl`` object and the ``resolve_impl`` +method is part of this NEP, the following code:: + + res = positive_impl(arr) + +may not be implemented initially and is not central to the redesign. + +In general NumPy universal functions can take many inputs. +This requires looking up the implementation by considering all of them +and makes ufuncs "multi-methods" with respect to the input DTypes:: + + add_impl = np.add.resolve_impl((type(arr1.dtype), type(arr2.dtype), None)) + +This NEP defines how ``positive_impl`` and ``add_impl`` will be represented +as a new ``ArrayMethod`` which can be implemented outside of NumPy. +Further, it defines how ``resolve_impl`` will implement and solve dispatching +and promotion. + +The reasons for this split may be more clear after reviewing the +`Steps involved in a UFunc call`_ section. + + +Defining a new ufunc implementation +=================================== + +The following is a mock-up of how a new implementation, in this case +to define string equality, will be added to a ufunc. + +.. code-block:: python + + class StringEquality(BoundArrayMethod): + nin = 1 + nout = 1 + DTypes = (String, String, Bool) + + def resolve_descriptors(context, given_descrs): + """The strided loop supports all input string dtype instances + and always returns a boolean. (String is always native byte order.) + + Defining this function is not necessary, since NumPy can provide + it by default. + """ + assert isinstance(given_descrs[0], context.DTypes[0]) + assert isinstance(given_descrs[1], context.DTypes[1]) + + # The operation is always "safe" casting (most ufuncs are) + return (given_descrs[0], given_descrs[1], context.DTypes[2]()), "safe" + + def strided_loop(context, dimensions, data, strides, innerloop_data): + """The 1-D strided loop, similar to those used in current ufuncs""" + # dimensions: Number of loop items and core dimensions + # data: Pointers to the array data. + # strides: strides to iterate all elements + n = dimensions[0] # number of items to loop over + num_chars1 = context.descriptors[0].itemsize + num_chars2 = context.descriptors[1].itemsize + + # C code using the above information to compare the strings in + # both arrays. In particular, this loop requires the `num_chars1` + # and `num_chars2`. Information which is currently not easily + # available. + + np.equal.register_impl(StringEquality) + del StringEquality # may be deleted. + + +This definition will be sufficient to create a new loop, and the +structure allows for expansion in the future; something that is already +required to implement casting within NumPy itself. +We use ``BoundArrayMethod`` and a ``context`` structure here. These +are described and motivated in details later. Briefly: + +* ``context`` is a generalization of the ``self`` that Python passes to its + methods. +* ``BoundArrayMethod`` is equivalent to the Python distinction that + ``class.method`` is a method, while ``class().method`` returns a "bound" method. + + +Customizing Dispatching and Promotion +===================================== + +Finding the correct implementation when ``np.positive.resolve_impl()`` is +called is largely an implementation detail. +But, in some cases it may be necessary to influence this process when no +implementation matches the requested DTypes exactly: + +.. code-block:: python + + np.multiple.resolve_impl((Timedelta64, Int8, None)) + +will not have an exact match, because NumPy only has an implementation for +multiplying ``Timedelta64`` with ``Int64``. +In simple cases, NumPy will use a default promotion step to attempt to find +the correct implementation, but to implement the above step, we will allow +the following: + +.. code-block:: python + + def promote_timedelta_integer(ufunc, dtypes): + new_dtypes = (Timdelta64, Int64, dtypes[-1]) + # Resolve again, using Int64: + return ufunc.resolve_impl(new_dtypes) + + np.multiple.register_promoter( + (Timedelta64, Integer, None), promote_timedelta_integer) + +Where ``Integer`` is an abstract DType (compare NEP 42). + + +.. _steps_of_a_ufunc_call: + +**************************************************************************** +Steps involved in a UFunc call +**************************************************************************** + +Before going into more detailed API choices, it is helpful to review the +steps involved in a call to a universal function in NumPy. + +A UFunc call is split into the following steps: + +1. *Handle ``__array_ufunc__`` protocol:* + + * For array-likes such as a Dask arrays, NumPy can defer the operation. + This step is performed first, and unaffected by this NEP (compare :ref:`NEP18`). + +2. *Promotion and dispatching* + + * Given the DTypes of all inputs, find the correct implementation. + E.g. an implementation for ``float64``, ``int64`` or a user-defined DType. + + * When no exact implementation exists, *promotion* has to be performed. + For example, adding a ``float32`` and a ``float64`` is implemented by + first casting the ``float32`` to ``float64``. + +3. *Parametric ``dtype`` resolution:* + + * In general, whenever an output DType is parametric the parameters have + to be found (resolved). + * For example, if a loop adds two strings, it is necessary to define the + correct output (and possibly input) dtypes. ``S5 + S4 -> S9``, while + an ``upper`` function has the signature ``S5 -> S5``. + * When they are not parametric, a default implementation is provided + which fills in the default dtype instances (ensuring for example native + byte order). + +4. *Preparing the iteration:* + + * This step is largely handled by ``NpyIter`` internally (the iterator). + * Allocate all outputs and temporary buffers necessary to perform casts. + * Find the best iteration order, which includes information to efficiently + implement broadcasting. For example, adding a single value to an array + repeats the same value. + +5. *Setup and fetch the C-level function:* + + * If necessary, allocate temporary working space. + * Find the C-implemented, light weight, inner-loop function. + Finding the inner-loop function can allow specialized implementations + in the future. + For example casting currently optimizes contiguous casts and + reductions have optimizations that are currently handled + inside the inner-loop function itself. + * Signal whether the inner-loop requires the Python API or whether + the GIL may be released (to allow threading). + * Clear floating point exception flags. + +6. *Perform the actual calculation:* + + * Run the DType specific inner-loop function. + * The inner-loop may require access to additional data, such as dtypes or + additional data set in the previous step. + * The inner-loop function may be called an undefined number of times. + +7. *Finalize:* + + * Free any temporary working space allocated in 5. + * Check for floating point exception flags. + * Return the result. + +The ``ArrayMethod`` provides a concept to group steps 3 to 6 and partially 7. +However, implementers of a new ufunc or ``ArrayMethod`` do not need to +customize the behaviour in steps 4 or 6, aside from the inner-loop function. +For the ``ArrayMethod`` implementer, the central steps to have control over +are step 3 and step 5 to provide the custom inner-loop function. +Further customization is a potential future extension. + +Step 2. is promotion and dispatching which will also be restructured +with new API which allows influencing the process where necessary. + +Step 1 is listed for completeness and is unaffected by this NEP. + +The following sketch provides an overview of step 2 to 6 with an emphasize +of how dtypes are handled: + +.. figure:: _static/nep43-sketch.svg + :figclass: align-center + + +***************************************************************************** +ArrayMethod +***************************************************************************** + +A central proposal of this NEP is the creation of the ``ArrayMethod`` as an object +describing each implementation specific to a given set of DTypes. +We use the ``class`` syntax to describe the information required to create +a new ``ArrayMethod`` object: + +.. code-block:: python + :dedent: 0 + + class ArrayMethod: + name: str # Name, mainly useful for debugging + + # Casting safety information (almost always "safe", necessary to + # unify casting and universal functions) + casting: Casting = "safe" + + # More general flags: + flags: int + + @staticmethod + def resolve_descriptors( + Context: context, Tuple[DType]: given_descrs)-> Casting, Tuple[DType]: + """Returns the safety of the operation (casting safety) and the + """ + # A default implementation can be provided for non-parametric + # output dtypes. + raise NotImplementedError + + @staticmethod + def get_loop(Context : context, strides, ...) -> strided_loop_function, flags: + """Returns the low-level C (strided inner-loop) function which + performs the actual operation. + + This method may initially private, users will be able to provide + a set of optimized inner-loop functions instead: + + * `strided_inner_loop` + * `contiguous_inner_loop` + * `unaligned_strided_loop` + * ... + """ + raise NotImplementedError + + @staticmethod + def strided_inner_loop( + Context : context, data, dimensions, strides, innerloop_data): + """The inner-loop (equivalent to the current ufunc loop) + which is returned by the default `get_loop()` implementation.""" + raise NotImplementedError + +With ``Context`` providing mostly static information about the function call: + +.. code-block:: python + :dedent: 0 + + class Context: + # The ArrayMethod object itself: + ArrayMethod : method + + # Information about the caller, e.g. the ufunc, such as `np.add`: + callable : caller = None + # The number of input arguments: + int : nin = 1 + # The number of output arguments: + int : nout = 1 + # The DTypes this Method operates on/is defined for: + Tuple[DTypeMeta] : dtypes + # The actual dtypes instances the inner-loop operates on: + Tuple[DType] : descriptors + + # Any additional information required. In the future, this will + # generalize or duplicate things currently stored on the ufunc: + # - The ufunc signature of generalized ufuncs + # - The identity used for reductions + +And ``flags`` stored properties, for whether: + +* the ``ArrayMethod`` supports unaligned input and output arrays +* the inner-loop function requires the Python API (GIL) +* NumPy has to check the floating point error CPU flags. + +*Note: More information is expected to be added as necessary.* + + +The call ``Context`` +==================== + +The "context" object is analogous to Python's ``self`` that is +passed to all methods. +To understand why the "context" object is necessary and its +internal structure, it is helpful to remember +that a Python method can be written in the following way +(see also the `documentation of __get__ +`_): + +.. code-block:: python + + class BoundMethod: + def __init__(self, instance, method): + self.instance = instance + self.method = method + + def __call__(self, *args, **kwargs): + return self.method.function(self.instance, *args, **kwargs) + + + class Method: + def __init__(self, function): + self.function = function + + def __get__(self, instance, owner=None): + assert instance is not None # unsupported here + return BoundMethod(instance, self) + + +With which the following ``method1`` and ``method2`` below, behave identically: + +.. code-block:: python + + def function(self): + print(self) + + class MyClass: + def method1(self): + print(self) + + method2 = Method(function) + +And both will print the same result: + +.. code-block:: python + + >>> myinstance = MyClass() + >>> myinstance.method1() + <__main__.MyClass object at 0x7eff65436d00> + >>> myinstance.method2() + <__main__.MyClass object at 0x7eff65436d00> + +Here ``self.instance`` would be all information passed on by ``Context``. +The ``Context`` is a generalization and has to pass additional information: + +* Unlike a method which operates on a single class instance, the ``ArrayMethod`` + operates on many input arrays and thus multiple dtypes. +* The ``__call__`` of the ``BoundMethod`` above contains only a single call + to a function. But an ``ArrayMethod`` has to call ``resolve_descriptors`` + and later pass on that information to the inner-loop function. +* A Python function has no state except that defined by its outer scope. + Within C, ``Context`` is able to provide additional state if necessary. + +Just as Python requires the distinction of a method and a bound method, +NumPy will have a ``BoundArrayMethod``. +This stores all of the constant information that is part of the ``Context``, +such as: + +* the ``DTypes`` +* the number of input and ouput arguments +* the ufunc signature (specific to generalized ufuncs, compare :ref:`NEP20`). + +Fortunately, most users and even ufunc implementers will not have to worry +about these internal details; just like few Python users need to know +about the ``__get__`` dunder method. +The ``Context`` object or C-structure provides all necessary data to the +fast C-functions and NumPy API creates the new ``ArrayMethod`` or +``BoundArrayMethod`` as required. + + +.. _ArrayMethod_specs: + +ArrayMethod Specifications +========================== + +.. highlight:: c + +These specifications provide a minimal initial C-API, which shall be expanded +in the future, for example to allow specialized inner-loops. + +Briefly, NumPy currently relies on strided inner-loops and this +will be the only allowed method of defining a ufunc initially. +We expect the addition of a ``setup`` function or exposure of ``get_loop`` +in the future. + +UFuncs require the same information as casting, giving the following +definitions (see also :ref:`NEP 42 ` ``CastingImpl``): + +* A new structure to be passed to the resolve function and inner-loop:: + + typedef struct { + PyObject *caller; /* The ufunc object */ + PyArrayMethodObject *method; + + int nin, nout; + + PyArray_DTypeMeta **dtypes; + /* Operand descriptors, filled in by resolve_desciptors */ + PyArray_Descr **descriptors; + + void *reserved; // For Potential in threading (Interpreter state) + } PyArrayMethod_Context + + This structure may be appended to include additional information in future + versions of NumPy and includes all constant loop metadata. + + We could version this structure, although it may be simpler to version + the ``ArrayMethod`` itself. + +* Similar to casting, ufuncs may need to find the correct loop dtype + or indicate that a loop is only capable of handling certain instances of + the involved DTypes (e.g. only native byteorder). This is handled by + a ``resolve_descriptors`` function (identical to the ``resolve_descriptors`` + of ``CastingImpl``):: + + NPY_CASTING + resolve_descriptors( + PyArrayMethod_Context *context, + PyArray_Descr *given_dtypes[nin+nout], + PyArray_Descr *loop_dtypes[nin+nout]); + + The function fills ``loop_dtypes`` based on the given ``given_dtypes``. + This requires filling in the descriptor of the output(s). + Often also the input descriptor(s) have to be found, e.g. to ensure native + byteorder when needed by the inner-loop. + + In most cases an ``ArrayMethod`` will have non-parametric output DTypes + so that a default implementation can be provided. + +* An additional ``void *user_data`` will usually be typed to extend + the existing ``NpyAuxData *`` struct:: + + struct { + NpyAuxData_FreeFunc *free; + NpyAuxData_CloneFunc *clone; + /* To allow for a bit of expansion without breaking the ABI */ + void *reserved[2]; + } NpyAuxData; + + This struct is currently mainly used for the NumPy internal casting + machinery and as of now both ``free`` and ``clone`` must be provided, + although this could be relaxed. + + Unlike NumPy casts, the vast majority of ufuncs currently do not require + this additional scratch-space, but may need simple flagging capability + for example for implementing warnings (see Error and Warning Handling below). + To simplify this NumPy will pass a single zero initialized ``npy_intp *`` + when ``user_data`` is not set. + *NOTE that it would be possible to pass this as part of ``Context``.* + +* The optional ``get_loop`` function will not be public initially, to avoid + finalizing the API which requires design choices also with casting: + + .. code-block:: + + innerloop * + get_loop( + PyArrayMethod_Context *context, + /* (move_references is currently used internally for casting) */ + int aligned, int move_references, + npy_intp *strides, + PyArray_StridedUnaryOp **out_loop, + NpyAuxData **innerloop_data, + NPY_ARRAYMETHOD_FLAGS *flags); + + The ``NPY_ARRAYMETHOD_FLAGS`` can indicate whether the Python API is required + and floating point errors must be checked. + +* The inner-loop function:: + + int inner_loop(PyArrayMethod_Context *context, ..., void *innerloop_data); + + Will have the identical signature to current inner-loops with the following + changes: + + * A return value to indicate an error when returning ``-1`` instead of ``0``. + When returning ``-1`` a Python error must be set. + * The new first argument ``PyArrayMethod_Context *`` is used to pass in + potentially required information about the ufunc or descriptors in a + convenient way. + * The ``void *innerloop_data`` will be the ``NpyAuxData **innerloop_data`` as set by + ``get_loop``. If ``get_loop`` does not set ``innerloop_data`` an ``npy_intp *`` + is passed instead (see `Error Handling`_ below for the motivation). + + *Note:* Since ``get_loop`` is expected to be private, the exact implementation + of ``innerloop_data`` can be modified until final exposure. + +Creation of a new ``BoundArrayMethod`` will use a ``PyArrayMethod_FromSpec()`` +function. A shorthand will allow direct registration to a ufunc using +``PyUFunc_AddImplementationFromSpec()``. The specification is expected +to contain the following (this may extend in the future):: + + typedef struct { + const char *name; /* Generic name, mainly for debugging */ + int nin, nout; + NPY_CASTING casting; + NPY_ARRAYMETHOD_FLAGS flags; + PyArray_DTypeMeta **dtypes; + PyType_Slot *slots; + } PyArrayMethod_Spec; + +.. highlight:: python + +Discussion and alternatives +=========================== + +The above split into an ``ArrayMethod`` and ``Context`` and the additional +requirement of a ``BoundArrayMethod`` is a necessary split mirroring the +implementation of methods and bound methods in Python. + +One reason for this requirement is that it allows storing the ``ArrayMethod`` +object in many cases without holding references to the ``DTypes`` which may +be important if DTypes are created (and deleted) dynamically. +(This is a complex topic, which does not have a complete solution in current +Python, but the approach solves the issue with respect to casting.) + +There seem to be no alternatives to this structure. Separating the +DType-specific steps from the general ufunc dispatching and promotion is +absolutely necessary to allow future extension and flexibility. +Furthermore, it allows unifying casting and ufuncs. + +Since the structure of ``ArrayMethod`` and ``BoundArrayMethod`` will be +opaque and can be extended, there are few long-term design implications aside +from the choice of making them Python objects. + + +``resolve_descriptors`` +----------------------- + +The ``resolve_descriptors`` method is possibly the main innovation of this +NEP and it is central also in the implementation of casting in NEP 42. + +By ensuring that every ``ArrayMethod`` provides ``resolve_descriptors`` we +define a unified, clear API for step 3 in `Steps involved in a UFunc call`_. +This step is required to allocate output arrays and has to happen before +casting can be prepared. + +While the returned casting-safety (``NPY_CASTING``) will almost always be +"safe" for universal functions, including it has two big advantages: + +* Returning the casting safety is central to NEP 42 for casting and + allows the unmodified use of ``ArrayMethod`` there. +* There may be a future desire to implement fast but unsafe implementations. + For example for ``int64 + int64 -> int32`` which is unsafe from a casting + perspective. Currently, this would use ``int64 + int64 -> int64`` and then + cast to ``int32``. An implementation that skips the cast would + have to signal that it effectively includes the "same-kind" cast and is + thus not considered "safe". + + +``get_loop`` method +------------------- + +Currently, NumPy ufuncs typically only provide a single strided loop, so that +the ``get_loop`` method may seem unnecessary. +For this reason we plan for ``get_loop`` to be a private function initially. + +However, ``get_loop`` is required for casting where specialized loops are +used even beyond strided and contiguous loops. +Thus, the ``get_loop`` function must be a full replacement for +the internal ``PyArray_GetDTypeTransferFunction``. + +In the future, ``get_loop`` may be made public or a new ``setup`` function +be exposed to allow more control, for example to allow allocating +working memory. +Further, we could expand ``get_loop`` and allow the ``ArrayMethod`` implementer +to also control the outer iteration and not only the 1-D inner-loop. + + +Extending the inner-loop signature +---------------------------------- + +Extending the inner-loop signature is another central and necessary part of +the NEP. + +**Passing in the ``Context``:** + +Passing in the ``Context`` potentially allows for the future extension of +the signature by adding new fields to the context struct. +Furthermore it provides direct access to the dtype instances which +the inner-loop operates on. +This is necessary information for parametric dtypes since for example comparing +two strings requires knowing the length of both strings. +The ``Context`` can also hold potentially useful information such as the +the original ``ufunc``, which can be helpful when reporting errors. + +In principle passing in Context is not necessary, as all information could be +included in ``innerloop_data`` and set up in the ``get_loop`` function. +In this NEP we propose passing the struct to simplify creation of loops for +parametric DTypes. + +**Passing in user data:** + +The current casting implementation uses the existing ``NpyAuxData *`` to pass +in additional data as defined by ``get_loop``. +There are certainly alternatives to the use of this structure, but it +provides a simple solution, which is already used in NumPy and public API. + +``NpyAyxData *`` is a light weight, allocated structure and since it already +exists in NumPy for this purpose, it seems a natural choice. +To simplify some use-cases (see "Error Handling" below), we will pass a +``npy_intp *innerloop_data = 0`` instead when ``innerloop_data`` is not provided. + +*Note: Since ``get_loop`` is expected to be private initially we can gain +experience with ``innerloop_data`` before exposing it as public API.* + +**Return value:** + +The return value to indicate an error is an important, but currently missing +feature in NumPy. The error handling is further complicated by the way +CPUs signal floating point errors. +Both are discussed in the next section. + +Error Handling +"""""""""""""" + +.. highlight:: c + +We expect that future inner-loops will generally set Python errors as soon +as an error is found. This is complicated when the inner-loop is run without +locking the GIL. In this case the function will have to lock the GIL, +set the Python error and return ``-1`` to indicate an error occurred::: + + int + inner_loop(PyArrayMethod_Context *context, ..., void *innerloop_data) + { + NPY_ALLOW_C_API_DEF + + for (npy_intp i = 0; i < N; i++) { + /* calculation */ + + if (error_occurred) { + NPY_ALLOW_C_API; + PyErr_SetString(PyExc_ValueError, + "Error occurred inside inner_loop."); + NPY_DISABLE_C_API + return -1; + } + } + return 0; + } + +Floating point errors are special, since they require checking the hardware +state which is too expensive if done within the inner-loop function itself. +Thus, NumPy will handle these if flagged by the ``ArrayMethod``. +An ``ArrayMethod`` should never cause floating point error flags to be set +if it flags that these should not be checked. This could interfere when +calling multiple functions; in particular when casting is necessary. + +An alternative solution would be to allow setting the error only at the later +finalization step when NumPy will also check the floating point error flags. + +We decided against this pattern at this time. It seems more complex and +generally unnecessary. +While safely grabbing the GIL in the loop may require passing in an additional +``PyThreadState`` or ``PyInterpreterState`` in the future (for subinterpreter +support), this is acceptable and can be anticipated. +Setting the error at a later point would add complexity: for instance +if an operation is paused (which can currently happen for casting in particular), +the error check needs to run explicitly ever time this happens. + +We expect that setting errors immediately is the easiest and most convenient +solution and more complex solution may be possible future extensions. + +Handling *warnings* is slightly more complex: A warning should be +given exactly once for each function call (i.e. for the whole array) even +if naively it would be given many times. +To simplify such a use case, we will pass in ``npy_intp *innerloop_data = 0`` +by default which can be used to store flags (or other simple persistent data). +For instance, we could imagine an integer multiplication loop which warns +when an overflow occurred:: + + int + integer_multiply(PyArrayMethod_Context *context, ..., npy_intp *innerloop_data) + { + int overflow; + NPY_ALLOW_C_API_DEF + + for (npy_intp i = 0; i < N; i++) { + *out = multiply_integers(*in1, *in2, &overflow); + + if (overflow && !*innerloop_data) { + NPY_ALLOW_C_API; + if (PyErr_Warn(PyExc_UserWarning, + "Integer overflow detected.") < 0) { + NPY_DISABLE_C_API + return -1; + } + *innerloop_data = 1; + NPY_DISABLE_C_API + } + return 0; + } + +*TODO:* The idea of passing an ``npy_intp`` scratch space when ``innerloop_data`` +is not set seems convenient, but I am uncertain about it, since I am not +aware of any similar prior art. This "scratch space" could also be part of +the ``context`` in principle. + +.. highlight:: python + +Reusing existing Loops/Implementations +====================================== + +For many DTypes the above definition for adding additional C-level loops will be +sufficient and require no more than a single strided loop implementation +and if the loop works with parametric DTypes, the +``resolve_descriptors`` function *must* additionally be provided. + +However, in some use-cases it is desirable to call back to an existing implementation. +In Python, this could be achieved by simply calling into the original ufunc. + +For better performance in C, and for large arrays, it is desirable to reuse +an existing ``ArrayMethod`` as directly as possible, so that its inner-loop function +can be used directly without additional overhead. +We will thus allow to create a new, wrapping, ``ArrayMethod`` from an existing +``ArrayMethod``. + +This wrapped ``ArrayMethod`` will have two additional methods: + +* ``view_inputs(Tuple[DType]: input_descr) -> Tuple[DType]`` replacing the + user input descriptors with descriptors matching the wrapped loop. + It must be possible to *view* the inputs as the output. + For example for ``Unit[Float64]("m") + Unit[Float32]("km")`` this will + return ``float64 + int32``. The original ``resolve_descriptors`` will + convert this to ``float64 + float64``. + +* ``wrap_outputs(Tuple[DType]: input_descr) -> Tuple[DType]`` replacing the + resolved descriptors with with the desired actual loop descriptors. + The original ``resolve_descriptors`` function will be called between these + two calls, so that the output descriptors may not be set in the first call. + In the above example it will use the ``float64`` as returned (which might + have changed the byte-order), and further resolve the physical unit making + the final signature:: + + ``Unit[Float64]("m") + Unit[Float64]("m") -> Unit[Float64]("m")`` + + the UFunc machinery will take care of casting the "km" input to "m". + + +The ``view_inputs`` method allows passing the correct inputs into the +original ``resolve_descriptors`` function, while ``wrap_outputs`` ensures +the correct descriptors are used for output allocation and input buffering casts. + +An important use-case for this is that of an abstract Unit DType +with subclasses for each numeric dtype (which could be dynamically created):: + + Unit[Float64]("m") + # with Unit[Float64] being the concrete DType: + isinstance(Unit[Float64], Unit) # is True + +Such a ``Unit[Float64]("m")`` instance has a well-defined signature with +respect to type promotion. +The author of the ``Unit`` DType can implement most necessary logic by +wrapping the existing math functions and using the two additional methods +above. +Using the *promotion* step, this will allow to create a register a single +promoter for the abstract ``Unit`` DType with the ``ufunc``. +The promoter can then add the wrapped concrete ``ArrayMethod`` dynamically +at promotion time, and NumPy can cache (or store it) after the first call. + +**Alternative use-case:** + +A different use-case is that of a ``Unit(float64, "m")`` DType, where +the numerical type is part of the DType parameter. +This approach is possible, but will require a custom ``ArrayMethod`` +which wraps existing loops. +It must also always require require two steps of dispatching +(one to the ``Unit`` DType and a second one for the numerical type). + +Furthermore, the efficient implementation will require the ability to +fetch and reuse the inner-loop function from another ``ArrayMethod``. +(Which is probably necessary for users like Numba, but it is uncertain +whether it should be a common pattern and it cannot be accessible from +Python itself.) + + +.. _promotion_and_dispatching: + +************************* +Promotion and dispatching +************************* + +NumPy ufuncs are multi-methods in the sense that they operate on (or with) +multiple DTypes at once. +While the input (and output) dtypes are attached to NumPy arrays, +the ``ndarray`` type itself does not carry the information of which +function to apply to the data. + +For example, given the input:: + + int_arr = np.array([1, 2, 3], dtype=np.int64) + float_arr = np.array([1, 2, 3], dtype=np.float64) + np.add(int_arr, float_arr) + +has to find the correct ``ArrayMethod`` to perform the operation. +Ideally, there is an exact match defined, e.g. for ``np.add(int_arr, int_arr)`` +the ``ArrayMethod[Int64, Int64, out=Int64]`` matches exactly and can be used. +However, for ``np.add(int_arr, float_arr)`` there is no direct match, +requiring a promotion step. + +Promotion and dispatching process +================================= + +In general the ``ArrayMethod`` is found by searching for an exact match of +all input DTypes. +The output dtypes should *not* affect calculation, but if multiple registered +``ArrayMethod``\ s match exactly, the output DType will be used to find the +better match. +This will allow the current distinction for ``np.equal`` loops which define +both ``Object, Object -> Bool`` (default) and ``Object, Object -> Object``. + +Initially, an ``ArrayMethod`` will be defined for *concrete* DTypes only +and since these cannot be subclassed an exact match is guaranteed. +In the future we expect that ``ArrayMethod``\ s can also be defined for +*abstract* DTypes. In which case the best match is found as detailed below. + +**Promotion:** + +If a matching ``ArrayMethod`` exists, dispatching is straight forward. +However, when it does not, require additional definitions to implement +promotion: + +* By default any UFunc has a promotion which uses the common DType of all + inputs and dispatches a second time. This is well-defined for most + mathematical functions, but can be disabled or customized if necessary. + For instances ``int32 + float64`` tries again using ``float64 + float64`` + which is the common DType. + +* Users can *register* new Promoters just as they can register a + new ``ArrayMethod``. These will use abstract DTypes to allow matching + a large variety of signatures. + The return value of a promotion function shall be a new ``ArrayMethod`` + or ``NotImplemented``. It must be consistent over multiple calls with + the same input to allow caching of the result. + +The signature of a promotion function would be:: + + promoter(np.ufunc: ufunc, Tuple[DTypeMeta]: DTypes): -> Union[ArrayMethod, NotImplemented] + +Note that DTypes may include the output's DType, however, normally the +output DType will *not* affect which ``ArrayMethod`` is chosen. + +In most cases, it should not be necessary to add a custom promotion function. +An example which requires this is multiplication with a unit: +in NumPy ``timedelta64`` can be multiplied with most integers, +but NumPy only defines a loop (``ArrayMethod``) for ``timedelta64 * int64`` +so that multiplying with ``int32`` would fail. + +To allow this, the following promoter can be registered for +``(Timedelta64, Integral, None)``:: + + def promote(ufunc, DTypes): + res = list(DTypes) + try: + res[1] = np.common_dtype(DTypes[1], Int64) + except TypeError: + return NotImplemented + + # Could check that res[1] is actually Int64 + return ufunc.resolve_impl(tuple(res)) + +In this case, just as a ``Timedelta64 * int64`` and ``int64 * timedelta64`` +``ArrayMethod`` is necessary, a second promoter will have to be registered to +handle the case where the integer is passed first. + +**Dispatching rules for ``ArrayMethod`` and Promoters:** + +Promoter and ``ArrayMethod`` are discovered by finding the best match as +defined by the DType class hierarchy. +The best match is defined if: + +* The signature matches for all input DTypes, so that + ``issubclass(input_DType, registered_DType)`` returns true. +* No other promoter or ``ArrayMethod`` is more precise in any input: + ``issubclass(other_DType, this_DType)`` is true (this may include if both + are identical). +* This promoter or ``ArrayMethod`` is more precise in at least one input or + output DType. + +It will be an error if ``NotImplemented`` is returned or if two +promoters match the input equally well. +When an existing promoter is not precise enough for new functionality, a +new promoter has to be added. +To ensure that this promoter takes precedence it may be necessary to define +new abstract DTypes as more precise subclasses of existing ones. + +The above rules enable specialization if an output is supplied +or the full loop is specified. This should not typically be necessary, +but allows resolving ``np.logic_or``, etc. which have both +``Object, Object -> Bool`` and ``Object, Object -> Object`` loops (using the +first by default). + + +Discussion and alternatives +=========================== + +Instead of resolving and returning a new implementation, we could also +return a new set of DTypes to use for dispatching. This works, however, +it has the disadvantage that it is impossible to dispatch to a loop +defined on a different ufunc or to dynamically create a new ``ArrayMethod``. + + +**Rejected Alternatives:** + +In the above the promoters use a multiple dispatching style type resolution +while the current UFunc machinery uses the first +"safe" loop (see also :ref:`NEP 40 `) in an ordered hierarchy. + +While the "safe" casting rule is not restrictive enough, we could imagine +using a new "promote" casting rule, or the common-DType logic to find the +best matching loop by upcasting the inputs as necessary. + +One downside to this approach is that upcasting alone allows upcasting the +result beyond what is expected by users: +Currently (which will remain supported as a fallback) any ufunc which defines +only a float64 loop will also work for float16 and float32 by *upcasting*:: + + >>> from scipy.special import erf + >>> erf(np.array([4.], dtype=np.float16)) # float16 + array([1.], dtype=float32) + +with a float32 result. It is impossible to change the ``erf`` function to +return a float16 result without changing the result of following code. +In general, we argue that automatic upcasting should not occur in cases +where a less precise loop can be defined, *unless* the ufunc +author does this intentionally using a promotion. + +This consideration means that upcasting has to be limited by some additional +method. + +*Alternative 1:* + +Assuming general upcasting is not intended, a rule must be defined to +limit upcasting the input from ``float16 -> float32`` either using generic +logic on the DTypes or the UFunc itself (or a combination of both). +The UFunc cannot do this easily on its own, since it cannot know all possible +DTypes which register loops. +Consider the two examples: + +First (should be rejected): + +* Input: ``float16 * float16`` +* Existing loop: ``float32 * float32`` + +Second (should be accepted): + +* Input: ``timedelta64 * int32`` +* Existing loop: ``timedelta64 * int16`` + + +This requires either: + +1. The ``timedelta64`` to somehow signal that the ``int64`` upcast is + always supported if it is involved in the operation. +2. The ``float32 * float32`` loop to reject upcasting. + +Implementing the first approach requires signaling that upcasts are +acceptable in the specific context. This would require additional hooks +and may not be simple for complex DTypes. + +For the second approach in most cases a simple ``np.common_dtype`` rule will +work for initial dispatching, however, even this is only clearly the case +for homogeneous loops. +This option will require adding a function to check whether the input +is a valid upcast to each loop individually, which seems problematic. +In many cases a default could be provided (homogeneous signature). + +*Alternative 2:* + +An alternative "promotion" step is to ensure that the *output* DType matches +with the loop after first finding the correct output DType. +If the output DTypes are known, finding a safe loop becomes easy. +In the majority of cases this works, the correct output dtype is just:: + + np.common_dtype(*input_DTypes) + +or some fixed DType (e.g. Bool for logical functions). + +However, it fails for example in the ``timedelta64 * int32`` case above since +there is a-priori no way to know that the "expected" result type of this +output is indeed ``timedelta64`` (``np.common_dtype(Datetime64, Int32)`` fails). +This requires some additional knowledge of the timedelta64 precision being +int64. Since a ufunc can have an arbitrary number of (relevant) inputs +it would thus at least require an additional ``__promoted_dtypes__`` method +on ``Datetime64`` (and all DTypes). + +A further limitation is shown by masked DTypes. Logical functions do not +have a boolean result when masked are involved, which would thus require the +original ufunc author to anticipate masked DTypes in this scheme. +Similarly, some functions defined for complex values will return real numbers +while others return complex numbers. If the original author did not anticipate +complex numbers, the promotion may be incorrect for a later added complex loop. + + +We believe that promoters, while allowing for an huge theoretical complexity, +are the best solution: + +1. Promotion allows for dynamically adding new loops. E.g. it is possible + to define an abstract Unit DType, which dynamically creates classes to + wrap other existing DTypes. Using a single promoter, this DType can + dynamically wrap existing ``ArrayMethod`` enabling it to find the correct + loop in a single lookup instead of two. +2. The promotion logic will usually err on the safe side: A newly-added + loop cannot be misused unless a promoter is added as well. +3. They put the burden of carefully thinking of whether the logic is correct + on the programmer adding new loops to a UFunc. (Compared to Alternative 2) +4. In case of incorrect existing promotion, writing a promoter to restrict + or refine a generic rule is possible. In general a promotion rule should + never return an *incorrect* promotion, but if it the existing promotion + logic fails or is incorrect for a newly-added loop, the loop can add a + new promoter to refine the logic. + +The option of having each loop verify that no upcast occured is probably +the best alternative, but does not include the ability to dynamically +adding new loops. + +The main downsides of general promoters is that they allow a possible +very large complexity. +A third-party library *could* add incorrect promotions to NumPy, however, +this is already possible by adding new incorrect loops. +In general we believe we can rely on downstream projects to use this +power and complexity carefully and responsibly. + + +*************** +User Guidelines +*************** + +In general adding a promoter to a UFunc must be done very carefully. +A promoter should never affect loops which can be reasonably defined +by other datatypes. Defining a hypothetical ``erf(UnitFloat16)`` loop +must not lead to ``erf(float16)``. +In general a promoter should fulfill the following requirements: + +* Be conservative when defining a new promotion rule. An incorrect result + is a much more dangerous error than an unexpected error. +* One of the (abstract) DTypes added should typically match specifically with a + DType (or family of DTypes) defined by your project. + Never add promotion rules which go beyond normal common DType rules! + It is *not* reasonable to add a loop for ``int16 + uint16 -> int24`` if + you write an ``int24`` dtype. The result of this operation was already + defined previously as ``int32`` and will be used with this assumption. +* A promoter (or loop) should never affect existing loop results. + This includes adding faster but less precise loops/promoters to replace + existing ones. +* Try to stay within a clear, linear hierarchy for all promotion (and casting) + related logic. NumPy itself breaks this logic for integers and floats + (they are not strictly linear, since int64 cannot promote to float32). +* Loops and promoters can be added by any project, which could be: + + * The project defining the ufunc + * The project defining the DType + * A third-party project + + Try to find out which is the best project to add the loop. If neither + the project defining the ufunc nor the project defining the DType add the + loop, issues with multiple definitions (which are rejected) may arise + and care should be taken that the loop behaviour is always more desirable + than an error. + +In some cases exceptions to these rules may make sense, however, in general +we ask you to use extreme caution and when in doubt create a new UFunc +instead. This clearly notifies the users of differing rules. +When in doubt, ask on the NumPy mailing list or issue tracker! + + +************** +Implementation +************** + +Implementation of this NEP will entail a large refactor and restructuring +of the current ufunc machinery (as well as casting). + +The implementation unfortunately will require large maintenance of the +UFunc machinery, since both the actual UFunc loop calls, as well as the +the initial dispatching steps have to be modified. + +In general, the correct ``ArrayMethod``, also those returned by a promoter, +will be cached (or stored) inside a hashtable for efficient lookup. + + +********** +Discussion +********** + +There is a large space of possible implementations with many discussions +in various places, as well as initial thoughts and design documents. +These are listed in the discussion of :ref:`NEP 40 ` and not repeated here for +brevity. + +A long discussion which touches many of these points and points towards +similar solutions can be found in +`the github issue 12518 "What should be the calling convention for ufunc inner loop signatures?" `_ + + +********** +References +********** + +Please see NEP 40 and 41 for more discussion and references. + + +********* +Copyright +********* + +This document has been placed in the public domain. diff --git a/doc/neps/nep-0045-c_style_guide.rst b/doc/neps/nep-0045-c_style_guide.rst index f579e4f69b81..5a2fcf946aa7 100644 --- a/doc/neps/nep-0045-c_style_guide.rst +++ b/doc/neps/nep-0045-c_style_guide.rst @@ -10,6 +10,8 @@ NEP 45 — C Style Guide :Created: 2012-02-26 :Resolution: https://github.com/numpy/numpy/issues/11911 +.. highlight:: c + Abstract -------- diff --git a/doc/release/upcoming_changes/16200.compatibility.rst b/doc/release/upcoming_changes/16200.compatibility.rst index d0fd51265f6e..2bbdd883ed48 100644 --- a/doc/release/upcoming_changes/16200.compatibility.rst +++ b/doc/release/upcoming_changes/16200.compatibility.rst @@ -8,14 +8,26 @@ error:: np.array([np.float64(np.nan)], dtype=np.int64) -will succeed at this time (this may change) and return an undefined result -(usually the smallest possible integer). This also affects assignments:: +will succeed and return an undefined result (usually the smallest possible +integer). This also affects assignments:: arr[0] = np.float64(np.nan) -Note, this already happened for ``np.array(np.float64(np.nan), dtype=np.int64)`` -and that the behaviour is unchanged for ``np.nan`` itself which is a Python -float. +At this time, NumPy retains the behaviour for:: + + np.array(np.float64(np.nan), dtype=np.int64) + +The above changes do not affect Python scalars: + + np.array([float("NaN")], dtype=np.int64) + +remains unaffected (``np.nan`` is a Python ``float``, not a NumPy one). +Unlike signed integers, unsigned integers do not retain this special case, +since they always behaved more like casting. +The following code stops raising an error:: + + np.array([np.float64(np.nan)], dtype=np.uint64) + To avoid backward compatibility issues, at this time assignment from ``datetime64`` scalar to strings of too short length remains supported. This means that ``np.asarray(np.datetime64("2020-10-10"), dtype="S5")`` diff --git a/doc/release/upcoming_changes/17219.new_feature.rst b/doc/release/upcoming_changes/17219.new_feature.rst new file mode 100644 index 000000000000..a6985ef0d2ad --- /dev/null +++ b/doc/release/upcoming_changes/17219.new_feature.rst @@ -0,0 +1,12 @@ +Negation of user-defined BLAS/LAPACK detection order +---------------------------------------------------- +`distutils` allows negation of libraries when determining BLAS/LAPACK +libraries. +This may be used to remove an item from the library resolution phase, i.e. +to disallow NetLIB libraries one could do:: + +.. code:: bash + + NPY_BLAS_ORDER='^blas' NPY_LAPACK_ORDER='^lapack' python setup.py build + +which will use any of the accelerated libraries instead. diff --git a/doc/release/upcoming_changes/17456.new_feature.rst b/doc/release/upcoming_changes/17456.new_feature.rst new file mode 100644 index 000000000000..7ab014e77df9 --- /dev/null +++ b/doc/release/upcoming_changes/17456.new_feature.rst @@ -0,0 +1,5 @@ +``dtype`` option for `cov` and `corrcoef` +---------------------------------------------------- +The ``dtype`` option is now available for `numpy.cov` and `numpy.corrcoef`. +It specifies which data-type the returned result should have. +By default the functions still return a `numpy.float64` result. diff --git a/doc/release/upcoming_changes/17535.new_function.rst b/doc/release/upcoming_changes/17535.new_function.rst new file mode 100644 index 000000000000..4c3c11de4e26 --- /dev/null +++ b/doc/release/upcoming_changes/17535.new_function.rst @@ -0,0 +1,15 @@ +`numpy.broadcast_shapes` is a new user-facing function +------------------------------------------------------ +`broadcast_shapes` gets the resulting shape from +broadcasting the given shape tuples against each other. + +.. code:: python + + >>> np.broadcast_shapes((1, 2), (3, 1)) + (3, 2) + + >>> np.broadcast_shapes(2, (3, 1)) + (3, 2) + + >>> np.broadcast_shapes((6, 7), (5, 6, 1), (7,), (5, 1, 7)) + (5, 6, 7) diff --git a/doc/release/upcoming_changes/17577.compatibility.rst b/doc/release/upcoming_changes/17577.compatibility.rst new file mode 100644 index 000000000000..d08805607323 --- /dev/null +++ b/doc/release/upcoming_changes/17577.compatibility.rst @@ -0,0 +1,6 @@ +poly1d respects the dtype of all-zero argument +---------------------------------------------- +Previously, constructing an instance of ``poly1d`` with all-zero +coefficients would cast the coefficients to ``np.float64``. +This affected the output dtype of methods which construct +``poly1d`` instances internally, such as ``np.polymul``. \ No newline at end of file diff --git a/doc/release/upcoming_changes/17580.compatibility.rst b/doc/release/upcoming_changes/17580.compatibility.rst new file mode 100644 index 000000000000..b8e1849afcbe --- /dev/null +++ b/doc/release/upcoming_changes/17580.compatibility.rst @@ -0,0 +1,4 @@ +The numpy.i file for swig is Python 3 only. +------------------------------------------- +Uses of Python 2.7 C-API functions have been updated to Python 3 only. Users +who need the old version should take it from an older version of NumPy. diff --git a/doc/release/upcoming_changes/17596.future.rst b/doc/release/upcoming_changes/17596.future.rst new file mode 100644 index 000000000000..6e697c8d1f76 --- /dev/null +++ b/doc/release/upcoming_changes/17596.future.rst @@ -0,0 +1,30 @@ +Arrays cannot be using subarray dtypes +-------------------------------------- +Array creation and casting using ``np.array(arr, dtype)`` +and ``arr.astype(dtype)`` will use different logic when ``dtype`` +is a subarray dtype such as ``np.dtype("(2)i,")``. + +For such a ``dtype`` the following behaviour is true:: + + res = np.array(arr, dtype) + + res.dtype is not dtype + res.dtype is dtype.base + res.shape == arr.shape + dtype.shape + +But ``res`` is filled using the logic: + + res = np.empty(arr.shape + dtype.shape, dtype=dtype.base) + res[...] = arr + +which uses incorrect broadcasting (and often leads to an error). +In the future, this will instead cast each element individually, +leading to the same result as:: + + res = np.array(arr, dtype=np.dtype(["f", dtype]))["f"] + +Which can normally be used to opt-in to the new behaviour. + +This change does not affect ``np.array(list, dtype="(2)i,")`` unless the +``list`` itself includes at least one array. In particular, the behaviour +is unchanged for a list of tuples. diff --git a/doc/source/_static/numpy.css b/doc/source/_static/numpy.css new file mode 100644 index 000000000000..22d08cc0dca6 --- /dev/null +++ b/doc/source/_static/numpy.css @@ -0,0 +1,40 @@ +@import url('https://fonts.googleapis.com/css2?family=Lato:ital,wght@0,400;0,700;0,900;1,400;1,700;1,900&family=Open+Sans:ital,wght@0,400;0,600;1,400;1,600&display=swap'); + +.navbar-brand img { + height: 75px; +} +.navbar-brand { + height: 75px; +} + +body { + font-family: 'Open Sans', sans-serif; + color:#4A4A4A; /* numpy.org body color */ +} + +pre, code { + font-size: 100%; + line-height: 155%; +} + +h1 { + font-style: "Lato", sans-serif; + color: #013243; /* warm black */ + font-weight: 700; + letter-spacing: -.04em; + text-align: right; + margin-top: 3rem; + margin-bottom: 4rem; + font-size: 3rem; +} + + +h2 { + color: #4d77cf; /* han blue */ + letter-spacing: -.03em; +} + +h3 { + color: #013243; /* warm black */ + letter-spacing: -.03em; +} diff --git a/doc/source/_templates/indexcontent.html b/doc/source/_templates/indexcontent.html index 5929e755dee6..6dd6bf9b0851 100644 --- a/doc/source/_templates/indexcontent.html +++ b/doc/source/_templates/indexcontent.html @@ -12,21 +12,22 @@

{{ docstitle|e }}

For users:

-

Setting Up
- Learn about what NumPy is and how to install it

-

Quickstart Tutorial
+

What is NumPy?
+ Who uses it and why

+

Installation
+

Quickstart
Aimed at domain experts or people migrating to NumPy

-

Absolute Beginners Tutorial
+

Absolute beginner's guide
Start here for an overview of NumPy features and syntax

Tutorials
Learn about concepts and submodules

-

How Tos
+

How-tos
How to do common tasks with NumPy

-

NumPy API Reference
+

NumPy API reference
Automatically generated reference documentation

Explanations
In depth explanation of concepts, best practices and techniques

-

F2Py Guide
+

F2Py guide
Documentation for the f2py module (Fortran extensions for Python)

Glossary
List of the most important terms

@@ -36,11 +37,11 @@

{{ docstitle|e }}

For developers/contributors:

-

NumPy Contributor Guide
+

NumPy contributor guide
Contributing to NumPy

Under-the-hood docs
Specialized, in-depth documentation

-

Building and Extending the Documentation
+

Building and extending the documentation
How to contribute to this documentation (user and API)

The numpydoc docstring guide
How to write docstrings in the numpydoc format

@@ -54,9 +55,9 @@

{{ docstitle|e }}

Reporting bugs

-

Release Notes

+

Release notes

 -

About NumPy

+

Document conventions

License of NumPy

@@ -65,13 +66,13 @@

Acknowledgements

Large parts of this manual originate from Travis E. Oliphant's book "Guide to NumPy" - (which generously entered Public Domain in August 2008). The reference + (which generously entered public domain in August 2008). The reference documentation for many of the functions are written by numerous contributors and developers of NumPy.

The preferred way to update the documentation is by submitting a pull - request on Github (see the Documentation Index). + request on GitHub (see the Documentation index). Please help us to further improve the NumPy documentation!

{% endblock %} diff --git a/doc/source/_templates/layout.html b/doc/source/_templates/layout.html index 0b0ba6271bdf..e2812fdd5ff3 100644 --- a/doc/source/_templates/layout.html +++ b/doc/source/_templates/layout.html @@ -1,16 +1,10 @@ {% extends "!layout.html" %} {%- block extrahead %} - +{{ super() }} + + + - -{{ super() }} {% endblock %} diff --git a/doc/source/about.rst b/doc/source/about.rst deleted file mode 100644 index 3e83833d178d..000000000000 --- a/doc/source/about.rst +++ /dev/null @@ -1,62 +0,0 @@ -About NumPy -=========== - -NumPy is the fundamental package -needed for scientific computing with Python. This package contains: - -- a powerful N-dimensional :ref:`array object ` -- sophisticated :ref:`(broadcasting) functions ` -- basic :ref:`linear algebra functions ` -- basic :ref:`Fourier transforms ` -- sophisticated :ref:`random number capabilities ` -- tools for integrating Fortran code -- tools for integrating C/C++ code - -Besides its obvious scientific uses, *NumPy* can also be used as an -efficient multi-dimensional container of generic data. Arbitrary -data types can be defined. This allows *NumPy* to seamlessly and -speedily integrate with a wide variety of databases. - -NumPy is a successor for two earlier scientific Python libraries: -Numeric and Numarray. - -NumPy community ---------------- - -NumPy is a distributed, volunteer, open-source project. *You* can help -us make it better; if you believe something should be improved either -in functionality or in documentation, don't hesitate to contact us --- or -even better, contact us and participate in fixing the problem. - -Our main means of communication are: - -- `scipy.org website `__ - -- `Mailing lists `__ - -- `NumPy Issues `__ (bug reports go here) - -- `Old NumPy Trac `__ (dead link) - -More information about the development of NumPy can be found at our `Developer Zone `__. - -The project management structure can be found at our :doc:`governance page ` - - -About this documentation -======================== - -Conventions ------------ - -Names of classes, objects, constants, etc. are given in **boldface** font. -Often they are also links to a more detailed documentation of the -referred object. - -This manual contains many examples of use, usually prefixed with the -Python prompt ``>>>`` (which is not a part of the example code). The -examples assume that you have first entered:: - ->>> import numpy as np - -before running the examples. diff --git a/doc/source/conf.py b/doc/source/conf.py index e34be7f5c12a..381a01612401 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -6,6 +6,62 @@ # Minimum version, enforced by sphinx needs_sphinx = '2.2.0' + +# This is a nasty hack to use platform-agnostic names for types in the +# documentation. + +# must be kept alive to hold the patched names +_name_cache = {} + +def replace_scalar_type_names(): + """ Rename numpy types to use the canonical names to make sphinx behave """ + import ctypes + + Py_ssize_t = ctypes.c_int64 if ctypes.sizeof(ctypes.c_void_p) == 8 else ctypes.c_int32 + + class PyObject(ctypes.Structure): + pass + + class PyTypeObject(ctypes.Structure): + pass + + PyObject._fields_ = [ + ('ob_refcnt', Py_ssize_t), + ('ob_type', ctypes.POINTER(PyTypeObject)), + ] + + + PyTypeObject._fields_ = [ + # varhead + ('ob_base', PyObject), + ('ob_size', Py_ssize_t), + # declaration + ('tp_name', ctypes.c_char_p), + ] + + # prevent numpy attaching docstrings to the scalar types + assert 'numpy.core._add_newdocs_scalars' not in sys.modules + sys.modules['numpy.core._add_newdocs_scalars'] = object() + + import numpy + + # change the __name__ of the scalar types + for name in [ + 'byte', 'short', 'intc', 'int_', 'longlong', + 'ubyte', 'ushort', 'uintc', 'uint', 'ulonglong', + 'half', 'single', 'double', 'longdouble', + 'half', 'csingle', 'cdouble', 'clongdouble', + ]: + typ = getattr(numpy, name) + c_typ = PyTypeObject.from_address(id(typ)) + c_typ.tp_name = _name_cache[typ] = b"numpy." + name.encode('utf8') + + # now generate the docstrings as usual + del sys.modules['numpy.core._add_newdocs_scalars'] + import numpy.core._add_newdocs_scalars + +replace_scalar_type_names() + # ----------------------------------------------------------------------------- # General configuration # ----------------------------------------------------------------------------- @@ -227,6 +283,8 @@ def setup(app): 'matplotlib': ('https://matplotlib.org', None), 'imageio': ('https://imageio.readthedocs.io/en/stable', None), 'skimage': ('https://scikit-image.org/docs/stable', None), + 'pandas': ('https://pandas.pydata.org/pandas-docs/stable', None), + 'scipy-lecture-notes': ('https://scipy-lectures.org', None), } @@ -310,6 +368,17 @@ def setup(app): else: print("NOTE: linkcode extension not found -- no links to source generated") + +def _get_c_source_file(obj): + if issubclass(obj, numpy.generic): + return r"core/src/multiarray/scalartypes.c.src" + elif obj is numpy.ndarray: + return r"core/src/multiarray/arrayobject.c" + else: + # todo: come up with a better way to generate these + return None + + def linkcode_resolve(domain, info): """ Determine the URL corresponding to Python object @@ -340,25 +409,33 @@ def linkcode_resolve(domain, info): else: obj = unwrap(obj) - try: - fn = inspect.getsourcefile(obj) - except Exception: - fn = None - if not fn: - return None + fn = None + lineno = None - try: - source, lineno = inspect.getsourcelines(obj) - except Exception: - lineno = None + # Make a poor effort at linking C extension types + if isinstance(obj, type) and obj.__module__ == 'numpy': + fn = _get_c_source_file(obj) + + if fn is None: + try: + fn = inspect.getsourcefile(obj) + except Exception: + fn = None + if not fn: + return None + + try: + source, lineno = inspect.getsourcelines(obj) + except Exception: + lineno = None + + fn = relpath(fn, start=dirname(numpy.__file__)) if lineno: linespec = "#L%d-L%d" % (lineno, lineno + len(source) - 1) else: linespec = "" - fn = relpath(fn, start=dirname(numpy.__file__)) - if 'dev' in numpy.__version__: return "https://github.com/numpy/numpy/blob/master/numpy/%s%s" % ( fn, linespec) @@ -367,15 +444,15 @@ def linkcode_resolve(domain, info): numpy.__version__, fn, linespec) from pygments.lexers import CLexer -import copy +from pygments.lexer import inherit, bygroups +from pygments.token import Comment class NumPyLexer(CLexer): name = 'NUMPYLEXER' - tokens = copy.deepcopy(CLexer.tokens) - # Extend the regex for valid identifiers with @ - for k, val in tokens.items(): - for i, v in enumerate(val): - if isinstance(v, tuple): - if isinstance(v[0], str): - val[i] = (v[0].replace('a-zA-Z', 'a-zA-Z@'),) + v[1:] + tokens = { + 'statements': [ + (r'@[a-zA-Z_]*@', Comment.Preproc, 'macro'), + inherit, + ], + } diff --git a/doc/source/dev/conduct/code_of_conduct.rst b/doc/source/dev/conduct/code_of_conduct.rst deleted file mode 100644 index f2f0a536dc56..000000000000 --- a/doc/source/dev/conduct/code_of_conduct.rst +++ /dev/null @@ -1,163 +0,0 @@ -NumPy Code of Conduct -===================== - - -Introduction ------------- - -This code of conduct applies to all spaces managed by the NumPy project, -including all public and private mailing lists, issue trackers, wikis, blogs, -Twitter, and any other communication channel used by our community. The NumPy -project does not organise in-person events, however events related to our -community should have a code of conduct similar in spirit to this one. - -This code of conduct should be honored by everyone who participates in -the NumPy community formally or informally, or claims any affiliation with the -project, in any project-related activities and especially when representing the -project, in any role. - -This code is not exhaustive or complete. It serves to distill our common -understanding of a collaborative, shared environment and goals. Please try to -follow this code in spirit as much as in letter, to create a friendly and -productive environment that enriches the surrounding community. - - -Specific Guidelines -------------------- - -We strive to: - -1. Be open. We invite anyone to participate in our community. We prefer to use - public methods of communication for project-related messages, unless - discussing something sensitive. This applies to messages for help or - project-related support, too; not only is a public support request much more - likely to result in an answer to a question, it also ensures that any - inadvertent mistakes in answering are more easily detected and corrected. - -2. Be empathetic, welcoming, friendly, and patient. We work together to resolve - conflict, and assume good intentions. We may all experience some frustration - from time to time, but we do not allow frustration to turn into a personal - attack. A community where people feel uncomfortable or threatened is not a - productive one. - -3. Be collaborative. Our work will be used by other people, and in turn we will - depend on the work of others. When we make something for the benefit of the - project, we are willing to explain to others how it works, so that they can - build on the work to make it even better. Any decision we make will affect - users and colleagues, and we take those consequences seriously when making - decisions. - -4. Be inquisitive. Nobody knows everything! Asking questions early avoids many - problems later, so we encourage questions, although we may direct them to - the appropriate forum. We will try hard to be responsive and helpful. - -5. Be careful in the words that we choose. We are careful and respectful in - our communication and we take responsibility for our own speech. Be kind to - others. Do not insult or put down other participants. We will not accept - harassment or other exclusionary behaviour, such as: - - - Violent threats or language directed against another person. - - Sexist, racist, or otherwise discriminatory jokes and language. - - Posting sexually explicit or violent material. - - Posting (or threatening to post) other people's personally identifying information ("doxing"). - - Sharing private content, such as emails sent privately or non-publicly, - or unlogged forums such as IRC channel history, without the sender's consent. - - Personal insults, especially those using racist or sexist terms. - - Unwelcome sexual attention. - - Excessive profanity. Please avoid swearwords; people differ greatly in their sensitivity to swearing. - - Repeated harassment of others. In general, if someone asks you to stop, then stop. - - Advocating for, or encouraging, any of the above behaviour. - - -Diversity Statement -------------------- - -The NumPy project welcomes and encourages participation by everyone. We are -committed to being a community that everyone enjoys being part of. Although -we may not always be able to accommodate each individual's preferences, we try -our best to treat everyone kindly. - -No matter how you identify yourself or how others perceive you: we welcome you. -Though no list can hope to be comprehensive, we explicitly honour diversity in: -age, culture, ethnicity, genotype, gender identity or expression, language, -national origin, neurotype, phenotype, political beliefs, profession, race, -religion, sexual orientation, socioeconomic status, subculture and technical -ability, to the extent that these do not conflict with this code of conduct. - - -Though we welcome people fluent in all languages, NumPy development is -conducted in English. - -Standards for behaviour in the NumPy community are detailed in the Code of -Conduct above. Participants in our community should uphold these standards -in all their interactions and help others to do so as well (see next section). - - -Reporting Guidelines --------------------- - -We know that it is painfully common for internet communication to start at or -devolve into obvious and flagrant abuse. We also recognize that sometimes -people may have a bad day, or be unaware of some of the guidelines in this Code -of Conduct. Please keep this in mind when deciding on how to respond to a -breach of this Code. - -For clearly intentional breaches, report those to the Code of Conduct committee -(see below). For possibly unintentional breaches, you may reply to the person -and point out this code of conduct (either in public or in private, whatever is -most appropriate). If you would prefer not to do that, please feel free to -report to the Code of Conduct Committee directly, or ask the Committee for -advice, in confidence. - -You can report issues to the NumPy Code of Conduct committee, at -numpy-conduct@googlegroups.com. Currently, the committee consists of: - -- Stefan van der Walt -- Melissa Weber Mendonça -- Anirudh Subramanian - -If your report involves any members of the committee, or if they feel they have -a conflict of interest in handling it, then they will recuse themselves from -considering your report. Alternatively, if for any reason you feel -uncomfortable making a report to the committee, then you can also contact: - -- Senior `NumFOCUS staff `__: conduct@numfocus.org - - -Incident reporting resolution & Code of Conduct enforcement ------------------------------------------------------------ - -*This section summarizes the most important points, more details can be found -in* :ref:`CoC_reporting_manual`. - -We will investigate and respond to all complaints. The NumPy Code of Conduct -Committee and the NumPy Steering Committee (if involved) will protect the -identity of the reporter, and treat the content of complaints as confidential -(unless the reporter agrees otherwise). - -In case of severe and obvious breaches, e.g. personal threat or violent, sexist -or racist language, we will immediately disconnect the originator from NumPy -communication channels; please see the manual for details. - -In cases not involving clear severe and obvious breaches of this code of -conduct, the process for acting on any received code of conduct violation -report will be: - -1. acknowledge report is received -2. reasonable discussion/feedback -3. mediation (if feedback didn't help, and only if both reporter and reportee agree to this) -4. enforcement via transparent decision (see :ref:`CoC_resolutions`) by the - Code of Conduct Committee - -The committee will respond to any report as soon as possible, and at most -within 72 hours. - - -Endnotes --------- - -We are thankful to the groups behind the following documents, from which we -drew content and inspiration: - -- `The SciPy Code of Conduct `_ - diff --git a/doc/source/dev/conduct/report_handling_manual.rst b/doc/source/dev/conduct/report_handling_manual.rst deleted file mode 100644 index d39b615bb8c1..000000000000 --- a/doc/source/dev/conduct/report_handling_manual.rst +++ /dev/null @@ -1,220 +0,0 @@ -:orphan: - -.. _CoC_reporting_manual: - -NumPy Code of Conduct - How to follow up on a report ----------------------------------------------------- - -This is the manual followed by NumPy's Code of Conduct Committee. It's used -when we respond to an issue to make sure we're consistent and fair. - -Enforcing the Code of Conduct impacts our community today and for the future. -It's an action that we do not take lightly. When reviewing enforcement -measures, the Code of Conduct Committee will keep the following values and -guidelines in mind: - -* Act in a personal manner rather than impersonal. The Committee can engage - the parties to understand the situation, while respecting the privacy and any - necessary confidentiality of reporters. However, sometimes it is necessary - to communicate with one or more individuals directly: the Committee's goal is - to improve the health of our community rather than only produce a formal - decision. - -* Emphasize empathy for individuals rather than judging behavior, avoiding - binary labels of "good" and "bad/evil". Overt, clear-cut aggression and - harassment exists and we will be address that firmly. But many scenarios - that can prove challenging to resolve are those where normal disagreements - devolve into unhelpful or harmful behavior from multiple parties. - Understanding the full context and finding a path that re-engages all is - hard, but ultimately the most productive for our community. - -* We understand that email is a difficult medium and can be isolating. - Receiving criticism over email, without personal contact, can be - particularly painful. This makes it especially important to keep an - atmosphere of open-minded respect of the views of others. It also means - that we must be transparent in our actions, and that we will do everything - in our power to make sure that all our members are treated fairly and with - sympathy. - -* Discrimination can be subtle and it can be unconscious. It can show itself - as unfairness and hostility in otherwise ordinary interactions. We know - that this does occur, and we will take care to look out for it. We would - very much like to hear from you if you feel you have been treated unfairly, - and we will use these procedures to make sure that your complaint is heard - and addressed. - -* Help increase engagement in good discussion practice: try to identify where - discussion may have broken down and provide actionable information, pointers - and resources that can lead to positive change on these points. - -* Be mindful of the needs of new members: provide them with explicit support - and consideration, with the aim of increasing participation from - underrepresented groups in particular. - -* Individuals come from different cultural backgrounds and native languages. - Try to identify any honest misunderstandings caused by a non-native speaker - and help them understand the issue and what they can change to avoid causing - offence. Complex discussion in a foreign language can be very intimidating, - and we want to grow our diversity also across nationalities and cultures. - -*Mediation*: voluntary, informal mediation is a tool at our disposal. In -contexts such as when two or more parties have all escalated to the point of -inappropriate behavior (something sadly common in human conflict), it may be -useful to facilitate a mediation process. This is only an example: the -Committee can consider mediation in any case, mindful that the process is meant -to be strictly voluntary and no party can be pressured to participate. If the -Committee suggests mediation, it should: - -* Find a candidate who can serve as a mediator. -* Obtain the agreement of the reporter(s). The reporter(s) have complete - freedom to decline the mediation idea, or to propose an alternate mediator. -* Obtain the agreement of the reported person(s). -* Settle on the mediator: while parties can propose a different mediator than - the suggested candidate, only if common agreement is reached on all terms can - the process move forward. -* Establish a timeline for mediation to complete, ideally within two weeks. - -The mediator will engage with all the parties and seek a resolution that is -satisfactory to all. Upon completion, the mediator will provide a report -(vetted by all parties to the process) to the Committee, with recommendations -on further steps. The Committee will then evaluate these results (whether -satisfactory resolution was achieved or not) and decide on any additional -action deemed necessary. - - -How the committee will respond to reports -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -When the committee (or a committee member) receives a report, they will first -determine whether the report is about a clear and severe breach (as defined -below). If so, immediate action needs to be taken in addition to the regular -report handling process. - -Clear and severe breach actions -+++++++++++++++++++++++++++++++ - -We know that it is painfully common for internet communication to start at or -devolve into obvious and flagrant abuse. We will deal quickly with clear and -severe breaches like personal threats, violent, sexist or racist language. - -When a member of the Code of Conduct committee becomes aware of a clear and -severe breach, they will do the following: - -* Immediately disconnect the originator from all NumPy communication channels. -* Reply to the reporter that their report has been received and that the - originator has been disconnected. -* In every case, the moderator should make a reasonable effort to contact the - originator, and tell them specifically how their language or actions - qualify as a "clear and severe breach". The moderator should also say - that, if the originator believes this is unfair or they want to be - reconnected to NumPy, they have the right to ask for a review, as below, by - the Code of Conduct Committee. - The moderator should copy this explanation to the Code of Conduct Committee. -* The Code of Conduct Committee will formally review and sign off on all cases - where this mechanism has been applied to make sure it is not being used to - control ordinary heated disagreement. - -Report handling -+++++++++++++++ - -When a report is sent to the committee they will immediately reply to the -reporter to confirm receipt. This reply must be sent within 72 hours, and the -group should strive to respond much quicker than that. - -If a report doesn't contain enough information, the committee will obtain all -relevant data before acting. The committee is empowered to act on the Steering -Council’s behalf in contacting any individuals involved to get a more complete -account of events. - -The committee will then review the incident and determine, to the best of their -ability: - -* What happened. -* Whether this event constitutes a Code of Conduct violation. -* Who are the responsible party(ies). -* Whether this is an ongoing situation, and there is a threat to anyone's - physical safety. - -This information will be collected in writing, and whenever possible the -group's deliberations will be recorded and retained (i.e. chat transcripts, -email discussions, recorded conference calls, summaries of voice conversations, -etc). - -It is important to retain an archive of all activities of this committee to -ensure consistency in behavior and provide institutional memory for the -project. To assist in this, the default channel of discussion for this -committee will be a private mailing list accessible to current and future -members of the committee as well as members of the Steering Council upon -justified request. If the Committee finds the need to use off-list -communications (e.g. phone calls for early/rapid response), it should in all -cases summarize these back to the list so there's a good record of the process. - -The Code of Conduct Committee should aim to have a resolution agreed upon within -two weeks. In the event that a resolution can't be determined in that time, the -committee will respond to the reporter(s) with an update and projected timeline -for resolution. - - -.. _CoC_resolutions: - -Resolutions -~~~~~~~~~~~ - -The committee must agree on a resolution by consensus. If the group cannot reach -consensus and deadlocks for over a week, the group will turn the matter over to -the Steering Council for resolution. - - -Possible responses may include: - -* Taking no further action - - - if we determine no violations have occurred. - - if the matter has been resolved publicly while the committee was considering responses. - -* Coordinating voluntary mediation: if all involved parties agree, the - Committee may facilitate a mediation process as detailed above. -* Remind publicly, and point out that some behavior/actions/language have been - judged inappropriate and why in the current context, or can but hurtful to - some people, requesting the community to self-adjust. -* A private reprimand from the committee to the individual(s) involved. In this - case, the group chair will deliver that reprimand to the individual(s) over - email, cc'ing the group. -* A public reprimand. In this case, the committee chair will deliver that - reprimand in the same venue that the violation occurred, within the limits of - practicality. E.g., the original mailing list for an email violation, but - for a chat room discussion where the person/context may be gone, they can be - reached by other means. The group may choose to publish this message - elsewhere for documentation purposes. -* A request for a public or private apology, assuming the reporter agrees to - this idea: they may at their discretion refuse further contact with the - violator. The chair will deliver this request. The committee may, if it - chooses, attach "strings" to this request: for example, the group may ask a - violator to apologize in order to retain one’s membership on a mailing list. -* A "mutually agreed upon hiatus" where the committee asks the individual to - temporarily refrain from community participation. If the individual chooses - not to take a temporary break voluntarily, the committee may issue a - "mandatory cooling off period". -* A permanent or temporary ban from some or all NumPy spaces (mailing lists, - gitter.im, etc.). The group will maintain records of all such bans so that - they may be reviewed in the future or otherwise maintained. - -Once a resolution is agreed upon, but before it is enacted, the committee will -contact the original reporter and any other affected parties and explain the -proposed resolution. The committee will ask if this resolution is acceptable, -and must note feedback for the record. - -Finally, the committee will make a report to the NumPy Steering Council (as -well as the NumPy core team in the event of an ongoing resolution, such as a -ban). - -The committee will never publicly discuss the issue; all public statements will -be made by the chair of the Code of Conduct Committee or the NumPy Steering -Council. - - -Conflicts of Interest -~~~~~~~~~~~~~~~~~~~~~ - -In the event of any conflict of interest, a committee member must immediately -notify the other members, and recuse themselves if necessary. diff --git a/doc/source/dev/development_environment.rst b/doc/source/dev/development_environment.rst index ff78cecc5315..cb027c6620f8 100644 --- a/doc/source/dev/development_environment.rst +++ b/doc/source/dev/development_environment.rst @@ -273,7 +273,7 @@ pull requests aren't perfect, the community is always happy to help. As a volunteer project, things do sometimes get dropped and it's totally fine to ping us if something has sat without a response for about two to four weeks. -So go ahead and pick something that annoys or confuses you about numpy, +So go ahead and pick something that annoys or confuses you about NumPy, experiment with the code, hang around for discussions or go through the reference documents to try to fix it. Things will fall in place and soon you'll have a pretty good understanding of the project as a whole. Good Luck! diff --git a/doc/source/dev/development_workflow.rst b/doc/source/dev/development_workflow.rst index d5a49a9f9512..34535b2f54b6 100644 --- a/doc/source/dev/development_workflow.rst +++ b/doc/source/dev/development_workflow.rst @@ -188,6 +188,16 @@ Standard acronyms to start the commit message with are:: REL: related to releasing numpy +.. _workflow_mailing_list: + +Get the mailing list's opinion +======================================================= + +If you plan a new feature or API change, it's wisest to first email the +NumPy `mailing list `_ +asking for comment. If you haven't heard back in a week, it's +OK to ping the list again. + .. _asking-for-merging: Asking for your changes to be merged with the main repo @@ -197,15 +207,24 @@ When you feel your work is finished, you can create a pull request (PR). Github has a nice help page that outlines the process for `filing pull requests`_. If your changes involve modifications to the API or addition/modification of a -function, you should +function, add a release note to the ``doc/release/upcoming_changes/`` +directory, following the instructions and format in the +``doc/release/upcoming_changes/README.rst`` file. + + +.. _workflow_PR_timeline: + +Getting your PR reviewed +======================== + +We review pull requests as soon as we can, typically within a week. If you get +no review comments within two weeks, feel free to ask for feedback by +adding a comment on your PR (this will notify maintainers). + +If your PR is large or complicated, asking for input on the numpy-discussion +mailing list may also be useful. + -- send an email to the `NumPy mailing list`_ with a link to your PR along with - a description of and a motivation for your changes. This may generate - changes and feedback. It might be prudent to start with this step if your - change may be controversial. -- add a release note to the ``doc/release/upcoming_changes/`` directory, - following the instructions and format in the - ``doc/release/upcoming_changes/README.rst`` file. .. _rebasing-on-master: @@ -290,7 +309,7 @@ Rewriting commit history Do this only for your own feature branches. -There's an embarrassing typo in a commit you made? Or perhaps the you +There's an embarrassing typo in a commit you made? Or perhaps you made several false starts you would like the posterity not to see. This can be done via *interactive rebasing*. diff --git a/doc/source/dev/index.rst b/doc/source/dev/index.rst index c4f35b68ff6b..4641a7e2fff6 100644 --- a/doc/source/dev/index.rst +++ b/doc/source/dev/index.rst @@ -9,12 +9,11 @@ Contributing to NumPy .. toctree:: :hidden: - conduct/code_of_conduct Git Basics development_environment development_workflow ../benchmarking - style_guide + NumPy C style guide releasing governance/index howto-docs @@ -123,7 +122,8 @@ Here's the short summary, complete TOC links are below: overall code quality benefits. Therefore, please don't let the review discourage you from contributing: its only aim is to improve the quality of project, not to criticize (we are, after all, very grateful for the - time you're donating!). + time you're donating!). See our :ref:`Reviewer Guidelines + ` for more information. * To update your PR, make your changes on your local repository, commit, **run tests, and only if they succeed** push to your fork. As soon as @@ -180,6 +180,8 @@ be merged automatically, you have to incorporate changes that have been made since you started into your branch. Our recommended way to do this is to :ref:`rebase on master`. +.. _guidelines: + Guidelines ---------- @@ -187,9 +189,11 @@ Guidelines * All code should be `documented `_. * No changes are ever committed without review and approval by a core - team member.Please ask politely on the PR or on the `mailing list`_ if you + team member. Please ask politely on the PR or on the `mailing list`_ if you get no response to your pull request within a week. +.. _stylistic-guidelines: + Stylistic Guidelines -------------------- @@ -234,6 +238,8 @@ This will create a report in ``build/coverage``, which can be viewed with:: $ firefox build/coverage/index.html +.. _building-docs: + Building docs ------------- @@ -293,12 +299,12 @@ The rest of the story .. toctree:: :maxdepth: 2 - conduct/code_of_conduct Git Basics development_environment development_workflow + reviewer_guidelines ../benchmarking - style_guide + NumPy C style guide releasing governance/index howto-docs diff --git a/doc/source/dev/reviewer_guidelines.rst b/doc/source/dev/reviewer_guidelines.rst new file mode 100644 index 000000000000..0b225b9b6b14 --- /dev/null +++ b/doc/source/dev/reviewer_guidelines.rst @@ -0,0 +1,119 @@ +.. _reviewer-guidelines: + +=================== +Reviewer Guidelines +=================== + +Reviewing open pull requests (PRs) helps move the project forward. We encourage +people outside the project to get involved as well; it's a great way to get +familiar with the codebase. + +Who can be a reviewer? +====================== + +Reviews can come from outside the NumPy team -- we welcome contributions from +domain experts (for instance, `linalg` or `fft`) or maintainers of other +projects. You do not need to be a NumPy maintainer (a NumPy team member with +permission to merge a PR) to review. + +If we do not know you yet, consider introducing yourself in `the mailing list or +Slack `_ before you start reviewing pull requests. + +Communication Guidelines +======================== + +- Every PR, good or bad, is an act of generosity. Opening with a positive + comment will help the author feel rewarded, and your subsequent remarks may be + heard more clearly. You may feel good also. +- Begin if possible with the large issues, so the author knows they've been + understood. Resist the temptation to immediately go line by line, or to open + with small pervasive issues. +- You are the face of the project, and NumPy some time ago decided `the kind of + project it will be `_: open, empathetic, + welcoming, friendly and patient. Be `kind + `_ to contributors. +- Do not let perfect be the enemy of the good, particularly for documentation. + If you find yourself making many small suggestions, or being too nitpicky on + style or grammar, consider merging the current PR when all important concerns + are addressed. Then, either push a commit directly (if you are a maintainer) + or open a follow-up PR yourself. +- If you need help writing replies in reviews, check out some `Standard replies + for reviewing + `_. + +Reviewer Checklist +================== + +- Is the intended behavior clear under all conditions? Some things to watch: + - What happens with unexpected inputs like empty arrays or nan/inf values? + - Are axis or shape arguments tested to be `int` or `tuples`? + - Are unusual `dtypes` tested if a function supports those? +- Should variable names be improved for clarity or consistency? +- Should comments be added, or rather removed as unhelpful or extraneous? +- Does the documentation follow the :ref:`NumPy guidelines`? Are + the docstrings properly formatted? +- Does the code follow NumPy's :ref:`Stylistic Guidelines`? +- If you are a maintainer, and it is not obvious from the PR description, add a + short explanation of what a branch did to the merge message and, if closing an + issue, also add "Closes gh-123" where 123 is the issue number. +- For code changes, at least one maintainer (i.e. someone with commit rights) + should review and approve a pull request. If you are the first to review a + PR and approve of the changes use the GitHub `approve review + `_ tool + to mark it as such. If a PR is straightforward, for example it's a clearly + correct bug fix, it can be merged straight away. If it's more complex or + changes public API, please leave it open for at least a couple of days so + other maintainers get a chance to review. +- If you are a subsequent reviewer on an already approved PR, please use the + same review method as for a new PR (focus on the larger issues, resist the + temptation to add only a few nitpicks). If you have commit rights and think + no more review is needed, merge the PR. + +For maintainers +--------------- + +- Make sure all automated CI tests pass before merging a PR, and that the + :ref:`documentation builds ` without any errors. +- In case of merge conflicts, ask the PR submitter to :ref:`rebase on master + `. +- For PRs that add new features or are in some way complex, wait at least a day + or two before merging it. That way, others get a chance to comment before the + code goes in. Consider adding it to the release notes. +- When merging contributions, a committer is responsible for ensuring that those + meet the requirements outlined in the :ref:`Development process guidelines + ` for NumPy. Also, check that new features and backwards + compatibility breaks were discussed on the `numpy-discussion mailing list + `_. +- Squashing commits or cleaning up commit messages of a PR that you consider too + messy is OK. Remember to retain the original author's name when doing this. + Make sure commit messages follow the :ref:`rules for NumPy + `. +- When you want to reject a PR: if it's very obvious, you can just close it and + explain why. If it's not, then it's a good idea to first explain why you + think the PR is not suitable for inclusion in NumPy and then let a second + committer comment or close. + +GitHub Workflow +--------------- + +When reviewing pull requests, please use workflow tracking features on GitHub as +appropriate: + +- After you have finished reviewing, if you want to ask for the submitter to + make changes, change your review status to "Changes requested." This can be + done on GitHub, PR page, Files changed tab, Review changes (button on the top + right). +- If you're happy about the current status, mark the pull request as Approved + (same way as Changes requested). Alternatively (for maintainers): merge + the pull request, if you think it is ready to be merged. + +It may be helpful to have a copy of the pull request code checked out on your +own machine so that you can play with it locally. You can use the `GitHub CLI +`_ to +do this by clicking the ``Open with`` button in the upper right-hand corner of +the PR page. + +Assuming you have your :ref:`development environment` +set up, you can now build the code and test it. + +.. include:: gitwash/git_links.inc diff --git a/doc/source/dev/style_guide.rst b/doc/source/dev/style_guide.rst deleted file mode 100644 index bede3424d0af..000000000000 --- a/doc/source/dev/style_guide.rst +++ /dev/null @@ -1,8 +0,0 @@ -.. _style_guide: - -=================== -NumPy C Style Guide -=================== - -.. include:: ../../C_STYLE_GUIDE.rst.txt - :start-line: 4 diff --git a/doc/source/doc_conventions.rst b/doc/source/doc_conventions.rst new file mode 100644 index 000000000000..e2bc419d1691 --- /dev/null +++ b/doc/source/doc_conventions.rst @@ -0,0 +1,23 @@ +.. _documentation_conventions: + +############################################################################## +Documentation conventions +############################################################################## + +- Names that look like :func:`numpy.array` are links to detailed + documentation. + +- Examples often include the Python prompt ``>>>``. This is not part of the + code and will cause an error if typed or pasted into the Python + shell. It can be safely typed or pasted into the IPython shell; the ``>>>`` + is ignored. + +- Examples often use ``np`` as an alias for ``numpy``; that is, they assume + you've run:: + + >>> import numpy as np + +- If you're a code contributor writing a docstring, see :ref:`docstring_intro`. + +- If you're a writer contributing ordinary (non-docstring) documentation, see + :ref:`userdoc_guide`. diff --git a/doc/source/docs/howto_document.rst b/doc/source/docs/howto_document.rst index cf86b7e99870..ff726c67c215 100644 --- a/doc/source/docs/howto_document.rst +++ b/doc/source/docs/howto_document.rst @@ -1,12 +1,41 @@ .. _howto-document: -A Guide to NumPy/SciPy Documentation -==================================== +A Guide to NumPy Documentation +============================== + +.. _userdoc_guide: User documentation -******************* -NumPy text documents should follow the `Google developer documentation style guide `_. +****************** +- In general, we follow the + `Google developer documentation style guide `_. + +- NumPy style governs cases where: + + - Google has no guidance, or + - We prefer not to use the Google style + + Our current rules: + + - We pluralize *index* as *indices* rather than + `indexes `_, + following the precedent of :func:`numpy.indices`. + + - For consistency we also pluralize *matrix* as *matrices*. + +- Grammatical issues inadequately addressed by the NumPy or Google rules are + decided by the section on "Grammar and Usage" in the most recent edition of + the `Chicago Manual of Style + `_. + +- We welcome being + `alerted `_ to cases + we should add to the NumPy style rules. + + + +.. _docstring_intro: Docstrings ********** @@ -40,29 +69,7 @@ after which you may use it:: np.fft.fft2(...) -.. rubric:: - **For convenience the** `formatting standard`_ **is included below with an - example** - -.. include:: ../../sphinxext/doc/format.rst - -.. _example: - -Example Source -============== - -.. literalinclude:: ../../sphinxext/doc/example.py - - - -Example Rendered -================ - -.. ifconfig:: python_version_major < '3' - - The example is rendered only when sphinx is run with python3 and above - -.. automodule:: doc.example - :members: +Please use the numpydoc `formatting standard`_ as shown in their example_ .. _`formatting standard`: https://numpydoc.readthedocs.io/en/latest/format.html +.. _example: https://numpydoc.readthedocs.io/en/latest/example.html diff --git a/doc/source/f2py/allocarr_session.dat b/doc/source/f2py/allocarr_session.dat index 754d9cb8b5a2..ba168c22aa12 100644 --- a/doc/source/f2py/allocarr_session.dat +++ b/doc/source/f2py/allocarr_session.dat @@ -1,8 +1,11 @@ >>> import allocarr >>> print(allocarr.mod.__doc__) -b - 'f'-array(-1,-1), not allocated -foo - Function signature: - foo() +b : 'f'-array(-1,-1), not allocated +foo() + +Wrapper for ``foo``. + + >>> allocarr.mod.foo() b is not allocated diff --git a/doc/source/f2py/common_session.dat b/doc/source/f2py/common_session.dat index 0a38bec27b8a..2595bfbd5b20 100644 --- a/doc/source/f2py/common_session.dat +++ b/doc/source/f2py/common_session.dat @@ -1,8 +1,8 @@ >>> import common >>> print(common.data.__doc__) -i - 'i'-scalar -x - 'i'-array(4) -a - 'f'-array(2,3) +i : 'i'-scalar +x : 'i'-array(4) +a : 'f'-array(2,3) >>> common.data.i = 5 >>> common.data.x[1] = 2 diff --git a/doc/source/f2py/distutils.rst b/doc/source/f2py/distutils.rst index 71f6eab5a96f..4cf30045ec1d 100644 --- a/doc/source/f2py/distutils.rst +++ b/doc/source/f2py/distutils.rst @@ -2,6 +2,8 @@ Using via `numpy.distutils` ============================= +.. currentmodule:: numpy.distutils.core + :mod:`numpy.distutils` is part of NumPy extending standard Python ``distutils`` to deal with Fortran sources and F2PY signature files, e.g. compile Fortran sources, call F2PY to construct extension modules, etc. diff --git a/doc/source/f2py/moddata_session.dat b/doc/source/f2py/moddata_session.dat index e3c7580416f2..824bd86fc464 100644 --- a/doc/source/f2py/moddata_session.dat +++ b/doc/source/f2py/moddata_session.dat @@ -1,10 +1,14 @@ >>> import moddata >>> print(moddata.mod.__doc__) -i - 'i'-scalar -x - 'i'-array(4) -a - 'f'-array(2,3) -foo - Function signature: - foo() +i : 'i'-scalar +x : 'i'-array(4) +a : 'f'-array(2,3) +b : 'f'-array(-1,-1), not allocated +foo() + +Wrapper for ``foo``. + + >>> moddata.mod.i = 5 >>> moddata.mod.x[:2] = [1,2] diff --git a/doc/source/glossary.rst b/doc/source/glossary.rst index d375349603df..17071c8f175f 100644 --- a/doc/source/glossary.rst +++ b/doc/source/glossary.rst @@ -2,370 +2,510 @@ Glossary ******** -.. toctree:: - .. glossary:: + + (`n`,) + A parenthesized number followed by a comma denotes a tuple with one + element. The trailing comma distinguishes a one-element tuple from a + parenthesized ``n``. + + + -1 + - **In a dimension entry**, instructs NumPy to choose the length + that will keep the total number of array elements the same. + + >>> np.arange(12).reshape(4, -1).shape + (4, 3) + + - **In an index**, any negative value + `denotes `_ + indexing from the right. + + . . . + An :py:data:`Ellipsis`. + + - **When indexing an array**, shorthand that the missing axes, if they + exist, are full slices. + + >>> a = np.arange(24).reshape(2,3,4) + + >>> a[...].shape + (2, 3, 4) + + >>> a[...,0].shape + (2, 3) + + >>> a[0,...].shape + (3, 4) + + >>> a[0,...,0].shape + (3,) + + It can be used at most once; ``a[...,0,...]`` raises an :exc:`IndexError`. + + - **In printouts**, NumPy substitutes ``...`` for the middle elements of + large arrays. To see the entire array, use `numpy.printoptions` + + + : + The Python :term:`python:slice` + operator. In ndarrays, slicing can be applied to every + axis: + + >>> a = np.arange(24).reshape(2,3,4) + >>> a + array([[[ 0, 1, 2, 3], + [ 4, 5, 6, 7], + [ 8, 9, 10, 11]], + + [[12, 13, 14, 15], + [16, 17, 18, 19], + [20, 21, 22, 23]]]) + + >>> a[1:,-2:,:-1] + array([[[16, 17, 18], + [20, 21, 22]]]) + + Trailing slices can be omitted: :: + + >>> a[1] == a[1,:,:] + array([[ True, True, True, True], + [ True, True, True, True], + [ True, True, True, True]]) + + In contrast to Python, where slicing creates a copy, in NumPy slicing + creates a :term:`view`. + + For details, see :ref:`combining-advanced-and-basic-indexing`. + + + < + In a dtype declaration, indicates that the data is + :term:`little-endian` (the bracket is big on the right). :: + + >>> dt = np.dtype(' + In a dtype declaration, indicates that the data is + :term:`big-endian` (the bracket is big on the left). :: + + >>> dt = np.dtype('>H') # big-endian unsigned short + + + advanced indexing + Rather than using a :doc:`scalar ` or slice as + an index, an axis can be indexed with an array, providing fine-grained + selection. This is known as :ref:`advanced indexing` + or "fancy indexing". + + along an axis - Axes are defined for arrays with more than one dimension. A - 2-dimensional array has two corresponding axes: the first running - vertically downwards across rows (axis 0), and the second running - horizontally across columns (axis 1). + An operation `along axis n` of array ``a`` behaves as if its argument + were an array of slices of ``a`` where each slice has a successive + index of axis `n`. - Many operations can take place along one of these axes. For example, - we can sum each row of an array, in which case we operate along - columns, or axis 1:: + For example, if ``a`` is a 3 x `N` array, an operation along axis 0 + behaves as if its argument were an array containing slices of each row: - >>> x = np.arange(12).reshape((3,4)) + >>> np.array((a[0,:], a[1,:], a[2,:])) #doctest: +SKIP - >>> x - array([[ 0, 1, 2, 3], - [ 4, 5, 6, 7], - [ 8, 9, 10, 11]]) + To make it concrete, we can pick the operation to be the array-reversal + function :func:`numpy.flip`, which accepts an ``axis`` argument. We + construct a 3 x 4 array ``a``: - >>> x.sum(axis=1) - array([ 6, 22, 38]) + >>> a = np.arange(12).reshape(3,4) + >>> a + array([[ 0, 1, 2, 3], + [ 4, 5, 6, 7], + [ 8, 9, 10, 11]]) - array - A homogeneous container of numerical elements. Each element in the - array occupies a fixed amount of memory (hence homogeneous), and - can be a numerical element of a single type (such as float, int - or complex) or a combination (such as ``(float, int, float)``). Each - array has an associated data-type (or ``dtype``), which describes - the numerical type of its elements:: + Reversing along axis 0 (the row axis) yields - >>> x = np.array([1, 2, 3], float) + >>> np.flip(a,axis=0) + array([[ 8, 9, 10, 11], + [ 4, 5, 6, 7], + [ 0, 1, 2, 3]]) - >>> x - array([ 1., 2., 3.]) + Recalling the definition of `along an axis`, ``flip`` along axis 0 is + treating its argument as if it were - >>> x.dtype # floating point number, 64 bits of memory per element - dtype('float64') + >>> np.array((a[0,:], a[1,:], a[2,:])) + array([[ 0, 1, 2, 3], + [ 4, 5, 6, 7], + [ 8, 9, 10, 11]]) + and the result of ``np.flip(a,axis=0)`` is to reverse the slices: - # More complicated data type: each array element is a combination of - # and integer and a floating point number - >>> np.array([(1, 2.0), (3, 4.0)], dtype=[('x', np.int64), ('y', float)]) - array([(1, 2.), (3, 4.)], dtype=[('x', '>> np.array((a[2,:],a[1,:],a[0,:])) + array([[ 8, 9, 10, 11], + [ 4, 5, 6, 7], + [ 0, 1, 2, 3]]) + + + array + Used synonymously in the NumPy docs with :term:`ndarray`. - Fast element-wise operations, called a :term:`ufunc`, operate on arrays. array_like - Any sequence that can be interpreted as an ndarray. This includes - nested lists, tuples, scalars and existing arrays. + Any :doc:`scalar ` or + :term:`python:sequence` + that can be interpreted as an ndarray. In addition to ndarrays + and scalars this category includes lists (possibly nested and with + different element types) and tuples. Any argument accepted by + :doc:`numpy.array ` + is array_like. :: - big-endian - When storing a multi-byte value in memory as a sequence of bytes, the - sequence addresses/sends/stores the most significant byte first (lowest - address) and the least significant byte last (highest address). Common in - micro-processors and used for transmission of data over network protocols. + >>> a = np.array([[1, 2.0], [0, 0], (1+1j, 3.)]) - BLAS - `Basic Linear Algebra Subprograms `_ + >>> a + array([[1.+0.j, 2.+0.j], + [0.+0.j, 0.+0.j], + [1.+1.j, 3.+0.j]]) - broadcast - NumPy can do operations on arrays whose shapes are mismatched:: - >>> x = np.array([1, 2]) - >>> y = np.array([[3], [4]]) + array scalar + For uniformity in handling operands, NumPy treats + a :doc:`scalar ` as an array of zero + dimension. - >>> x - array([1, 2]) - >>> y - array([[3], - [4]]) + axis + Another term for an array dimension. Axes are numbered left to right; + axis 0 is the first element in the shape tuple. - >>> x + y - array([[4, 5], - [5, 6]]) + In a two-dimensional vector, the elements of axis 0 are rows and the + elements of axis 1 are columns. - See `basics.broadcasting` for more information. + In higher dimensions, the picture changes. NumPy prints + higher-dimensional vectors as replications of row-by-column building + blocks, as in this three-dimensional vector: - C order - See `row-major` + >>> a = np.arange(12).reshape(2,2,3) + >>> a + array([[[ 0, 1, 2], + [ 3, 4, 5]], + [[ 6, 7, 8], + [ 9, 10, 11]]]) - column-major - A way to represent items in a N-dimensional array in the 1-dimensional - computer memory. In column-major order, the leftmost index "varies the - fastest": for example the array:: + ``a`` is depicted as a two-element array whose elements are 2x3 vectors. + From this point of view, rows and columns are the final two axes, + respectively, in any shape. - [[1, 2, 3], - [4, 5, 6]] + This rule helps you anticipate how a vector will be printed, and + conversely how to find the index of any of the printed elements. For + instance, in the example, the last two values of 8's index must be 0 and + 2. Since 8 appears in the second of the two 2x3's, the first index must + be 1: - is represented in the column-major order as:: + >>> a[1,0,2] + 8 - [1, 4, 2, 5, 3, 6] + A convenient way to count dimensions in a printed vector is to + count ``[`` symbols after the open-parenthesis. This is + useful in distinguishing, say, a (1,2,3) shape from a (2,3) shape: - Column-major order is also known as the Fortran order, as the Fortran - programming language uses it. + >>> a = np.arange(6).reshape(2,3) + >>> a.ndim + 2 + >>> a + array([[0, 1, 2], + [3, 4, 5]]) - decorator - An operator that transforms a function. For example, a ``log`` - decorator may be defined to print debugging information upon - function execution:: + >>> a = np.arange(6).reshape(1,2,3) + >>> a.ndim + 3 + >>> a + array([[[0, 1, 2], + [3, 4, 5]]]) - >>> def log(f): - ... def new_logging_func(*args, **kwargs): - ... print("Logging call with parameters:", args, kwargs) - ... return f(*args, **kwargs) - ... - ... return new_logging_func - Now, when we define a function, we can "decorate" it using ``log``:: + .base - >>> @log - ... def add(a, b): - ... return a + b + If an array does not own its memory, then its + :doc:`base ` attribute returns + the object whose memory the array is referencing. That object may be + referencing the memory from still another object, so the owning object + may be ``a.base.base.base...``. Some writers erroneously claim that + testing ``base`` determines if arrays are :term:`view`\ s. For the + correct way, see :func:`numpy.shares_memory`. - Calling ``add`` then yields: - >>> add(1, 2) - Logging call with parameters: (1, 2) {} - 3 + big-endian + See `Endianness `_. - dictionary - Resembling a language dictionary, which provides a mapping between - words and descriptions thereof, a Python dictionary is a mapping - between two objects:: - >>> x = {1: 'one', 'two': [1, 2]} + BLAS + `Basic Linear Algebra Subprograms `_ - Here, `x` is a dictionary mapping keys to values, in this case - the integer 1 to the string "one", and the string "two" to - the list ``[1, 2]``. The values may be accessed using their - corresponding keys:: - >>> x[1] - 'one' + broadcast + *broadcasting* is NumPy's ability to process ndarrays of + different sizes as if all were the same size. - >>> x['two'] - [1, 2] + It permits an elegant do-what-I-mean behavior where, for instance, + adding a scalar to a vector adds the scalar value to every element. - Note that dictionaries are not stored in any specific order. Also, - most mutable (see *immutable* below) objects, such as lists, may not - be used as keys. + >>> a = np.arange(3) + >>> a + array([0, 1, 2]) - For more information on dictionaries, read the - `Python tutorial `_. + >>> a + [3, 3, 3] + array([3, 4, 5]) - field - In a :term:`structured data type`, each sub-type is called a `field`. - The `field` has a name (a string), a type (any valid dtype), and - an optional `title`. See :ref:`arrays.dtypes` + >>> a + 3 + array([3, 4, 5]) - Fortran order - See `column-major` + Ordinarly, vector operands must all be the same size, because NumPy + works element by element -- for instance, ``c = a * b`` is :: - flattened - Collapsed to a one-dimensional array. See `numpy.ndarray.flatten` - for details. + c[0,0,0] = a[0,0,0] * b[0,0,0] + c[0,0,1] = a[0,0,1] * b[0,0,1] + ... - homogeneous - Describes a block of memory comprised of blocks, each block comprised of - items and of the same size, and blocks are interpreted in exactly the - same way. In the simplest case each block contains a single item, for - instance int32 or float64. - - immutable - An object that cannot be modified after execution is called - immutable. Two common examples are strings and tuples. - - iterable - A sequence that allows "walking" (iterating) over items, typically - using a loop such as:: - - >>> x = [1, 2, 3] - >>> [item**2 for item in x] - [1, 4, 9] - - It is often used in combination with ``enumerate``:: - >>> keys = ['a','b','c'] - >>> for n, k in enumerate(keys): - ... print("Key %d: %s" % (n, k)) - ... - Key 0: a - Key 1: b - Key 2: c + But in certain useful cases, NumPy can duplicate data along "missing" + axes or "too-short" dimensions so shapes will match. The duplication + costs no memory or time. For details, see + :doc:`Broadcasting. ` - itemsize - The size of the dtype element in bytes. - list - A Python container that can hold any number of objects or items. - The items do not have to be of the same type, and can even be - lists themselves:: + C order + Same as :term:`row-major`. + + + column-major + See `Row- and column-major order `_. + - >>> x = [2, 2.0, "two", [2, 2.0]] + contiguous + An array is contiguous if + * it occupies an unbroken block of memory, and + * array elements with higher indexes occupy higher addresses (that + is, no :term:`stride` is negative). - The list `x` contains 4 items, each which can be accessed individually:: - >>> x[2] # the string 'two' - 'two' + copy + See :term:`view`. - >>> x[3] # a list, containing an integer 2 and a float 2.0 - [2, 2.0] - It is also possible to select more than one item at a time, - using *slicing*:: + dimension + See :term:`axis`. - >>> x[0:2] # or, equivalently, x[:2] - [2, 2.0] - In code, arrays are often conveniently expressed as nested lists:: + dtype + The datatype describing the (identically typed) elements in an ndarray. + It can be changed to reinterpret the array contents. For details, see + :doc:`Data type objects (dtype). ` - >>> np.array([[1, 2], [3, 4]]) - array([[1, 2], - [3, 4]]) + fancy indexing + Another term for :term:`advanced indexing`. + + + field + In a :term:`structured data type`, each subtype is called a `field`. + The `field` has a name (a string), a type (any valid dtype), and + an optional `title`. See :ref:`arrays.dtypes`. + + + Fortran order + Same as :term:`column-major`. + + + flattened + See :term:`ravel`. + + + homogeneous + All elements of a homogeneous array have the same type. ndarrays, in + contrast to Python lists, are homogeneous. The type can be complicated, + as in a :term:`structured array`, but all elements have that type. + + NumPy `object arrays <#term-object-array>`_, which contain references to + Python objects, fill the role of heterogeneous arrays. + + + itemsize + The size of the dtype element in bytes. - For more information, read the section on lists in the `Python - tutorial `_. For a mapping - type (key-value), see *dictionary*. little-endian - When storing a multi-byte value in memory as a sequence of bytes, the - sequence addresses/sends/stores the least significant byte first (lowest - address) and the most significant byte last (highest address). Common in - x86 processors. + See `Endianness `_. + mask - A boolean array, used to select only certain elements for an operation:: + A boolean array used to select only certain elements for an operation: - >>> x = np.arange(5) - >>> x - array([0, 1, 2, 3, 4]) + >>> x = np.arange(5) + >>> x + array([0, 1, 2, 3, 4]) - >>> mask = (x > 2) - >>> mask - array([False, False, False, True, True]) + >>> mask = (x > 2) + >>> mask + array([False, False, False, True, True]) + + >>> x[mask] = -1 + >>> x + array([ 0, 1, 2, -1, -1]) - >>> x[mask] = -1 - >>> x - array([ 0, 1, 2, -1, -1]) masked array - Array that suppressed values indicated by a mask:: + Bad or missing data can be cleanly ignored by putting it in a masked + array, which has an internal boolean array indicating invalid + entries. Operations with masked arrays ignore these entries. :: - >>> x = np.ma.masked_array([np.nan, 2, np.nan], [True, False, True]) - >>> x + >>> a = np.ma.masked_array([np.nan, 2, np.nan], [True, False, True]) + >>> a masked_array(data=[--, 2.0, --], mask=[ True, False, True], fill_value=1e+20) - >>> x + [1, 2, 3] + >>> a + [1, 2, 3] masked_array(data=[--, 4.0, --], mask=[ True, False, True], fill_value=1e+20) + For details, see :doc:`Masked arrays. ` - Masked arrays are often used when operating on arrays containing - missing or invalid entries. matrix - A 2-dimensional ndarray that preserves its two-dimensional nature - throughout operations. It has certain special operations, such as ``*`` - (matrix multiplication) and ``**`` (matrix power), defined:: - - >>> x = np.mat([[1, 2], [3, 4]]) - >>> x - matrix([[1, 2], - [3, 4]]) + NumPy's two-dimensional + :doc:`matrix class ` + should no longer be used; use regular ndarrays. - >>> x**2 - matrix([[ 7, 10], - [15, 22]]) ndarray - See *array*. + :doc:`NumPy's basic structure `. + + + object array + An array whose dtype is ``object``; that is, it contains references to + Python objects. Indexing the array dereferences the Python objects, so + unlike other ndarrays, an object array has the ability to hold + heterogeneous objects. + + + ravel + :doc:`numpy.ravel \ + ` + and :doc:`numpy.flatten \ + ` + both flatten an ndarray. ``ravel`` will return a view if possible; + ``flatten`` always returns a copy. + + Flattening collapses a multimdimensional array to a single dimension; + details of how this is done (for instance, whether ``a[n+1]`` should be + the next row or next column) are parameters. + record array - An :term:`ndarray` with :term:`structured data type` which has been - subclassed as ``np.recarray`` and whose dtype is of type ``np.record``, - making the fields of its data type to be accessible by attribute. + A :term:`structured array` with allowing access in an attribute style + (``a.field``) in addition to ``a['field']``. For details, see + :doc:`numpy.recarray. ` - reference - If ``a`` is a reference to ``b``, then ``(a is b) == True``. Therefore, - ``a`` and ``b`` are different names for the same Python object. row-major - A way to represent items in a N-dimensional array in the 1-dimensional - computer memory. In row-major order, the rightmost index "varies - the fastest": for example the array:: + See `Row- and column-major order `_. + NumPy creates arrays in row-major order by default. - [[1, 2, 3], - [4, 5, 6]] - is represented in the row-major order as:: + scalar + In NumPy, usually a synonym for :term:`array scalar`. - [1, 2, 3, 4, 5, 6] - Row-major order is also known as the C order, as the C programming - language uses it. New NumPy arrays are by default in row-major order. + shape + A tuple showing the length of each dimension of an ndarray. The + length of the tuple itself is the number of dimensions + (:doc:`numpy.ndim `). + The product of the tuple elements is the number of elements in the + array. For details, see + :doc:`numpy.ndarray.shape `. - slice - Used to select only certain elements from a sequence: - >>> x = range(5) - >>> x - [0, 1, 2, 3, 4] + stride + Physical memory is one-dimensional; strides provide a mechanism to map + a given index to an address in memory. For an N-dimensional array, its + ``strides`` attribute is an N-element tuple; advancing from index + ``i`` to index ``i+1`` on axis ``n`` means adding ``a.strides[n]`` bytes + to the address. - >>> x[1:3] # slice from 1 to 3 (excluding 3 itself) - [1, 2] + Strides are computed automatically from an array's dtype and + shape, but can be directly specified using + :doc:`as_strided. ` - >>> x[1:5:2] # slice from 1 to 5, but skipping every second element - [1, 3] + For details, see + :doc:`numpy.ndarray.strides `. - >>> x[::-1] # slice a sequence in reverse - [4, 3, 2, 1, 0] + To see how striding underlies the power of NumPy views, see + `The NumPy array: a structure for efficient numerical computation. \ + `_ - Arrays may have more than one dimension, each which can be sliced - individually: - >>> x = np.array([[1, 2], [3, 4]]) - >>> x - array([[1, 2], - [3, 4]]) + structured array + Array whose :term:`dtype` is a :term:`structured data type`. - >>> x[:, 1] - array([2, 4]) structured data type - A data type composed of other datatypes + Users can create arbitrarily complex :term:`dtypes ` + that can include other arrays and dtypes. These composite dtypes are called + :doc:`structured data types. ` + + + subarray + An array nested in a :term:`structured data type`, as ``b`` is here: + + >>> dt = np.dtype([('a', np.int32), ('b', np.float32, (3,))]) + >>> np.zeros(3, dtype=dt) + array([(0, [0., 0., 0.]), (0, [0., 0., 0.]), (0, [0., 0., 0.])], + dtype=[('a', '>> dt = np.dtype([('a', np.int32), ('b', np.float32, (3,))]) - >>> np.zeros(3, dtype=dt) - array([(0, [0., 0., 0.]), (0, [0., 0., 0.]), (0, [0., 0., 0.])], - dtype=[('a', '` which is an alias to the name and is - commonly used for plotting. + An alias for a field name in a structured datatype. + + + type + In NumPy, usually a synonym for :term:`dtype`. For the more general + Python meaning, :term:`see here. ` + ufunc - Universal function. A fast element-wise, :term:`vectorized - ` array operation. Examples include ``add``, ``sin`` and - ``logical_or``. + NumPy's fast element-by-element computation (:term:`vectorization`) + gives a choice which function gets applied. The general term for the + function is ``ufunc``, short for ``universal function``. NumPy routines + have built-in ufuncs, but users can also + :doc:`write their own. ` + vectorization - Optimizing a looping block by specialized code. In a traditional sense, - vectorization performs the same operation on multiple elements with - fixed strides between them via specialized hardware. Compilers know how - to take advantage of well-constructed loops to implement such - optimizations. NumPy uses :ref:`vectorization ` - to mean any optimization via specialized code performing the same - operations on multiple elements, typically achieving speedups by - avoiding some of the overhead in looking up and converting the elements. + NumPy hands off array processing to C, where looping and computation are + much faster than in Python. To exploit this, programmers using NumPy + eliminate Python loops in favor of array-to-array operations. + :term:`vectorization` can refer both to the C offloading and to + structuring NumPy code to leverage it. view - An array that does not own its data, but refers to another array's - data instead. For example, we may create a view that only shows - every second element of another array:: + Without touching underlying data, NumPy can make one array appear + to change its datatype and shape. + + An array created this way is a `view`, and NumPy often exploits the + performance gain of using a view versus making a new array. + + A potential drawback is that writing to a view can alter the original + as well. If this is a problem, NumPy instead needs to create a + physically distinct array -- a `copy`. + + Some NumPy routines always return views, some always return copies, some + may return one or the other, and for some the choice can be specified. + Responsiblity for managing views and copies falls to the programmer. + :func:`numpy.shares_memory` will check whether ``b`` is a view of + ``a``, but an exact answer isn't always feasible, as the documentation + page explains. >>> x = np.arange(5) >>> x @@ -379,15 +519,3 @@ Glossary >>> y array([3, 2, 4]) - wrapper - Python is a high-level (highly abstracted, or English-like) language. - This abstraction comes at a price in execution speed, and sometimes - it becomes necessary to use lower level languages to do fast - computations. A wrapper is code that provides a bridge between - high and the low level languages, allowing, e.g., Python to execute - code written in C or Fortran. - - Examples include ctypes, SWIG and Cython (which wraps C and C++) - and f2py (which wraps Fortran). - - diff --git a/doc/source/license.rst b/doc/source/license.rst index 8f360af8830e..beea023ce05a 100644 --- a/doc/source/license.rst +++ b/doc/source/license.rst @@ -1,35 +1,6 @@ ************* -NumPy License +NumPy license ************* -Copyright (c) 2005, NumPy Developers - -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - -* Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - -* Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following - disclaimer in the documentation and/or other materials provided - with the distribution. - -* Neither the name of the NumPy Developers nor the names of any - contributors may be used to endorse or promote products derived - from this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +.. include:: ../../LICENSE.txt + :literal: diff --git a/doc/source/reference/arrays.classes.rst b/doc/source/reference/arrays.classes.rst index c5563bdddd45..3a4ed21689e5 100644 --- a/doc/source/reference/arrays.classes.rst +++ b/doc/source/reference/arrays.classes.rst @@ -480,16 +480,16 @@ Character arrays (:mod:`numpy.char`) The `chararray` class exists for backwards compatibility with Numarray, it is not recommended for new development. Starting from numpy 1.4, if one needs arrays of strings, it is recommended to use arrays of - `dtype` `object_`, `string_` or `unicode_`, and use the free functions + `dtype` `object_`, `bytes_` or `str_`, and use the free functions in the `numpy.char` module for fast vectorized string operations. -These are enhanced arrays of either :class:`string_` type or -:class:`unicode_` type. These arrays inherit from the +These are enhanced arrays of either :class:`str_` type or +:class:`bytes_` type. These arrays inherit from the :class:`ndarray`, but specially-define the operations ``+``, ``*``, and ``%`` on a (broadcasting) element-by-element basis. These operations are not available on the standard :class:`ndarray` of character type. In addition, the :class:`chararray` has all of the -standard :class:`string ` (and :class:`unicode`) methods, +standard :class:`str` (and :class:`bytes`) methods, executing them on an element-by-element basis. Perhaps the easiest way to create a chararray is to use :meth:`self.view(chararray) ` where *self* is an ndarray of str or unicode diff --git a/doc/source/reference/arrays.dtypes.rst b/doc/source/reference/arrays.dtypes.rst index 575984707ee3..b5ffa1a8b9c1 100644 --- a/doc/source/reference/arrays.dtypes.rst +++ b/doc/source/reference/arrays.dtypes.rst @@ -152,14 +152,6 @@ Array-scalar types >>> dt = np.dtype(np.complex128) # 128-bit complex floating-point number Generic types - .. deprecated NumPy 1.19:: - - The use of generic types is deprecated. This is because it can be - unexpected in a context such as ``arr.astype(dtype=np.floating)``. - ``arr.astype(dtype=np.floating)`` which casts an array of ``float32`` - to an array of ``float64``, even though ``float32`` is a subdtype of - ``np.floating``. - The generic hierarchical type objects convert to corresponding type objects according to the associations: @@ -172,6 +164,15 @@ Generic types :class:`generic`, :class:`flexible` :class:`void` ===================================================== =============== + .. deprecated:: 1.19 + + This conversion of generic scalar types is deprecated. + This is because it can be unexpected in a context such as + ``arr.astype(dtype=np.floating)``, which casts an array of ``float32`` + to an array of ``float64``, even though ``float32`` is a subdtype of + ``np.floating``. + + Built-in Python types Several python types are equivalent to a corresponding array scalar when used to generate a :class:`dtype` object: @@ -263,9 +264,8 @@ Array-protocol type strings (see :ref:`arrays.interface`) .. admonition:: Note on string types For backward compatibility with Python 2 the ``S`` and ``a`` typestrings - remain zero-terminated bytes and ``np.string_`` continues to map to - ``np.bytes_``. - To use actual strings in Python 3 use ``U`` or ``np.unicode_``. + remain zero-terminated bytes and `numpy.string_` continues to alias + `numpy.bytes_`. To use actual strings in Python 3 use ``U`` or `numpy.str_`. For signed bytes that do not need zero-termination ``b`` or ``i1`` can be used. @@ -344,7 +344,7 @@ Type strings ``[(field_name, field_dtype, field_shape), ...]`` *obj* should be a list of fields where each field is described by a tuple of length 2 or 3. (Equivalent to the ``descr`` item in the - :obj:`__array_interface__` attribute.) + :obj:`~object.__array_interface__` attribute.) The first element, *field_name*, is the field name (if this is ``''`` then a standard field name, ``'f#'``, is assigned). The @@ -391,9 +391,9 @@ Type strings their values must each be lists of the same length as the *names* and *formats* lists. The *offsets* value is a list of byte offsets (limited to `ctypes.c_int`) for each field, while the *titles* value is a - list of titles for each field (None can be used if no title is - desired for that field). The *titles* can be any :class:`string` - or :class:`unicode` object and will add another entry to the + list of titles for each field (``None`` can be used if no title is + desired for that field). The *titles* can be any object, but when a + :class:`str` object will add another entry to the fields dictionary keyed by the title and referencing the same field tuple which will contain the title as an additional tuple member. diff --git a/doc/source/reference/arrays.indexing.rst b/doc/source/reference/arrays.indexing.rst index 3e600b7c456e..9f82875ea65a 100644 --- a/doc/source/reference/arrays.indexing.rst +++ b/doc/source/reference/arrays.indexing.rst @@ -34,7 +34,7 @@ Basic Slicing and Indexing Basic slicing extends Python's basic concept of slicing to N dimensions. Basic slicing occurs when *obj* is a :class:`slice` object (constructed by ``start:stop:step`` notation inside of brackets), an -integer, or a tuple of slice objects and integers. :const:`Ellipsis` +integer, or a tuple of slice objects and integers. :py:data:`Ellipsis` and :const:`newaxis` objects can be interspersed with these as well. @@ -43,7 +43,7 @@ well. In order to remain backward compatible with a common usage in Numeric, basic slicing is also initiated if the selection object is any non-ndarray and non-tuple sequence (such as a :class:`list`) containing - :class:`slice` objects, the :const:`Ellipsis` object, or the :const:`newaxis` + :class:`slice` objects, the :py:data:`Ellipsis` object, or the :const:`newaxis` object, but not for integer arrays or other embedded sequences. .. index:: @@ -129,7 +129,7 @@ concepts to remember include: [5], [6]]]) -- :const:`Ellipsis` expands to the number of ``:`` objects needed for the +- :py:data:`Ellipsis` expands to the number of ``:`` objects needed for the selection tuple to index all dimensions. In most cases, this means that length of the expanded selection tuple is ``x.ndim``. There may only be a single ellipsis present. @@ -198,6 +198,7 @@ concepts to remember include: create an axis of length one. :const:`newaxis` is an alias for 'None', and 'None' can be used in place of this with the same result. +.. _advanced-indexing: Advanced Indexing ----------------- @@ -304,6 +305,8 @@ understood with an example. most important thing to remember about indexing with multiple advanced indexes. +.. _combining-advanced-and-basic-indexing: + Combining advanced and basic indexing """"""""""""""""""""""""""""""""""""" @@ -330,7 +333,7 @@ the subspace defined by the basic indexing (excluding integers) and the subspace from the advanced indexing part. Two cases of index combination need to be distinguished: -* The advanced indexes are separated by a slice, :const:`Ellipsis` or :const:`newaxis`. +* The advanced indexes are separated by a slice, :py:data:`Ellipsis` or :const:`newaxis`. For example ``x[arr1, :, arr2]``. * The advanced indexes are all next to each other. For example ``x[..., arr1, arr2, :]`` but *not* ``x[arr1, :, 1]`` @@ -450,7 +453,7 @@ also supports boolean arrays and will work without any surprises. array([[ 3, 5], [ 9, 11]]) - Without the ``np.ix_`` call or only the diagonal elements would be + Without the ``np.ix_`` call, only the diagonal elements would be selected. Or without ``np.ix_`` (compare the integer array examples): diff --git a/doc/source/reference/arrays.interface.rst b/doc/source/reference/arrays.interface.rst index 73e4aef0c9f8..6a8c5f9c4d09 100644 --- a/doc/source/reference/arrays.interface.rst +++ b/doc/source/reference/arrays.interface.rst @@ -49,9 +49,9 @@ Python side =========== This approach to the interface consists of the object having an -:data:`__array_interface__` attribute. +:data:`~object.__array_interface__` attribute. -.. data:: __array_interface__ +.. data:: object.__array_interface__ A dictionary of items (3 required and 5 optional). The optional keys in the dictionary have implied defaults if they are not @@ -60,17 +60,15 @@ This approach to the interface consists of the object having an The keys are: **shape** (required) - Tuple whose elements are the array size in each dimension. Each - entry is an integer (a Python int or long). Note that these - integers could be larger than the platform "int" or "long" - could hold (a Python int is a C long). It is up to the code + entry is an integer (a Python :py:class:`int`). Note that these + integers could be larger than the platform ``int`` or ``long`` + could hold (a Python :py:class:`int` is a C ``long``). It is up to the code using this attribute to handle this appropriately; either by raising an error when overflow is possible, or by using - :c:data:`Py_LONG_LONG` as the C type for the shapes. + ``long long`` as the C type for the shapes. **typestr** (required) - A string providing the basic type of the homogeneous array The basic string format consists of 3 parts: a character describing the byteorder of the data (``<``: little-endian, ``>``: @@ -97,7 +95,6 @@ This approach to the interface consists of the object having an ===== ================================================================ **descr** (optional) - A list of tuples providing a more detailed description of the memory layout for each item in the homogeneous array. Each tuple in the list has two or three elements. Normally, this @@ -127,7 +124,6 @@ This approach to the interface consists of the object having an **Default**: ``[('', typestr)]`` **data** (optional) - A 2-tuple whose first argument is an integer (a long integer if necessary) that points to the data-area storing the array contents. This pointer must point to the first element of @@ -136,7 +132,7 @@ This approach to the interface consists of the object having an means the data area is read-only). This attribute can also be an object exposing the - :c:func:`buffer interface ` which + :ref:`buffer interface ` which will be used to share the data. If this key is not present (or returns None), then memory sharing will be done through the buffer interface of the object itself. In this @@ -148,25 +144,23 @@ This approach to the interface consists of the object having an **Default**: None **strides** (optional) - - Either None to indicate a C-style contiguous array or + Either ``None`` to indicate a C-style contiguous array or a Tuple of strides which provides the number of bytes needed to jump to the next array element in the corresponding dimension. Each entry must be an integer (a Python - :const:`int` or :const:`long`). As with shape, the values may - be larger than can be represented by a C "int" or "long"; the + :py:class:`int`). As with shape, the values may + be larger than can be represented by a C ``int`` or ``long``; the calling code should handle this appropriately, either by - raising an error, or by using :c:type:`Py_LONG_LONG` in C. The - default is None which implies a C-style contiguous - memory buffer. In this model, the last dimension of the array + raising an error, or by using ``long long`` in C. The + default is ``None`` which implies a C-style contiguous + memory buffer. In this model, the last dimension of the array varies the fastest. For example, the default strides tuple for an object whose array entries are 8 bytes long and whose - shape is (10,20,30) would be (4800, 240, 8) + shape is ``(10, 20, 30)`` would be ``(4800, 240, 8)`` - **Default**: None (C-style contiguous) + **Default**: ``None`` (C-style contiguous) **mask** (optional) - None or an object exposing the array interface. All elements of the mask array should be interpreted only as true or not true indicating which elements of this array are valid. @@ -177,15 +171,13 @@ This approach to the interface consists of the object having an **Default**: None (All array values are valid) **offset** (optional) - An integer offset into the array data region. This can only be - used when data is None or returns a :class:`buffer` + used when data is ``None`` or returns a :class:`buffer` object. **Default**: 0. **version** (required) - An integer showing the version of the interface (i.e. 3 for this version). Be careful not to use this to invalidate objects exposing future versions of the interface. @@ -197,7 +189,7 @@ C-struct access This approach to the array interface allows for faster access to an array using only one attribute lookup and a well-defined C-structure. -.. c:var:: __array_struct__ +.. data:: object.__array_struct__ A :c:type:`PyCapsule` whose ``pointer`` member contains a pointer to a filled :c:type:`PyArrayInterface` structure. Memory @@ -231,13 +223,15 @@ as:: The flags member may consist of 5 bits showing how the data should be interpreted and one bit showing how the Interface should be -interpreted. The data-bits are :const:`CONTIGUOUS` (0x1), -:const:`FORTRAN` (0x2), :const:`ALIGNED` (0x100), :const:`NOTSWAPPED` -(0x200), and :const:`WRITEABLE` (0x400). A final flag -:const:`ARR_HAS_DESCR` (0x800) indicates whether or not this structure +interpreted. The data-bits are :c:macro:`NPY_ARRAY_C_CONTIGUOUS` (0x1), +:c:macro:`NPY_ARRAY_F_CONTIGUOUS` (0x2), :c:macro:`NPY_ARRAY_ALIGNED` (0x100), +:c:macro:`NPY_ARRAY_NOTSWAPPED` (0x200), and :c:macro:`NPY_ARRAY_WRITEABLE` (0x400). A final flag +:c:macro:`NPY_ARR_HAS_DESCR` (0x800) indicates whether or not this structure has the arrdescr field. The field should not be accessed unless this flag is present. + .. c:macro:: NPY_ARR_HAS_DESCR + .. admonition:: New since June 16, 2006: In the past most implementations used the ``desc`` member of the ``PyCObject`` @@ -254,7 +248,7 @@ Type description examples ========================= For clarity it is useful to provide some examples of the type -description and corresponding :data:`__array_interface__` 'descr' +description and corresponding :data:`~object.__array_interface__` 'descr' entries. Thanks to Scott Gilbert for these examples: In every case, the 'descr' key is optional, but of course provides diff --git a/doc/source/reference/arrays.ndarray.rst b/doc/source/reference/arrays.ndarray.rst index 689240c7d6c5..191367058dba 100644 --- a/doc/source/reference/arrays.ndarray.rst +++ b/doc/source/reference/arrays.ndarray.rst @@ -1,11 +1,11 @@ +.. currentmodule:: numpy + .. _arrays.ndarray: ****************************************** The N-dimensional array (:class:`ndarray`) ****************************************** -.. currentmodule:: numpy - An :class:`ndarray` is a (usually fixed-size) multidimensional container of items of the same type and size. The number of dimensions and items in an array is defined by its :attr:`shape `, @@ -259,10 +259,10 @@ Array interface .. seealso:: :ref:`arrays.interface`. -========================== =================================== -:obj:`__array_interface__` Python-side of the array interface -:obj:`__array_struct__` C-side of the array interface -========================== =================================== +================================== =================================== +:obj:`~object.__array_interface__` Python-side of the array interface +:obj:`~object.__array_struct__` C-side of the array interface +================================== =================================== :mod:`ctypes` foreign function interface ---------------------------------------- @@ -469,7 +469,7 @@ Comparison operators: ndarray.__eq__ ndarray.__ne__ -Truth value of an array (:func:`bool()`): +Truth value of an array (:class:`bool() `): .. autosummary:: :toctree: generated/ @@ -604,9 +604,9 @@ Container customization: (see :ref:`Indexing `) ndarray.__setitem__ ndarray.__contains__ -Conversion; the operations :func:`int()`, :func:`float()` and -:func:`complex()`. -. They work only on arrays that have one element in them +Conversion; the operations :class:`int() `, +:class:`float() ` and :class:`complex() `. +They work only on arrays that have one element in them and return the appropriate scalar. .. autosummary:: diff --git a/doc/source/reference/arrays.nditer.cython.rst b/doc/source/reference/arrays.nditer.cython.rst index 2cc7763edfce..43aad99275c7 100644 --- a/doc/source/reference/arrays.nditer.cython.rst +++ b/doc/source/reference/arrays.nditer.cython.rst @@ -5,7 +5,7 @@ Those who want really good performance out of their low level operations should strongly consider directly using the iteration API provided in C, but for those who are not comfortable with C or C++, Cython is a good middle ground with reasonable performance tradeoffs. For -the :class:`nditer` object, this means letting the iterator take care +the :class:`~numpy.nditer` object, this means letting the iterator take care of broadcasting, dtype conversion, and buffering, while giving the inner loop to Cython. diff --git a/doc/source/reference/arrays.scalars.rst b/doc/source/reference/arrays.scalars.rst index f57a117244aa..4b5da2e13a5c 100644 --- a/doc/source/reference/arrays.scalars.rst +++ b/doc/source/reference/arrays.scalars.rst @@ -29,9 +29,9 @@ an array scalar object. Alternatively, what kind of array scalar is present can be determined using other members of the data type hierarchy. Thus, for example ``isinstance(val, np.complexfloating)`` will return :py:data:`True` if *val* is a complex valued type, while -:const:`isinstance(val, np.flexible)` will return true if *val* is one -of the flexible itemsize array types (:class:`string`, -:class:`unicode`, :class:`void`). +``isinstance(val, np.flexible)`` will return true if *val* is one +of the flexible itemsize array types (:class:`str_`, +:class:`bytes_`, :class:`void`). .. figure:: figures/dtype-hierarchy.png @@ -41,6 +41,13 @@ of the flexible itemsize array types (:class:`string`, pointer for the platform. All the number types can be obtained using bit-width names as well. + +.. TODO - use something like this instead of the diagram above, as it generates + links to the classes and is a vector graphic. Unfortunately it looks worse + and the html element providing the linked regions is misaligned. + + .. inheritance-diagram:: byte short intc int_ longlong ubyte ushort uintc uint ulonglong half single double longdouble csingle cdouble clongdouble bool_ datetime64 timedelta64 object_ bytes_ str_ void + .. [#] However, array scalars are immutable, so none of the array scalar attributes are settable. @@ -51,43 +58,33 @@ of the flexible itemsize array types (:class:`string`, Built-in scalar types ===================== -The built-in scalar types are shown below. Along with their (mostly) -C-derived names, the integer, float, and complex data-types are also -available using a bit-width convention so that an array of the right -size can always be ensured (e.g. :class:`int8`, :class:`float64`, -:class:`complex128`). Two aliases (:class:`intp` and :class:`uintp`) -pointing to the integer type that is sufficiently large to hold a C pointer -are also provided. The C-like names are associated with character codes, -which are shown in the table. Use of the character codes, however, +The built-in scalar types are shown below. The C-like names are associated with character codes, +which are shown in their descriptions. Use of the character codes, however, is discouraged. Some of the scalar types are essentially equivalent to fundamental Python types and therefore inherit from them as well as from the generic array scalar type: -==================== ================================ -Array scalar type Related Python type -==================== ================================ -:class:`int_` :class:`IntType` (Python 2 only) -:class:`float_` :class:`FloatType` -:class:`complex_` :class:`ComplexType` -:class:`bytes_` :class:`BytesType` -:class:`unicode_` :class:`UnicodeType` -==================== ================================ +==================== =========================== ============= +Array scalar type Related Python type Inherits? +==================== =========================== ============= +:class:`int_` :class:`int` Python 2 only +:class:`float_` :class:`float` yes +:class:`complex_` :class:`complex` yes +:class:`bytes_` :class:`bytes` yes +:class:`str_` :class:`str` yes +:class:`bool_` :class:`bool` no +:class:`datetime64` :class:`datetime.datetime` no +:class:`timedelta64` :class:`datetime.timedelta` no +==================== =========================== ============= The :class:`bool_` data type is very similar to the Python -:class:`BooleanType` but does not inherit from it because Python's -:class:`BooleanType` does not allow itself to be inherited from, and +:class:`bool` but does not inherit from it because Python's +:class:`bool` does not allow itself to be inherited from, and on the C-level the size of the actual bool data is not the same as a Python Boolean scalar. -.. warning:: - - The :class:`bool_` type is not a subclass of the :class:`int_` type - (the :class:`bool_` is not even a number type). This is different - than Python's default implementation of :class:`bool` as a - sub-class of int. - .. warning:: The :class:`int_` type does **not** inherit from the @@ -96,84 +93,113 @@ Python Boolean scalar. .. tip:: The default data type in NumPy is :class:`float_`. -In the tables below, ``platform?`` means that the type may not be -available on all platforms. Compatibility with different C or Python -types is indicated: two types are compatible if their data is of the -same size and interpreted in the same way. - -Booleans: - -=================== ============================= =============== -Type Remarks Character code -=================== ============================= =============== -:class:`bool_` compatible: Python bool ``'?'`` -:class:`bool8` 8 bits -=================== ============================= =============== - -Integers: - -=================== ============================= =============== -:class:`byte` compatible: C char ``'b'`` -:class:`short` compatible: C short ``'h'`` -:class:`intc` compatible: C int ``'i'`` -:class:`int_` compatible: Python int ``'l'`` -:class:`longlong` compatible: C long long ``'q'`` -:class:`intp` large enough to fit a pointer ``'p'`` -:class:`int8` 8 bits -:class:`int16` 16 bits -:class:`int32` 32 bits -:class:`int64` 64 bits -=================== ============================= =============== - -Unsigned integers: - -=================== ============================= =============== -:class:`ubyte` compatible: C unsigned char ``'B'`` -:class:`ushort` compatible: C unsigned short ``'H'`` -:class:`uintc` compatible: C unsigned int ``'I'`` -:class:`uint` compatible: Python int ``'L'`` -:class:`ulonglong` compatible: C long long ``'Q'`` -:class:`uintp` large enough to fit a pointer ``'P'`` -:class:`uint8` 8 bits -:class:`uint16` 16 bits -:class:`uint32` 32 bits -:class:`uint64` 64 bits -=================== ============================= =============== - -Floating-point numbers: - -=================== ============================= =============== -:class:`half` ``'e'`` -:class:`single` compatible: C float ``'f'`` -:class:`double` compatible: C double -:class:`float_` compatible: Python float ``'d'`` -:class:`longfloat` compatible: C long float ``'g'`` -:class:`float16` 16 bits -:class:`float32` 32 bits -:class:`float64` 64 bits -:class:`float96` 96 bits, platform? -:class:`float128` 128 bits, platform? -=================== ============================= =============== - -Complex floating-point numbers: - -=================== ============================= =============== -:class:`csingle` ``'F'`` -:class:`complex_` compatible: Python complex ``'D'`` -:class:`clongfloat` ``'G'`` -:class:`complex64` two 32-bit floats -:class:`complex128` two 64-bit floats -:class:`complex192` two 96-bit floats, - platform? -:class:`complex256` two 128-bit floats, - platform? -=================== ============================= =============== - -Any Python object: - -=================== ============================= =============== -:class:`object_` any Python object ``'O'`` -=================== ============================= =============== +.. autoclass:: numpy.generic + :exclude-members: + +.. autoclass:: numpy.number + :exclude-members: + +Integer types +~~~~~~~~~~~~~ + +.. autoclass:: numpy.integer + :exclude-members: + +Signed integer types +++++++++++++++++++++ + +.. autoclass:: numpy.signedinteger + :exclude-members: + +.. autoclass:: numpy.byte + :exclude-members: + +.. autoclass:: numpy.short + :exclude-members: + +.. autoclass:: numpy.intc + :exclude-members: + +.. autoclass:: numpy.int_ + :exclude-members: + +.. autoclass:: numpy.longlong + :exclude-members: + +Unsigned integer types +++++++++++++++++++++++ + +.. autoclass:: numpy.unsignedinteger + :exclude-members: + +.. autoclass:: numpy.ubyte + :exclude-members: + +.. autoclass:: numpy.ushort + :exclude-members: + +.. autoclass:: numpy.uintc + :exclude-members: + +.. autoclass:: numpy.uint + :exclude-members: + +.. autoclass:: numpy.ulonglong + :exclude-members: + +Inexact types +~~~~~~~~~~~~~ + +.. autoclass:: numpy.inexact + :exclude-members: + +Floating-point types +++++++++++++++++++++ + +.. autoclass:: numpy.floating + :exclude-members: + +.. autoclass:: numpy.half + :exclude-members: + +.. autoclass:: numpy.single + :exclude-members: + +.. autoclass:: numpy.double + :exclude-members: + +.. autoclass:: numpy.longdouble + :exclude-members: + +Complex floating-point types +++++++++++++++++++++++++++++ + +.. autoclass:: numpy.complexfloating + :exclude-members: + +.. autoclass:: numpy.csingle + :exclude-members: + +.. autoclass:: numpy.cdouble + :exclude-members: + +.. autoclass:: numpy.clongdouble + :exclude-members: + +Other types +~~~~~~~~~~~ + +.. autoclass:: numpy.bool_ + :exclude-members: + +.. autoclass:: numpy.datetime64 + :exclude-members: + +.. autoclass:: numpy.timedelta64 + :exclude-members: + +.. autoclass:: numpy.object_ + :exclude-members: .. note:: @@ -195,11 +221,17 @@ size and the data they describe can be of different length in different arrays. (In the character codes ``#`` is an integer denoting how many elements the data type consists of.) -=================== ============================== ======== -:class:`bytes_` compatible: Python bytes ``'S#'`` -:class:`unicode_` compatible: Python unicode/str ``'U#'`` -:class:`void` ``'V#'`` -=================== ============================== ======== +.. autoclass:: numpy.flexible + :exclude-members: + +.. autoclass:: numpy.bytes_ + :exclude-members: + +.. autoclass:: numpy.str_ + :exclude-members: + +.. autoclass:: numpy.void + :exclude-members: .. warning:: @@ -214,12 +246,123 @@ elements the data type consists of.) convention more consistent with other Python modules such as the :mod:`struct` module. +Sized aliases +~~~~~~~~~~~~~ + +Along with their (mostly) +C-derived names, the integer, float, and complex data-types are also +available using a bit-width convention so that an array of the right +size can always be ensured. Two aliases (:class:`numpy.intp` and :class:`numpy.uintp`) +pointing to the integer type that is sufficiently large to hold a C pointer +are also provided. + +.. note that these are documented with ..attribute because that is what + autoclass does for aliases under the hood. + +.. autoclass:: numpy.bool8 + +.. attribute:: int8 + int16 + int32 + int64 + + Aliases for the signed integer types (one of `numpy.byte`, `numpy.short`, + `numpy.intc`, `numpy.int_` and `numpy.longlong`) with the specified number + of bits. + + Compatible with the C99 ``int8_t``, ``int16_t``, ``int32_t``, and + ``int64_t``, respectively. + +.. attribute:: uint8 + uint16 + uint32 + uint64 + + Alias for the unsigned integer types (one of `numpy.byte`, `numpy.short`, + `numpy.intc`, `numpy.int_` and `numpy.longlong`) with the specified number + of bits. + + Compatible with the C99 ``uint8_t``, ``uint16_t``, ``uint32_t``, and + ``uint64_t``, respectively. + +.. attribute:: intp + + Alias for the signed integer type (one of `numpy.byte`, `numpy.short`, + `numpy.intc`, `numpy.int_` and `np.longlong`) that is the same size as a + pointer. + + Compatible with the C ``intptr_t``. + + :Character code: ``'p'`` + +.. attribute:: uintp + + Alias for the unsigned integer type (one of `numpy.byte`, `numpy.short`, + `numpy.intc`, `numpy.int_` and `np.longlong`) that is the same size as a + pointer. + + Compatible with the C ``uintptr_t``. + + :Character code: ``'P'`` + +.. autoclass:: numpy.float16 + +.. autoclass:: numpy.float32 + +.. autoclass:: numpy.float64 + +.. attribute:: float96 + float128 + + Alias for `numpy.longdouble`, named after its size in bits. + The existence of these aliases depends on the platform. + +.. autoclass:: numpy.complex64 + +.. autoclass:: numpy.complex128 + +.. attribute:: complex192 + complex256 + + Alias for `numpy.clongdouble`, named after its size in bits. + The existance of these aliases depends on the platform. + +Other aliases +~~~~~~~~~~~~~ + +The first two of these are conveniences which resemble the names of the +builtin types, in the same style as `bool_`, `int_`, `str_`, `bytes_`, and +`object_`: + +.. autoclass:: numpy.float_ + +.. autoclass:: numpy.complex_ + +Some more use alternate naming conventions for extended-precision floats and +complex numbers: + +.. autoclass:: numpy.longfloat + +.. autoclass:: numpy.singlecomplex + +.. autoclass:: numpy.cfloat + +.. autoclass:: numpy.longcomplex + +.. autoclass:: numpy.clongfloat + +The following aliases originate from Python 2, and it is recommended that they +not be used in new code. + +.. autoclass:: numpy.string_ + +.. autoclass:: numpy.unicode_ Attributes ========== The array scalar objects have an :obj:`array priority -<__array_priority__>` of :c:data:`NPY_SCALAR_PRIORITY` +` of :c:data:`NPY_SCALAR_PRIORITY` (-1,000,000.0). They also do not (yet) have a :attr:`ctypes ` attribute. Otherwise, they share the same attributes as arrays: @@ -273,7 +416,6 @@ The exceptions to the above rules are given below: .. autosummary:: :toctree: generated/ - generic generic.__array__ generic.__array_wrap__ generic.squeeze diff --git a/doc/source/reference/c-api/array.rst b/doc/source/reference/c-api/array.rst index cfe4d2d51fa5..3aa541b79e64 100644 --- a/doc/source/reference/c-api/array.rst +++ b/doc/source/reference/c-api/array.rst @@ -24,7 +24,7 @@ These macros access the :c:type:`PyArrayObject` structure members and are defined in ``ndarraytypes.h``. The input argument, *arr*, can be any :c:type:`PyObject *` that is directly interpretable as a :c:type:`PyArrayObject *` (any instance of the :c:data:`PyArray_Type` -and itssub-types). +and its sub-types). .. c:function:: int PyArray_NDIM(PyArrayObject *arr) @@ -326,7 +326,7 @@ From scratch Create a new array with the provided data-type descriptor, *descr*, of the shape determined by *nd* and *dims*. -.. c:function:: PyArray_FILLWBYTE(PyObject* obj, int val) +.. c:function:: void PyArray_FILLWBYTE(PyObject* obj, int val) Fill the array pointed to by *obj* ---which must be a (subclass of) ndarray---with the contents of *val* (evaluated as a byte). @@ -604,14 +604,14 @@ From other objects .. c:function:: PyObject* PyArray_FromStructInterface(PyObject* op) Returns an ndarray object from a Python object that exposes the - :obj:`__array_struct__` attribute and follows the array interface + :obj:`~object.__array_struct__` attribute and follows the array interface protocol. If the object does not contain this attribute then a borrowed reference to :c:data:`Py_NotImplemented` is returned. .. c:function:: PyObject* PyArray_FromInterface(PyObject* op) Returns an ndarray object from a Python object that exposes the - :obj:`__array_interface__` attribute following the array interface + :obj:`~object.__array_interface__` attribute following the array interface protocol. If the object does not contain this attribute then a borrowed reference to :c:data:`Py_NotImplemented` is returned. @@ -790,17 +790,17 @@ Dealing with types General check of Python Type ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -.. c:function:: PyArray_Check(PyObject *op) +.. c:function:: int PyArray_Check(PyObject *op) Evaluates true if *op* is a Python object whose type is a sub-type of :c:data:`PyArray_Type`. -.. c:function:: PyArray_CheckExact(PyObject *op) +.. c:function:: int PyArray_CheckExact(PyObject *op) Evaluates true if *op* is a Python object with type :c:data:`PyArray_Type`. -.. c:function:: PyArray_HasArrayInterface(PyObject *op, PyObject *out) +.. c:function:: int PyArray_HasArrayInterface(PyObject *op, PyObject *out) If ``op`` implements any part of the array interface, then ``out`` will contain a new reference to the newly created ndarray using @@ -808,7 +808,8 @@ General check of Python Type conversion occurs. Otherwise, out will contain a borrowed reference to :c:data:`Py_NotImplemented` and no error condition is set. -.. c:function:: PyArray_HasArrayInterfaceType(op, dtype, context, out) +.. c:function:: int PyArray_HasArrayInterfaceType(\ + PyObject *op, PyArray_Descr *dtype, PyObject *context, PyObject *out) If ``op`` implements any part of the array interface, then ``out`` will contain a new reference to the newly created ndarray using @@ -819,7 +820,7 @@ General check of Python Type that looks for the :obj:`~numpy.class.__array__` attribute. `context` is unused. -.. c:function:: PyArray_IsZeroDim(op) +.. c:function:: int PyArray_IsZeroDim(PyObject *op) Evaluates true if *op* is an instance of (a subclass of) :c:data:`PyArray_Type` and has 0 dimensions. @@ -828,29 +829,29 @@ General check of Python Type Evaluates true if *op* is an instance of ``Py{cls}ArrType_Type``. -.. c:function:: PyArray_CheckScalar(op) +.. c:function:: int PyArray_CheckScalar(PyObject *op) Evaluates true if *op* is either an array scalar (an instance of a sub-type of :c:data:`PyGenericArr_Type` ), or an instance of (a sub-class of) :c:data:`PyArray_Type` whose dimensionality is 0. -.. c:function:: PyArray_IsPythonNumber(op) +.. c:function:: int PyArray_IsPythonNumber(PyObject *op) Evaluates true if *op* is an instance of a builtin numeric type (int, float, complex, long, bool) -.. c:function:: PyArray_IsPythonScalar(op) +.. c:function:: int PyArray_IsPythonScalar(PyObject *op) Evaluates true if *op* is a builtin Python scalar object (int, float, complex, bytes, str, long, bool). -.. c:function:: PyArray_IsAnyScalar(op) +.. c:function:: int PyArray_IsAnyScalar(PyObject *op) Evaluates true if *op* is either a Python scalar object (see :c:func:`PyArray_IsPythonScalar`) or an array scalar (an instance of a sub- type of :c:data:`PyGenericArr_Type` ). -.. c:function:: PyArray_CheckAnyScalar(op) +.. c:function:: int PyArray_CheckAnyScalar(PyObject *op) Evaluates true if *op* is a Python scalar object (see :c:func:`PyArray_IsPythonScalar`), an array scalar (an instance of a @@ -866,82 +867,82 @@ enumerated array data type. For the array type checking macros the argument must be a :c:type:`PyObject *` that can be directly interpreted as a :c:type:`PyArrayObject *`. -.. c:function:: PyTypeNum_ISUNSIGNED(int num) +.. c:function:: int PyTypeNum_ISUNSIGNED(int num) -.. c:function:: PyDataType_ISUNSIGNED(PyArray_Descr *descr) +.. c:function:: int PyDataType_ISUNSIGNED(PyArray_Descr *descr) -.. c:function:: PyArray_ISUNSIGNED(PyArrayObject *obj) +.. c:function:: int PyArray_ISUNSIGNED(PyArrayObject *obj) Type represents an unsigned integer. -.. c:function:: PyTypeNum_ISSIGNED(int num) +.. c:function:: int PyTypeNum_ISSIGNED(int num) -.. c:function:: PyDataType_ISSIGNED(PyArray_Descr *descr) +.. c:function:: int PyDataType_ISSIGNED(PyArray_Descr *descr) -.. c:function:: PyArray_ISSIGNED(PyArrayObject *obj) +.. c:function:: int PyArray_ISSIGNED(PyArrayObject *obj) Type represents a signed integer. -.. c:function:: PyTypeNum_ISINTEGER(int num) +.. c:function:: int PyTypeNum_ISINTEGER(int num) -.. c:function:: PyDataType_ISINTEGER(PyArray_Descr* descr) +.. c:function:: int PyDataType_ISINTEGER(PyArray_Descr* descr) -.. c:function:: PyArray_ISINTEGER(PyArrayObject *obj) +.. c:function:: int PyArray_ISINTEGER(PyArrayObject *obj) Type represents any integer. -.. c:function:: PyTypeNum_ISFLOAT(int num) +.. c:function:: int PyTypeNum_ISFLOAT(int num) -.. c:function:: PyDataType_ISFLOAT(PyArray_Descr* descr) +.. c:function:: int PyDataType_ISFLOAT(PyArray_Descr* descr) -.. c:function:: PyArray_ISFLOAT(PyArrayObject *obj) +.. c:function:: int PyArray_ISFLOAT(PyArrayObject *obj) Type represents any floating point number. -.. c:function:: PyTypeNum_ISCOMPLEX(int num) +.. c:function:: int PyTypeNum_ISCOMPLEX(int num) -.. c:function:: PyDataType_ISCOMPLEX(PyArray_Descr* descr) +.. c:function:: int PyDataType_ISCOMPLEX(PyArray_Descr* descr) -.. c:function:: PyArray_ISCOMPLEX(PyArrayObject *obj) +.. c:function:: int PyArray_ISCOMPLEX(PyArrayObject *obj) Type represents any complex floating point number. -.. c:function:: PyTypeNum_ISNUMBER(int num) +.. c:function:: int PyTypeNum_ISNUMBER(int num) -.. c:function:: PyDataType_ISNUMBER(PyArray_Descr* descr) +.. c:function:: int PyDataType_ISNUMBER(PyArray_Descr* descr) -.. c:function:: PyArray_ISNUMBER(PyArrayObject *obj) +.. c:function:: int PyArray_ISNUMBER(PyArrayObject *obj) Type represents any integer, floating point, or complex floating point number. -.. c:function:: PyTypeNum_ISSTRING(int num) +.. c:function:: int PyTypeNum_ISSTRING(int num) -.. c:function:: PyDataType_ISSTRING(PyArray_Descr* descr) +.. c:function:: int PyDataType_ISSTRING(PyArray_Descr* descr) -.. c:function:: PyArray_ISSTRING(PyArrayObject *obj) +.. c:function:: int PyArray_ISSTRING(PyArrayObject *obj) Type represents a string data type. -.. c:function:: PyTypeNum_ISPYTHON(int num) +.. c:function:: int PyTypeNum_ISPYTHON(int num) -.. c:function:: PyDataType_ISPYTHON(PyArray_Descr* descr) +.. c:function:: int PyDataType_ISPYTHON(PyArray_Descr* descr) -.. c:function:: PyArray_ISPYTHON(PyArrayObject *obj) +.. c:function:: int PyArray_ISPYTHON(PyArrayObject *obj) Type represents an enumerated type corresponding to one of the standard Python scalar (bool, int, float, or complex). -.. c:function:: PyTypeNum_ISFLEXIBLE(int num) +.. c:function:: int PyTypeNum_ISFLEXIBLE(int num) -.. c:function:: PyDataType_ISFLEXIBLE(PyArray_Descr* descr) +.. c:function:: int PyDataType_ISFLEXIBLE(PyArray_Descr* descr) -.. c:function:: PyArray_ISFLEXIBLE(PyArrayObject *obj) +.. c:function:: int PyArray_ISFLEXIBLE(PyArrayObject *obj) Type represents one of the flexible array types ( :c:data:`NPY_STRING`, :c:data:`NPY_UNICODE`, or :c:data:`NPY_VOID` ). -.. c:function:: PyDataType_ISUNSIZED(PyArray_Descr* descr): +.. c:function:: int PyDataType_ISUNSIZED(PyArray_Descr* descr) Type has no size information attached, and can be resized. Should only be called on flexible dtypes. Types that are attached to an array will always @@ -951,55 +952,55 @@ argument must be a :c:type:`PyObject *` that can be directly interpret For structured datatypes with no fields this function now returns False. -.. c:function:: PyTypeNum_ISUSERDEF(int num) +.. c:function:: int PyTypeNum_ISUSERDEF(int num) -.. c:function:: PyDataType_ISUSERDEF(PyArray_Descr* descr) +.. c:function:: int PyDataType_ISUSERDEF(PyArray_Descr* descr) -.. c:function:: PyArray_ISUSERDEF(PyArrayObject *obj) +.. c:function:: int PyArray_ISUSERDEF(PyArrayObject *obj) Type represents a user-defined type. -.. c:function:: PyTypeNum_ISEXTENDED(int num) +.. c:function:: int PyTypeNum_ISEXTENDED(int num) -.. c:function:: PyDataType_ISEXTENDED(PyArray_Descr* descr) +.. c:function:: int PyDataType_ISEXTENDED(PyArray_Descr* descr) -.. c:function:: PyArray_ISEXTENDED(PyArrayObject *obj) +.. c:function:: int PyArray_ISEXTENDED(PyArrayObject *obj) Type is either flexible or user-defined. -.. c:function:: PyTypeNum_ISOBJECT(int num) +.. c:function:: int PyTypeNum_ISOBJECT(int num) -.. c:function:: PyDataType_ISOBJECT(PyArray_Descr* descr) +.. c:function:: int PyDataType_ISOBJECT(PyArray_Descr* descr) -.. c:function:: PyArray_ISOBJECT(PyArrayObject *obj) +.. c:function:: int PyArray_ISOBJECT(PyArrayObject *obj) Type represents object data type. -.. c:function:: PyTypeNum_ISBOOL(int num) +.. c:function:: int PyTypeNum_ISBOOL(int num) -.. c:function:: PyDataType_ISBOOL(PyArray_Descr* descr) +.. c:function:: int PyDataType_ISBOOL(PyArray_Descr* descr) -.. c:function:: PyArray_ISBOOL(PyArrayObject *obj) +.. c:function:: int PyArray_ISBOOL(PyArrayObject *obj) Type represents Boolean data type. -.. c:function:: PyDataType_HASFIELDS(PyArray_Descr* descr) +.. c:function:: int PyDataType_HASFIELDS(PyArray_Descr* descr) -.. c:function:: PyArray_HASFIELDS(PyArrayObject *obj) +.. c:function:: int PyArray_HASFIELDS(PyArrayObject *obj) Type has fields associated with it. -.. c:function:: PyArray_ISNOTSWAPPED(m) +.. c:function:: int PyArray_ISNOTSWAPPED(PyArrayObject *m) Evaluates true if the data area of the ndarray *m* is in machine byte-order according to the array's data-type descriptor. -.. c:function:: PyArray_ISBYTESWAPPED(m) +.. c:function:: int PyArray_ISBYTESWAPPED(PyArrayObject *m) Evaluates true if the data area of the ndarray *m* is **not** in machine byte-order according to the array's data-type descriptor. -.. c:function:: Bool PyArray_EquivTypes( \ +.. c:function:: npy_bool PyArray_EquivTypes( \ PyArray_Descr* type1, PyArray_Descr* type2) Return :c:data:`NPY_TRUE` if *type1* and *type2* actually represent @@ -1008,18 +1009,18 @@ argument must be a :c:type:`PyObject *` that can be directly interpret :c:data:`NPY_LONG` and :c:data:`NPY_INT` are equivalent. Otherwise return :c:data:`NPY_FALSE`. -.. c:function:: Bool PyArray_EquivArrTypes( \ +.. c:function:: npy_bool PyArray_EquivArrTypes( \ PyArrayObject* a1, PyArrayObject * a2) Return :c:data:`NPY_TRUE` if *a1* and *a2* are arrays with equivalent types for this platform. -.. c:function:: Bool PyArray_EquivTypenums(int typenum1, int typenum2) +.. c:function:: npy_bool PyArray_EquivTypenums(int typenum1, int typenum2) Special case of :c:func:`PyArray_EquivTypes` (...) that does not accept flexible data types but may be easier to call. -.. c:function:: int PyArray_EquivByteorders({byteorder} b1, {byteorder} b2) +.. c:function:: int PyArray_EquivByteorders(int b1, int b2) True if byteorder characters ( :c:data:`NPY_LITTLE`, :c:data:`NPY_BIG`, :c:data:`NPY_NATIVE`, :c:data:`NPY_IGNORE` ) are @@ -1334,7 +1335,7 @@ Special functions for NPY_OBJECT locations in the structure with object data-types. No checking is performed but *arr* must be of data-type :c:type:`NPY_OBJECT` and be single-segment and uninitialized (no previous objects in - position). Use :c:func:`PyArray_DECREF` (*arr*) if you need to + position). Use :c:func:`PyArray_XDECREF` (*arr*) if you need to decrement all the items in the object array prior to calling this function. @@ -1353,7 +1354,7 @@ Special functions for NPY_OBJECT strides, ordering, etc.) Sets the :c:data:`NPY_ARRAY_WRITEBACKIFCOPY` flag and ``arr->base``, and set ``base`` to READONLY. Call :c:func:`PyArray_ResolveWritebackIfCopy` before calling - `Py_DECREF`` in order copy any changes back to ``base`` and + `Py_DECREF` in order copy any changes back to ``base`` and reset the READONLY flag. Returns 0 for success, -1 for failure. @@ -1529,7 +1530,7 @@ Flag checking For all of these macros *arr* must be an instance of a (subclass of) :c:data:`PyArray_Type`. -.. c:function:: PyArray_CHKFLAGS(PyObject *arr, flags) +.. c:function:: int PyArray_CHKFLAGS(PyObject *arr, int flags) The first parameter, arr, must be an ndarray or subclass. The parameter, *flags*, should be an integer consisting of bitwise @@ -1539,60 +1540,60 @@ For all of these macros *arr* must be an instance of a (subclass of) :c:data:`NPY_ARRAY_WRITEABLE`, :c:data:`NPY_ARRAY_WRITEBACKIFCOPY`, :c:data:`NPY_ARRAY_UPDATEIFCOPY`. -.. c:function:: PyArray_IS_C_CONTIGUOUS(PyObject *arr) +.. c:function:: int PyArray_IS_C_CONTIGUOUS(PyObject *arr) Evaluates true if *arr* is C-style contiguous. -.. c:function:: PyArray_IS_F_CONTIGUOUS(PyObject *arr) +.. c:function:: int PyArray_IS_F_CONTIGUOUS(PyObject *arr) Evaluates true if *arr* is Fortran-style contiguous. -.. c:function:: PyArray_ISFORTRAN(PyObject *arr) +.. c:function:: int PyArray_ISFORTRAN(PyObject *arr) Evaluates true if *arr* is Fortran-style contiguous and *not* C-style contiguous. :c:func:`PyArray_IS_F_CONTIGUOUS` is the correct way to test for Fortran-style contiguity. -.. c:function:: PyArray_ISWRITEABLE(PyObject *arr) +.. c:function:: int PyArray_ISWRITEABLE(PyObject *arr) Evaluates true if the data area of *arr* can be written to -.. c:function:: PyArray_ISALIGNED(PyObject *arr) +.. c:function:: int PyArray_ISALIGNED(PyObject *arr) Evaluates true if the data area of *arr* is properly aligned on the machine. -.. c:function:: PyArray_ISBEHAVED(PyObject *arr) +.. c:function:: int PyArray_ISBEHAVED(PyObject *arr) Evaluates true if the data area of *arr* is aligned and writeable and in machine byte-order according to its descriptor. -.. c:function:: PyArray_ISBEHAVED_RO(PyObject *arr) +.. c:function:: int PyArray_ISBEHAVED_RO(PyObject *arr) Evaluates true if the data area of *arr* is aligned and in machine byte-order. -.. c:function:: PyArray_ISCARRAY(PyObject *arr) +.. c:function:: int PyArray_ISCARRAY(PyObject *arr) Evaluates true if the data area of *arr* is C-style contiguous, and :c:func:`PyArray_ISBEHAVED` (*arr*) is true. -.. c:function:: PyArray_ISFARRAY(PyObject *arr) +.. c:function:: int PyArray_ISFARRAY(PyObject *arr) Evaluates true if the data area of *arr* is Fortran-style contiguous and :c:func:`PyArray_ISBEHAVED` (*arr*) is true. -.. c:function:: PyArray_ISCARRAY_RO(PyObject *arr) +.. c:function:: int PyArray_ISCARRAY_RO(PyObject *arr) Evaluates true if the data area of *arr* is C-style contiguous, aligned, and in machine byte-order. -.. c:function:: PyArray_ISFARRAY_RO(PyObject *arr) +.. c:function:: int PyArray_ISFARRAY_RO(PyObject *arr) Evaluates true if the data area of *arr* is Fortran-style contiguous, aligned, and in machine byte-order **.** -.. c:function:: PyArray_ISONESEGMENT(PyObject *arr) +.. c:function:: int PyArray_ISONESEGMENT(PyObject *arr) Evaluates true if the data area of *arr* consists of a single (C-style or Fortran-style) contiguous segment. @@ -1650,7 +1651,7 @@ Conversion destination must be an integer multiple of the number of elements in *val*. -.. c:function:: PyObject* PyArray_Byteswap(PyArrayObject* self, Bool inplace) +.. c:function:: PyObject* PyArray_Byteswap(PyArrayObject* self, npy_bool inplace) Equivalent to :meth:`ndarray.byteswap` (*self*, *inplace*). Return an array whose data area is byteswapped. If *inplace* is non-zero, then do @@ -2309,7 +2310,7 @@ Array Functions Other functions ^^^^^^^^^^^^^^^ -.. c:function:: Bool PyArray_CheckStrides( \ +.. c:function:: npy_bool PyArray_CheckStrides( \ int elsize, int nd, npy_intp numbytes, npy_intp const* dims, \ npy_intp const* newstrides) @@ -2352,7 +2353,9 @@ it is possible to do this. Defining an :c:type:`NpyAuxData` is similar to defining a class in C++, but the object semantics have to be tracked manually since the API is in C. Here's an example for a function which doubles up an element using -an element copier function as a primitive.:: +an element copier function as a primitive. + +.. code-block:: c typedef struct { NpyAuxData base; @@ -2416,12 +2419,12 @@ an element copier function as a primitive.:: functions should never set the Python exception on error, because they may be called from a multi-threaded context. -.. c:function:: NPY_AUXDATA_FREE(auxdata) +.. c:function:: void NPY_AUXDATA_FREE(NpyAuxData *auxdata) A macro which calls the auxdata's free function appropriately, does nothing if auxdata is NULL. -.. c:function:: NPY_AUXDATA_CLONE(auxdata) +.. c:function:: NpyAuxData *NPY_AUXDATA_CLONE(NpyAuxData *auxdata) A macro which calls the auxdata's clone function appropriately, returning a deep copy of the auxiliary data. @@ -2488,7 +2491,7 @@ this useful approach to looping over an array. *destination*, which must have size at least *iterator* ->nd_m1+1. -.. c:function:: PyArray_ITER_GOTO1D(PyObject* iterator, npy_intp index) +.. c:function:: void PyArray_ITER_GOTO1D(PyObject* iterator, npy_intp index) Set the *iterator* index and dataptr to the location in the array indicated by the integer *index* which points to an element in the @@ -2809,11 +2812,21 @@ Data-type descriptors Create a new data-type object with the byteorder set according to *newendian*. All referenced data-type objects (in subdescr and fields members of the data-type object) are also changed - (recursively). If a byteorder of :c:data:`NPY_IGNORE` is encountered it + (recursively). + + The value of *newendian* is one of these macros: + + .. c:macro:: NPY_IGNORE + NPY_SWAP + NPY_NATIVE + NPY_LITTLE + NPY_BIG + + If a byteorder of :c:data:`NPY_IGNORE` is encountered it is left alone. If newendian is :c:data:`NPY_SWAP`, then all byte-orders are swapped. Other valid newendian values are :c:data:`NPY_NATIVE`, - :c:data:`NPY_LITTLE`, and :c:data:`NPY_BIG` which all cause the returned - data-typed descriptor (and all it's + :c:data:`NPY_LITTLE`, and :c:data:`NPY_BIG` which all cause + the returned data-typed descriptor (and all it's referenced data-type descriptors) to have the corresponding byte- order. @@ -2947,9 +2960,9 @@ to. already a buffer object pointing to another object). If you need to hold on to the memory be sure to INCREF the base member. The chunk of memory is pointed to by *buf* ->ptr member and has length - *buf* ->len. The flags member of *buf* is :c:data:`NPY_BEHAVED_RO` with - the :c:data:`NPY_ARRAY_WRITEABLE` flag set if *obj* has a writeable buffer - interface. + *buf* ->len. The flags member of *buf* is :c:data:`NPY_ARRAY_ALIGNED` + with the :c:data:`NPY_ARRAY_WRITEABLE` flag set if *obj* has + a writeable buffer interface. .. c:function:: int PyArray_AxisConverter(PyObject* obj, int* axis) @@ -2959,7 +2972,7 @@ to. :c:data:`NPY_MAXDIMS` which is interpreted correctly by the C-API functions that take axis arguments. -.. c:function:: int PyArray_BoolConverter(PyObject* obj, Bool* value) +.. c:function:: int PyArray_BoolConverter(PyObject* obj, npy_bool* value) Convert any Python object, *obj*, to :c:data:`NPY_TRUE` or :c:data:`NPY_FALSE`, and place the result in *value*. @@ -3123,7 +3136,7 @@ the C-API is needed then some additional steps must be taken. be defined in another compilation unit. * Whenever :c:macro:`PY_ARRAY_UNIQUE_SYMBOL` is #defined, it also changes the name of the variable holding the C-API, which - defaults to :c:data:`PyArray_API`, to whatever the macro is + defaults to ``PyArray_API``, to whatever the macro is #defined to. Checking the API Version @@ -3138,21 +3151,31 @@ calling the function). That's why several functions are provided to check for numpy versions. The macros :c:data:`NPY_VERSION` and :c:data:`NPY_FEATURE_VERSION` corresponds to the numpy version used to build the extension, whereas the versions returned by the functions -PyArray_GetNDArrayCVersion and PyArray_GetNDArrayCFeatureVersion corresponds to -the runtime numpy's version. +:c:func:`PyArray_GetNDArrayCVersion` and :c:func:`PyArray_GetNDArrayCFeatureVersion` +corresponds to the runtime numpy's version. The rules for ABI and API compatibilities can be summarized as follows: - * Whenever :c:data:`NPY_VERSION` != PyArray_GetNDArrayCVersion, the + * Whenever :c:data:`NPY_VERSION` != ``PyArray_GetNDArrayCVersion()``, the extension has to be recompiled (ABI incompatibility). - * :c:data:`NPY_VERSION` == PyArray_GetNDArrayCVersion and - :c:data:`NPY_FEATURE_VERSION` <= PyArray_GetNDArrayCFeatureVersion means + * :c:data:`NPY_VERSION` == ``PyArray_GetNDArrayCVersion()`` and + :c:data:`NPY_FEATURE_VERSION` <= ``PyArray_GetNDArrayCFeatureVersion()`` means backward compatible changes. ABI incompatibility is automatically detected in every numpy's version. API incompatibility detection was added in numpy 1.4.0. If you want to supported many different numpy versions with one extension binary, you have to build your -extension with the lowest NPY_FEATURE_VERSION as possible. +extension with the lowest :c:data:`NPY_FEATURE_VERSION` as possible. + +.. c:macro:: NPY_VERSION + + The current version of the ndarray object (check to see if this + variable is defined to guarantee the ``numpy/arrayobject.h`` header is + being used). + +.. c:macro:: NPY_FEATURE_VERSION + + The current version of the C-API. .. c:function:: unsigned int PyArray_GetNDArrayCVersion(void) @@ -3233,7 +3256,7 @@ Memory management .. c:function:: char* PyDataMem_NEW(size_t nbytes) -.. c:function:: PyDataMem_FREE(char* ptr) +.. c:function:: void PyDataMem_FREE(char* ptr) .. c:function:: char* PyDataMem_RENEW(void * ptr, size_t newbytes) @@ -3242,7 +3265,7 @@ Memory management .. c:function:: npy_intp* PyDimMem_NEW(int nd) -.. c:function:: PyDimMem_FREE(char* ptr) +.. c:function:: void PyDimMem_FREE(char* ptr) .. c:function:: npy_intp* PyDimMem_RENEW(void* ptr, size_t newnd) @@ -3250,7 +3273,7 @@ Memory management .. c:function:: void* PyArray_malloc(size_t nbytes) -.. c:function:: PyArray_free(void* ptr) +.. c:function:: void PyArray_free(void* ptr) .. c:function:: void* PyArray_realloc(npy_intp* ptr, size_t nbytes) @@ -3259,6 +3282,8 @@ Memory management :c:data:`NPY_USE_PYMEM` is 0, if :c:data:`NPY_USE_PYMEM` is 1, then the Python memory allocator is used. + .. c:macro:: NPY_USE_PYMEM + .. c:function:: int PyArray_ResolveWritebackIfCopy(PyArrayObject* obj) If ``obj.flags`` has :c:data:`NPY_ARRAY_WRITEBACKIFCOPY` or (deprecated) @@ -3289,9 +3314,13 @@ be accomplished using two groups of macros. Typically, if one macro in a group is used in a code block, all of them must be used in the same code block. Currently, :c:data:`NPY_ALLOW_THREADS` is defined to the python-defined :c:data:`WITH_THREADS` constant unless the environment -variable :c:data:`NPY_NOSMP` is set in which case +variable ``NPY_NOSMP`` is set in which case :c:data:`NPY_ALLOW_THREADS` is defined to be 0. +.. c:macro:: NPY_ALLOW_THREADS + +.. c:macro:: WITH_THREADS + Group 1 """"""" @@ -3328,18 +3357,18 @@ Group 1 interpreter. This macro acquires the GIL and restores the Python state from the saved variable. - .. c:function:: NPY_BEGIN_THREADS_DESCR(PyArray_Descr *dtype) + .. c:function:: void NPY_BEGIN_THREADS_DESCR(PyArray_Descr *dtype) Useful to release the GIL only if *dtype* does not contain arbitrary Python objects which may need the Python interpreter during execution of the loop. - .. c:function:: NPY_END_THREADS_DESCR(PyArray_Descr *dtype) + .. c:function:: void NPY_END_THREADS_DESCR(PyArray_Descr *dtype) Useful to regain the GIL in situations where it was released using the BEGIN form of this macro. - .. c:function:: NPY_BEGIN_THREADS_THRESHOLDED(int loop_size) + .. c:function:: void NPY_BEGIN_THREADS_THRESHOLDED(int loop_size) Useful to release the GIL only if *loop_size* exceeds a minimum threshold, currently set to 500. Should be matched @@ -3430,12 +3459,6 @@ Other constants The maximum number of array arguments that can be used in functions. -.. c:macro:: NPY_VERSION - - The current version of the ndarray object (check to see if this - variable is defined to guarantee the numpy/arrayobject.h header is - being used). - .. c:macro:: NPY_FALSE Defined as 0 for use with Bool. @@ -3458,7 +3481,7 @@ Other constants Miscellaneous Macros ^^^^^^^^^^^^^^^^^^^^ -.. c:function:: PyArray_SAMESHAPE(PyArrayObject *a1, PyArrayObject *a2) +.. c:function:: int PyArray_SAMESHAPE(PyArrayObject *a1, PyArrayObject *a2) Evaluates as True if arrays *a1* and *a2* have the same shape. @@ -3493,11 +3516,11 @@ Miscellaneous Macros of the ordering which is lexicographic: comparing the real parts first and then the complex parts if the real parts are equal. -.. c:function:: PyArray_REFCOUNT(PyObject* op) +.. c:function:: npy_intp PyArray_REFCOUNT(PyObject* op) Returns the reference count of any Python object. -.. c:function:: PyArray_DiscardWritebackIfCopy(PyObject* obj) +.. c:function:: void PyArray_DiscardWritebackIfCopy(PyObject* obj) If ``obj.flags`` has :c:data:`NPY_ARRAY_WRITEBACKIFCOPY` or (deprecated) :c:data:`NPY_ARRAY_UPDATEIFCOPY`, this function clears the flags, `DECREF` s @@ -3508,7 +3531,7 @@ Miscellaneous Macros error when you are finished with ``obj``, just before ``Py_DECREF(obj)``. It may be called multiple times, or with ``NULL`` input. -.. c:function:: PyArray_XDECREF_ERR(PyObject* obj) +.. c:function:: void PyArray_XDECREF_ERR(PyObject* obj) Deprecated in 1.14, use :c:func:`PyArray_DiscardWritebackIfCopy` followed by ``Py_XDECREF`` @@ -3614,6 +3637,22 @@ Enumerated Types Wraps an index to the valid range if it is out of bounds. +.. c:type:: NPY_SEARCHSIDE + + A variable type indicating whether the index returned should be that of + the first suitable location (if :c:data:`NPY_SEARCHLEFT`) or of the last + (if :c:data:`NPY_SEARCHRIGHT`). + + .. c:var:: NPY_SEARCHLEFT + + .. c:var:: NPY_SEARCHRIGHT + +.. c:type:: NPY_SELECTKIND + + A variable type indicating the selection algorithm being used. + + .. c:var:: NPY_INTROSELECT + .. c:type:: NPY_CASTING .. versionadded:: 1.6 diff --git a/doc/source/reference/c-api/config.rst b/doc/source/reference/c-api/config.rst index c3e2c98af078..87130699bbf8 100644 --- a/doc/source/reference/c-api/config.rst +++ b/doc/source/reference/c-api/config.rst @@ -52,12 +52,15 @@ information is available to the pre-processor. .. c:macro:: NPY_SIZEOF_LONG_DOUBLE - sizeof(longdouble) (A macro defines **NPY_SIZEOF_LONGDOUBLE** as well.) +.. c:macro:: NPY_SIZEOF_LONGDOUBLE + + sizeof(longdouble) .. c:macro:: NPY_SIZEOF_PY_INTPTR_T - Size of a pointer on this platform (sizeof(void \*)) (A macro defines - NPY_SIZEOF_INTP as well.) +.. c:macro:: NPY_SIZEOF_INTP + + Size of a pointer on this platform (sizeof(void \*)) Platform information @@ -94,7 +97,7 @@ Platform information Defined in ``numpy/npy_endian.h``. -.. c:function:: PyArray_GetEndianness() +.. c:function:: int PyArray_GetEndianness() .. versionadded:: 1.3.0 @@ -102,6 +105,12 @@ Platform information One of :c:data:`NPY_CPU_BIG`, :c:data:`NPY_CPU_LITTLE`, or :c:data:`NPY_CPU_UNKNOWN_ENDIAN`. + .. c:macro:: NPY_CPU_BIG + + .. c:macro:: NPY_CPU_LITTLE + + .. c:macro:: NPY_CPU_UNKNOWN_ENDIAN + Compiler directives ------------------- diff --git a/doc/source/reference/c-api/deprecations.rst b/doc/source/reference/c-api/deprecations.rst index a382017a2ad7..5b1abc6f2add 100644 --- a/doc/source/reference/c-api/deprecations.rst +++ b/doc/source/reference/c-api/deprecations.rst @@ -48,7 +48,9 @@ warnings). To use the NPY_NO_DEPRECATED_API mechanism, you need to #define it to the target API version of NumPy before #including any NumPy headers. -If you want to confirm that your code is clean against 1.7, use:: +If you want to confirm that your code is clean against 1.7, use: + +.. code-block:: c #define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION diff --git a/doc/source/reference/c-api/dtype.rst b/doc/source/reference/c-api/dtype.rst index a04d852123ed..a1a53cdb600b 100644 --- a/doc/source/reference/c-api/dtype.rst +++ b/doc/source/reference/c-api/dtype.rst @@ -221,24 +221,17 @@ Defines Max and min values for integers ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -.. c:var:: NPY_MAX_INT{bits} - -.. c:var:: NPY_MAX_UINT{bits} - -.. c:var:: NPY_MIN_INT{bits} - +``NPY_MAX_INT{bits}``, ``NPY_MAX_UINT{bits}``, ``NPY_MIN_INT{bits}`` These are defined for ``{bits}`` = 8, 16, 32, 64, 128, and 256 and provide the maximum (minimum) value of the corresponding (unsigned) integer type. Note: the actual integer type may not be available on all platforms (i.e. 128-bit and 256-bit integers are rare). -.. c:var:: NPY_MIN_{type} - +``NPY_MIN_{type}`` This is defined for ``{type}`` = **BYTE**, **SHORT**, **INT**, **LONG**, **LONGLONG**, **INTP** -.. c:var:: NPY_MAX_{type} - +``NPY_MAX_{type}`` This is defined for all defined for ``{type}`` = **BYTE**, **UBYTE**, **SHORT**, **USHORT**, **INT**, **UINT**, **LONG**, **ULONG**, **LONGLONG**, **ULONGLONG**, **INTP**, **UINTP** @@ -414,6 +407,12 @@ Printf Formatting For help in printing, the following strings are defined as the correct format specifier in printf and related commands. - :c:data:`NPY_LONGLONG_FMT`, :c:data:`NPY_ULONGLONG_FMT`, - :c:data:`NPY_INTP_FMT`, :c:data:`NPY_UINTP_FMT`, - :c:data:`NPY_LONGDOUBLE_FMT` +.. c:macro:: NPY_LONGLONG_FMT + +.. c:macro:: NPY_ULONGLONG_FMT + +.. c:macro:: NPY_INTP_FMT + +.. c:macro:: NPY_UINTP_FMT + +.. c:macro:: NPY_LONGDOUBLE_FMT diff --git a/doc/source/reference/c-api/iterator.rst b/doc/source/reference/c-api/iterator.rst index 7eac8c367dab..ae96bb3fb056 100644 --- a/doc/source/reference/c-api/iterator.rst +++ b/doc/source/reference/c-api/iterator.rst @@ -1264,7 +1264,7 @@ functions provide that information. NPY_MAX_INTP is placed in the stride. Once the iterator is prepared for iteration (after a reset if - :c:data:`NPY_DELAY_BUFALLOC` was used), call this to get the strides + :c:data:`NPY_ITER_DELAY_BUFALLOC` was used), call this to get the strides which may be used to select a fast inner loop function. For example, if the stride is 0, that means the inner loop can always load its value into a variable once, then use the variable throughout the loop, diff --git a/doc/source/reference/c-api/types-and-structures.rst b/doc/source/reference/c-api/types-and-structures.rst index ee57d4680cb1..6a9c4a9cf766 100644 --- a/doc/source/reference/c-api/types-and-structures.rst +++ b/doc/source/reference/c-api/types-and-structures.rst @@ -69,6 +69,7 @@ PyArray_Type and PyArrayObject typeobject. .. c:type:: PyArrayObject + NPY_AO The :c:type:`PyArrayObject` C-structure contains all of the required information for an array. All instances of an ndarray (and its @@ -77,7 +78,7 @@ PyArray_Type and PyArrayObject provided macros. If you need a shorter name, then you can make use of :c:type:`NPY_AO` (deprecated) which is defined to be equivalent to :c:type:`PyArrayObject`. Direct access to the struct fields are - deprecated. Use the `PyArray_*(arr)` form instead. + deprecated. Use the ``PyArray_*(arr)`` form instead. .. code-block:: c @@ -93,84 +94,84 @@ PyArray_Type and PyArrayObject PyObject *weakreflist; } PyArrayObject; -.. c:macro:: PyArrayObject.PyObject_HEAD + .. c:macro:: PyObject_HEAD - This is needed by all Python objects. It consists of (at least) - a reference count member ( ``ob_refcnt`` ) and a pointer to the - typeobject ( ``ob_type`` ). (Other elements may also be present - if Python was compiled with special options see - Include/object.h in the Python source tree for more - information). The ob_type member points to a Python type - object. + This is needed by all Python objects. It consists of (at least) + a reference count member ( ``ob_refcnt`` ) and a pointer to the + typeobject ( ``ob_type`` ). (Other elements may also be present + if Python was compiled with special options see + Include/object.h in the Python source tree for more + information). The ob_type member points to a Python type + object. -.. c:member:: char *PyArrayObject.data + .. c:member:: char *data - Accessible via :c:data:`PyArray_DATA`, this data member is a - pointer to the first element of the array. This pointer can - (and normally should) be recast to the data type of the array. + Accessible via :c:data:`PyArray_DATA`, this data member is a + pointer to the first element of the array. This pointer can + (and normally should) be recast to the data type of the array. -.. c:member:: int PyArrayObject.nd + .. c:member:: int nd - An integer providing the number of dimensions for this - array. When nd is 0, the array is sometimes called a rank-0 - array. Such arrays have undefined dimensions and strides and - cannot be accessed. Macro :c:data:`PyArray_NDIM` defined in - ``ndarraytypes.h`` points to this data member. :c:data:`NPY_MAXDIMS` - is the largest number of dimensions for any array. + An integer providing the number of dimensions for this + array. When nd is 0, the array is sometimes called a rank-0 + array. Such arrays have undefined dimensions and strides and + cannot be accessed. Macro :c:data:`PyArray_NDIM` defined in + ``ndarraytypes.h`` points to this data member. :c:data:`NPY_MAXDIMS` + is the largest number of dimensions for any array. -.. c:member:: npy_intp PyArrayObject.dimensions + .. c:member:: npy_intp dimensions - An array of integers providing the shape in each dimension as - long as nd :math:`\geq` 1. The integer is always large enough - to hold a pointer on the platform, so the dimension size is - only limited by memory. :c:data:`PyArray_DIMS` is the macro - associated with this data member. + An array of integers providing the shape in each dimension as + long as nd :math:`\geq` 1. The integer is always large enough + to hold a pointer on the platform, so the dimension size is + only limited by memory. :c:data:`PyArray_DIMS` is the macro + associated with this data member. -.. c:member:: npy_intp *PyArrayObject.strides + .. c:member:: npy_intp *strides - An array of integers providing for each dimension the number of - bytes that must be skipped to get to the next element in that - dimension. Associated with macro :c:data:`PyArray_STRIDES`. + An array of integers providing for each dimension the number of + bytes that must be skipped to get to the next element in that + dimension. Associated with macro :c:data:`PyArray_STRIDES`. -.. c:member:: PyObject *PyArrayObject.base + .. c:member:: PyObject *base - Pointed to by :c:data:`PyArray_BASE`, this member is used to hold a - pointer to another Python object that is related to this array. - There are two use cases: + Pointed to by :c:data:`PyArray_BASE`, this member is used to hold a + pointer to another Python object that is related to this array. + There are two use cases: - - If this array does not own its own memory, then base points to the - Python object that owns it (perhaps another array object) - - If this array has the (deprecated) :c:data:`NPY_ARRAY_UPDATEIFCOPY` or - :c:data:`NPY_ARRAY_WRITEBACKIFCOPY` flag set, then this array is a working - copy of a "misbehaved" array. + - If this array does not own its own memory, then base points to the + Python object that owns it (perhaps another array object) + - If this array has the (deprecated) :c:data:`NPY_ARRAY_UPDATEIFCOPY` or + :c:data:`NPY_ARRAY_WRITEBACKIFCOPY` flag set, then this array is a working + copy of a "misbehaved" array. - When ``PyArray_ResolveWritebackIfCopy`` is called, the array pointed to - by base will be updated with the contents of this array. + When ``PyArray_ResolveWritebackIfCopy`` is called, the array pointed to + by base will be updated with the contents of this array. -.. c:member:: PyArray_Descr *PyArrayObject.descr + .. c:member:: PyArray_Descr *descr - A pointer to a data-type descriptor object (see below). The - data-type descriptor object is an instance of a new built-in - type which allows a generic description of memory. There is a - descriptor structure for each data type supported. This - descriptor structure contains useful information about the type - as well as a pointer to a table of function pointers to - implement specific functionality. As the name suggests, it is - associated with the macro :c:data:`PyArray_DESCR`. + A pointer to a data-type descriptor object (see below). The + data-type descriptor object is an instance of a new built-in + type which allows a generic description of memory. There is a + descriptor structure for each data type supported. This + descriptor structure contains useful information about the type + as well as a pointer to a table of function pointers to + implement specific functionality. As the name suggests, it is + associated with the macro :c:data:`PyArray_DESCR`. -.. c:member:: int PyArrayObject.flags + .. c:member:: int flags - Pointed to by the macro :c:data:`PyArray_FLAGS`, this data member represents - the flags indicating how the memory pointed to by data is to be - interpreted. Possible flags are :c:data:`NPY_ARRAY_C_CONTIGUOUS`, - :c:data:`NPY_ARRAY_F_CONTIGUOUS`, :c:data:`NPY_ARRAY_OWNDATA`, - :c:data:`NPY_ARRAY_ALIGNED`, :c:data:`NPY_ARRAY_WRITEABLE`, - :c:data:`NPY_ARRAY_WRITEBACKIFCOPY`, and :c:data:`NPY_ARRAY_UPDATEIFCOPY`. + Pointed to by the macro :c:data:`PyArray_FLAGS`, this data member represents + the flags indicating how the memory pointed to by data is to be + interpreted. Possible flags are :c:data:`NPY_ARRAY_C_CONTIGUOUS`, + :c:data:`NPY_ARRAY_F_CONTIGUOUS`, :c:data:`NPY_ARRAY_OWNDATA`, + :c:data:`NPY_ARRAY_ALIGNED`, :c:data:`NPY_ARRAY_WRITEABLE`, + :c:data:`NPY_ARRAY_WRITEBACKIFCOPY`, and :c:data:`NPY_ARRAY_UPDATEIFCOPY`. -.. c:member:: PyObject *PyArrayObject.weakreflist + .. c:member:: PyObject *weakreflist - This member allows array objects to have weak references (using the - weakref module). + This member allows array objects to have weak references (using the + weakref module). PyArrayDescr_Type and PyArray_Descr @@ -226,197 +227,196 @@ PyArrayDescr_Type and PyArray_Descr npy_hash_t hash; } PyArray_Descr; -.. c:member:: PyTypeObject *PyArray_Descr.typeobj + .. c:member:: PyTypeObject *typeobj - Pointer to a typeobject that is the corresponding Python type for - the elements of this array. For the builtin types, this points to - the corresponding array scalar. For user-defined types, this - should point to a user-defined typeobject. This typeobject can - either inherit from array scalars or not. If it does not inherit - from array scalars, then the :c:data:`NPY_USE_GETITEM` and - :c:data:`NPY_USE_SETITEM` flags should be set in the ``flags`` member. + Pointer to a typeobject that is the corresponding Python type for + the elements of this array. For the builtin types, this points to + the corresponding array scalar. For user-defined types, this + should point to a user-defined typeobject. This typeobject can + either inherit from array scalars or not. If it does not inherit + from array scalars, then the :c:data:`NPY_USE_GETITEM` and + :c:data:`NPY_USE_SETITEM` flags should be set in the ``flags`` member. -.. c:member:: char PyArray_Descr.kind + .. c:member:: char kind - A character code indicating the kind of array (using the array - interface typestring notation). A 'b' represents Boolean, a 'i' - represents signed integer, a 'u' represents unsigned integer, 'f' - represents floating point, 'c' represents complex floating point, 'S' - represents 8-bit zero-terminated bytes, 'U' represents 32-bit/character - unicode string, and 'V' represents arbitrary. + A character code indicating the kind of array (using the array + interface typestring notation). A 'b' represents Boolean, a 'i' + represents signed integer, a 'u' represents unsigned integer, 'f' + represents floating point, 'c' represents complex floating point, 'S' + represents 8-bit zero-terminated bytes, 'U' represents 32-bit/character + unicode string, and 'V' represents arbitrary. -.. c:member:: char PyArray_Descr.type + .. c:member:: char type - A traditional character code indicating the data type. + A traditional character code indicating the data type. -.. c:member:: char PyArray_Descr.byteorder + .. c:member:: char byteorder - A character indicating the byte-order: '>' (big-endian), '<' (little- - endian), '=' (native), '\|' (irrelevant, ignore). All builtin data- - types have byteorder '='. + A character indicating the byte-order: '>' (big-endian), '<' (little- + endian), '=' (native), '\|' (irrelevant, ignore). All builtin data- + types have byteorder '='. -.. c:member:: char PyArray_Descr.flags + .. c:member:: char flags - A data-type bit-flag that determines if the data-type exhibits object- - array like behavior. Each bit in this member is a flag which are named - as: + A data-type bit-flag that determines if the data-type exhibits object- + array like behavior. Each bit in this member is a flag which are named + as: - .. c:macro:: NPY_ITEM_REFCOUNT + .. c:macro:: NPY_ITEM_REFCOUNT - Indicates that items of this data-type must be reference - counted (using :c:func:`Py_INCREF` and :c:func:`Py_DECREF` ). + Indicates that items of this data-type must be reference + counted (using :c:func:`Py_INCREF` and :c:func:`Py_DECREF` ). - .. c:macro:: NPY_ITEM_HASOBJECT + .. c:macro:: NPY_ITEM_HASOBJECT - Same as :c:data:`NPY_ITEM_REFCOUNT`. + Same as :c:data:`NPY_ITEM_REFCOUNT`. - .. c:macro:: NPY_LIST_PICKLE + .. c:macro:: NPY_LIST_PICKLE - Indicates arrays of this data-type must be converted to a list - before pickling. + Indicates arrays of this data-type must be converted to a list + before pickling. - .. c:macro:: NPY_ITEM_IS_POINTER + .. c:macro:: NPY_ITEM_IS_POINTER - Indicates the item is a pointer to some other data-type + Indicates the item is a pointer to some other data-type - .. c:macro:: NPY_NEEDS_INIT + .. c:macro:: NPY_NEEDS_INIT - Indicates memory for this data-type must be initialized (set - to 0) on creation. + Indicates memory for this data-type must be initialized (set + to 0) on creation. - .. c:macro:: NPY_NEEDS_PYAPI + .. c:macro:: NPY_NEEDS_PYAPI - Indicates this data-type requires the Python C-API during - access (so don't give up the GIL if array access is going to - be needed). + Indicates this data-type requires the Python C-API during + access (so don't give up the GIL if array access is going to + be needed). - .. c:macro:: NPY_USE_GETITEM + .. c:macro:: NPY_USE_GETITEM - On array access use the ``f->getitem`` function pointer - instead of the standard conversion to an array scalar. Must - use if you don't define an array scalar to go along with - the data-type. + On array access use the ``f->getitem`` function pointer + instead of the standard conversion to an array scalar. Must + use if you don't define an array scalar to go along with + the data-type. - .. c:macro:: NPY_USE_SETITEM + .. c:macro:: NPY_USE_SETITEM - When creating a 0-d array from an array scalar use - ``f->setitem`` instead of the standard copy from an array - scalar. Must use if you don't define an array scalar to go - along with the data-type. + When creating a 0-d array from an array scalar use + ``f->setitem`` instead of the standard copy from an array + scalar. Must use if you don't define an array scalar to go + along with the data-type. - .. c:macro:: NPY_FROM_FIELDS + .. c:macro:: NPY_FROM_FIELDS - The bits that are inherited for the parent data-type if these - bits are set in any field of the data-type. Currently ( - :c:data:`NPY_NEEDS_INIT` \| :c:data:`NPY_LIST_PICKLE` \| - :c:data:`NPY_ITEM_REFCOUNT` \| :c:data:`NPY_NEEDS_PYAPI` ). + The bits that are inherited for the parent data-type if these + bits are set in any field of the data-type. Currently ( + :c:data:`NPY_NEEDS_INIT` \| :c:data:`NPY_LIST_PICKLE` \| + :c:data:`NPY_ITEM_REFCOUNT` \| :c:data:`NPY_NEEDS_PYAPI` ). - .. c:macro:: NPY_OBJECT_DTYPE_FLAGS + .. c:macro:: NPY_OBJECT_DTYPE_FLAGS - Bits set for the object data-type: ( :c:data:`NPY_LIST_PICKLE` - \| :c:data:`NPY_USE_GETITEM` \| :c:data:`NPY_ITEM_IS_POINTER` \| - :c:data:`NPY_REFCOUNT` \| :c:data:`NPY_NEEDS_INIT` \| - :c:data:`NPY_NEEDS_PYAPI`). + Bits set for the object data-type: ( :c:data:`NPY_LIST_PICKLE` + \| :c:data:`NPY_USE_GETITEM` \| :c:data:`NPY_ITEM_IS_POINTER` \| + :c:data:`NPY_ITEM_REFCOUNT` \| :c:data:`NPY_NEEDS_INIT` \| + :c:data:`NPY_NEEDS_PYAPI`). - .. c:function:: PyDataType_FLAGCHK(PyArray_Descr *dtype, int flags) + .. c:function:: int PyDataType_FLAGCHK(PyArray_Descr *dtype, int flags) - Return true if all the given flags are set for the data-type - object. + Return true if all the given flags are set for the data-type + object. - .. c:function:: PyDataType_REFCHK(PyArray_Descr *dtype) + .. c:function:: int PyDataType_REFCHK(PyArray_Descr *dtype) - Equivalent to :c:func:`PyDataType_FLAGCHK` (*dtype*, - :c:data:`NPY_ITEM_REFCOUNT`). + Equivalent to :c:func:`PyDataType_FLAGCHK` (*dtype*, + :c:data:`NPY_ITEM_REFCOUNT`). -.. c:member:: int PyArray_Descr.type_num + .. c:member:: int type_num - A number that uniquely identifies the data type. For new data-types, - this number is assigned when the data-type is registered. + A number that uniquely identifies the data type. For new data-types, + this number is assigned when the data-type is registered. -.. c:member:: int PyArray_Descr.elsize + .. c:member:: int elsize - For data types that are always the same size (such as long), this - holds the size of the data type. For flexible data types where - different arrays can have a different elementsize, this should be - 0. + For data types that are always the same size (such as long), this + holds the size of the data type. For flexible data types where + different arrays can have a different elementsize, this should be + 0. -.. c:member:: int PyArray_Descr.alignment + .. c:member:: int alignment - A number providing alignment information for this data type. - Specifically, it shows how far from the start of a 2-element - structure (whose first element is a ``char`` ), the compiler - places an item of this type: ``offsetof(struct {char c; type v;}, - v)`` + A number providing alignment information for this data type. + Specifically, it shows how far from the start of a 2-element + structure (whose first element is a ``char`` ), the compiler + places an item of this type: ``offsetof(struct {char c; type v;}, + v)`` -.. c:member:: PyArray_ArrayDescr *PyArray_Descr.subarray + .. c:member:: PyArray_ArrayDescr *subarray - If this is non- ``NULL``, then this data-type descriptor is a - C-style contiguous array of another data-type descriptor. In - other-words, each element that this descriptor describes is - actually an array of some other base descriptor. This is most - useful as the data-type descriptor for a field in another - data-type descriptor. The fields member should be ``NULL`` if this - is non- ``NULL`` (the fields member of the base descriptor can be - non- ``NULL`` however). The :c:type:`PyArray_ArrayDescr` structure is - defined using + If this is non- ``NULL``, then this data-type descriptor is a + C-style contiguous array of another data-type descriptor. In + other-words, each element that this descriptor describes is + actually an array of some other base descriptor. This is most + useful as the data-type descriptor for a field in another + data-type descriptor. The fields member should be ``NULL`` if this + is non- ``NULL`` (the fields member of the base descriptor can be + non- ``NULL`` however). - .. code-block:: c - - typedef struct { - PyArray_Descr *base; - PyObject *shape; - } PyArray_ArrayDescr; + .. c:type:: PyArray_ArrayDescr - The elements of this structure are: + .. code-block:: c - .. c:member:: PyArray_Descr *PyArray_ArrayDescr.base + typedef struct { + PyArray_Descr *base; + PyObject *shape; + } PyArray_ArrayDescr; - The data-type-descriptor object of the base-type. + .. c:member:: PyArray_Descr *base - .. c:member:: PyObject *PyArray_ArrayDescr.shape + The data-type-descriptor object of the base-type. - The shape (always C-style contiguous) of the sub-array as a Python - tuple. + .. c:member:: PyObject *shape + The shape (always C-style contiguous) of the sub-array as a Python + tuple. -.. c:member:: PyObject *PyArray_Descr.fields + .. c:member:: PyObject *fields - If this is non-NULL, then this data-type-descriptor has fields - described by a Python dictionary whose keys are names (and also - titles if given) and whose values are tuples that describe the - fields. Recall that a data-type-descriptor always describes a - fixed-length set of bytes. A field is a named sub-region of that - total, fixed-length collection. A field is described by a tuple - composed of another data- type-descriptor and a byte - offset. Optionally, the tuple may contain a title which is - normally a Python string. These tuples are placed in this - dictionary keyed by name (and also title if given). + If this is non-NULL, then this data-type-descriptor has fields + described by a Python dictionary whose keys are names (and also + titles if given) and whose values are tuples that describe the + fields. Recall that a data-type-descriptor always describes a + fixed-length set of bytes. A field is a named sub-region of that + total, fixed-length collection. A field is described by a tuple + composed of another data- type-descriptor and a byte + offset. Optionally, the tuple may contain a title which is + normally a Python string. These tuples are placed in this + dictionary keyed by name (and also title if given). -.. c:member:: PyObject *PyArray_Descr.names + .. c:member:: PyObject *names - An ordered tuple of field names. It is NULL if no field is - defined. + An ordered tuple of field names. It is NULL if no field is + defined. -.. c:member:: PyArray_ArrFuncs *PyArray_Descr.f + .. c:member:: PyArray_ArrFuncs *f - A pointer to a structure containing functions that the type needs - to implement internal features. These functions are not the same - thing as the universal functions (ufuncs) described later. Their - signatures can vary arbitrarily. + A pointer to a structure containing functions that the type needs + to implement internal features. These functions are not the same + thing as the universal functions (ufuncs) described later. Their + signatures can vary arbitrarily. -.. c:member:: PyObject *PyArray_Descr.metadata + .. c:member:: PyObject *metadata - Metadata about this dtype. + Metadata about this dtype. -.. c:member:: NpyAuxData *PyArray_Descr.c_metadata + .. c:member:: NpyAuxData *c_metadata - Metadata specific to the C implementation - of the particular dtype. Added for NumPy 1.7.0. + Metadata specific to the C implementation + of the particular dtype. Added for NumPy 1.7.0. -.. c:member:: Npy_hash_t *PyArray_Descr.hash + .. c:type:: npy_hash_t + .. c:member:: npy_hash_t *hash - Currently unused. Reserved for future use in caching - hash values. + Currently unused. Reserved for future use in caching + hash values. .. c:type:: PyArray_ArrFuncs @@ -568,7 +568,7 @@ PyArrayDescr_Type and PyArray_Descr This function should be called without holding the Python GIL, and has to grab it for error reporting. - .. c:member:: Bool nonzero(void* data, void* arr) + .. c:member:: npy_bool nonzero(void* data, void* arr) A pointer to a function that returns TRUE if the item of ``arr`` pointed to by ``data`` is nonzero. This function can @@ -612,7 +612,8 @@ PyArrayDescr_Type and PyArray_Descr Either ``NULL`` or a dictionary containing low-level casting functions for user- defined data-types. Each function is - wrapped in a :c:type:`PyCObject *` and keyed by the data-type number. + wrapped in a :c:type:`PyCapsule *` and keyed by + the data-type number. .. c:member:: NPY_SCALARKIND scalarkind(PyArrayObject* arr) @@ -791,35 +792,37 @@ PyUFunc_Type and PyUFuncObject npy_uint32 *iter_flags; /* new in API version 0x0000000D */ npy_intp *core_dim_sizes; - npy_intp *core_dim_flags; - + npy_uint32 *core_dim_flags; + PyObject *identity_value; } PyUFuncObject; - .. c:macro: PyUFuncObject.PyObject_HEAD + .. c:macro: PyObject_HEAD required for all Python objects. - .. c:member:: int PyUFuncObject.nin + .. c:member:: int nin The number of input arguments. - .. c:member:: int PyUFuncObject.nout + .. c:member:: int nout The number of output arguments. - .. c:member:: int PyUFuncObject.nargs + .. c:member:: int nargs The total number of arguments (*nin* + *nout*). This must be less than :c:data:`NPY_MAXARGS`. - .. c:member:: int PyUFuncObject.identity + .. c:member:: int identity Either :c:data:`PyUFunc_One`, :c:data:`PyUFunc_Zero`, - :c:data:`PyUFunc_None` or :c:data:`PyUFunc_AllOnes` to indicate + :c:data:`PyUFunc_MinusOne`, :c:data:`PyUFunc_None`, + :c:data:`PyUFunc_ReorderableNone`, or + :c:data:`PyUFunc_IdentityValue` to indicate the identity for this operation. It is only used for a reduce-like call on an empty array. - .. c:member:: void PyUFuncObject.functions( \ + .. c:member:: void functions( \ char** args, npy_intp* dims, npy_intp* steps, void* extradata) An array of function pointers --- one for each data type @@ -837,7 +840,7 @@ PyUFunc_Type and PyUFuncObject passed in as *extradata*. The size of this function pointer array is ntypes. - .. c:member:: void **PyUFuncObject.data + .. c:member:: void **data Extra data to be passed to the 1-d vector loops or ``NULL`` if no extra-data is needed. This C-array must be the same size ( @@ -846,22 +849,22 @@ PyUFunc_Type and PyUFuncObject just 1-d vector loops that make use of this extra data to receive a pointer to the actual function to call. - .. c:member:: int PyUFuncObject.ntypes + .. c:member:: int ntypes The number of supported data types for the ufunc. This number specifies how many different 1-d loops (of the builtin data types) are available. - .. c:member:: int PyUFuncObject.reserved1 + .. c:member:: int reserved1 Unused. - .. c:member:: char *PyUFuncObject.name + .. c:member:: char *name A string name for the ufunc. This is used dynamically to build the __doc\__ attribute of ufuncs. - .. c:member:: char *PyUFuncObject.types + .. c:member:: char *types An array of :math:`nargs \times ntypes` 8-bit type_numbers which contains the type signature for the function for each of @@ -871,24 +874,24 @@ PyUFunc_Type and PyUFuncObject vector loop. These type numbers do not have to be the same type and mixed-type ufuncs are supported. - .. c:member:: char *PyUFuncObject.doc + .. c:member:: char *doc Documentation for the ufunc. Should not contain the function signature as this is generated dynamically when __doc\__ is retrieved. - .. c:member:: void *PyUFuncObject.ptr + .. c:member:: void *ptr Any dynamically allocated memory. Currently, this is used for dynamic ufuncs created from a python function to store room for the types, data, and name members. - .. c:member:: PyObject *PyUFuncObject.obj + .. c:member:: PyObject *obj For ufuncs dynamically created from python functions, this member holds a reference to the underlying Python function. - .. c:member:: PyObject *PyUFuncObject.userloops + .. c:member:: PyObject *userloops A dictionary of user-defined 1-d vector loops (stored as CObject ptrs) for user-defined types. A loop may be registered by the @@ -896,74 +899,85 @@ PyUFunc_Type and PyUFuncObject User defined type numbers are always larger than :c:data:`NPY_USERDEF`. - .. c:member:: int PyUFuncObject.core_enabled + .. c:member:: int core_enabled 0 for scalar ufuncs; 1 for generalized ufuncs - .. c:member:: int PyUFuncObject.core_num_dim_ix + .. c:member:: int core_num_dim_ix Number of distinct core dimension names in the signature - .. c:member:: int *PyUFuncObject.core_num_dims + .. c:member:: int *core_num_dims Number of core dimensions of each argument - .. c:member:: int *PyUFuncObject.core_dim_ixs + .. c:member:: int *core_dim_ixs Dimension indices in a flattened form; indices of argument ``k`` are stored in ``core_dim_ixs[core_offsets[k] : core_offsets[k] + core_numdims[k]]`` - .. c:member:: int *PyUFuncObject.core_offsets + .. c:member:: int *core_offsets Position of 1st core dimension of each argument in ``core_dim_ixs``, equivalent to cumsum(``core_num_dims``) - .. c:member:: char *PyUFuncObject.core_signature + .. c:member:: char *core_signature Core signature string - .. c:member:: PyUFunc_TypeResolutionFunc *PyUFuncObject.type_resolver + .. c:member:: PyUFunc_TypeResolutionFunc *type_resolver A function which resolves the types and fills an array with the dtypes for the inputs and outputs - .. c:member:: PyUFunc_LegacyInnerLoopSelectionFunc *PyUFuncObject.legacy_inner_loop_selector + .. c:member:: PyUFunc_LegacyInnerLoopSelectionFunc *legacy_inner_loop_selector A function which returns an inner loop. The ``legacy`` in the name arises because for NumPy 1.6 a better variant had been planned. This variant has not yet come about. - .. c:member:: void *PyUFuncObject.reserved2 + .. c:member:: void *reserved2 For a possible future loop selector with a different signature. - .. c:member:: PyUFunc_MaskedInnerLoopSelectionFunc *PyUFuncObject.masked_inner_loop_selector + .. c:member:: PyUFunc_MaskedInnerLoopSelectionFunc *masked_inner_loop_selector Function which returns a masked inner loop for the ufunc - .. c:member:: npy_uint32 PyUFuncObject.op_flags + .. c:member:: npy_uint32 op_flags Override the default operand flags for each ufunc operand. - .. c:member:: npy_uint32 PyUFuncObject.iter_flags + .. c:member:: npy_uint32 iter_flags Override the default nditer flags for the ufunc. Added in API version 0x0000000D - .. c:member:: npy_intp *PyUFuncObject.core_dim_sizes + .. c:member:: npy_intp *core_dim_sizes For each distinct core dimension, the possible - :ref:`frozen ` size if :c:data:`UFUNC_CORE_DIM_SIZE_INFERRED` is 0 + :ref:`frozen ` size if + :c:data:`UFUNC_CORE_DIM_SIZE_INFERRED` is ``0`` - .. c:member:: npy_uint32 *PyUFuncObject.core_dim_flags + .. c:member:: npy_uint32 *core_dim_flags For each distinct core dimension, a set of ``UFUNC_CORE_DIM*`` flags - - :c:data:`UFUNC_CORE_DIM_CAN_IGNORE` if the dim name ends in ``?`` - - :c:data:`UFUNC_CORE_DIM_SIZE_INFERRED` if the dim size will be - determined from the operands and not from a :ref:`frozen ` signature + .. c:macro:: UFUNC_CORE_DIM_CAN_IGNORE + + if the dim name ends in ``?`` + + .. c:macro:: UFUNC_CORE_DIM_SIZE_INFERRED + + if the dim size will be determined from the operands + and not from a :ref:`frozen ` signature + + .. c:member:: PyObject *identity_value + + Identity for reduction, when :c:member:`PyUFuncObject.identity` + is equal to :c:data:`PyUFunc_IdentityValue`. PyArrayIter_Type and PyArrayIterObject -------------------------------------- @@ -1009,57 +1023,57 @@ PyArrayIter_Type and PyArrayIterObject npy_intp factors[NPY_MAXDIMS]; PyArrayObject *ao; char *dataptr; - Bool contiguous; + npy_bool contiguous; } PyArrayIterObject; - .. c:member:: int PyArrayIterObject.nd_m1 + .. c:member:: int nd_m1 :math:`N-1` where :math:`N` is the number of dimensions in the underlying array. - .. c:member:: npy_intp PyArrayIterObject.index + .. c:member:: npy_intp index The current 1-d index into the array. - .. c:member:: npy_intp PyArrayIterObject.size + .. c:member:: npy_intp size The total size of the underlying array. - .. c:member:: npy_intp *PyArrayIterObject.coordinates + .. c:member:: npy_intp *coordinates An :math:`N` -dimensional index into the array. - .. c:member:: npy_intp *PyArrayIterObject.dims_m1 + .. c:member:: npy_intp *dims_m1 The size of the array minus 1 in each dimension. - .. c:member:: npy_intp *PyArrayIterObject.strides + .. c:member:: npy_intp *strides The strides of the array. How many bytes needed to jump to the next element in each dimension. - .. c:member:: npy_intp *PyArrayIterObject.backstrides + .. c:member:: npy_intp *backstrides How many bytes needed to jump from the end of a dimension back to its beginning. Note that ``backstrides[k] == strides[k] * dims_m1[k]``, but it is stored here as an optimization. - .. c:member:: npy_intp *PyArrayIterObject.factors + .. c:member:: npy_intp *factors This array is used in computing an N-d index from a 1-d index. It contains needed products of the dimensions. - .. c:member:: PyArrayObject *PyArrayIterObject.ao + .. c:member:: PyArrayObject *ao A pointer to the underlying ndarray this iterator was created to represent. - .. c:member:: char *PyArrayIterObject.dataptr + .. c:member:: char *dataptr This member points to an element in the ndarray indicated by the index. - .. c:member:: Bool PyArrayIterObject.contiguous + .. c:member:: npy_bool contiguous This flag is true if the underlying array is :c:data:`NPY_ARRAY_C_CONTIGUOUS`. It is used to simplify @@ -1106,32 +1120,32 @@ PyArrayMultiIter_Type and PyArrayMultiIterObject PyArrayIterObject *iters[NPY_MAXDIMS]; } PyArrayMultiIterObject; - .. c:macro: PyArrayMultiIterObject.PyObject_HEAD + .. c:macro: PyObject_HEAD Needed at the start of every Python object (holds reference count and type identification). - .. c:member:: int PyArrayMultiIterObject.numiter + .. c:member:: int numiter The number of arrays that need to be broadcast to the same shape. - .. c:member:: npy_intp PyArrayMultiIterObject.size + .. c:member:: npy_intp size The total broadcasted size. - .. c:member:: npy_intp PyArrayMultiIterObject.index + .. c:member:: npy_intp index The current (1-d) index into the broadcasted result. - .. c:member:: int PyArrayMultiIterObject.nd + .. c:member:: int nd The number of dimensions in the broadcasted result. - .. c:member:: npy_intp *PyArrayMultiIterObject.dimensions + .. c:member:: npy_intp *dimensions The shape of the broadcasted result (only ``nd`` slots are used). - .. c:member:: PyArrayIterObject **PyArrayMultiIterObject.iters + .. c:member:: PyArrayIterObject **iters An array of iterator objects that holds the iterators for the arrays to be broadcast together. On return, the iterators are @@ -1213,8 +1227,8 @@ are ``Py{TYPE}ArrType_Type`` where ``{TYPE}`` can be **Object**. These type names are part of the C-API and can therefore be created in -extension C-code. There is also a :c:data:`PyIntpArrType_Type` and a -:c:data:`PyUIntpArrType_Type` that are simple substitutes for one of the +extension C-code. There is also a ``PyIntpArrType_Type`` and a +``PyUIntpArrType_Type`` that are simple substitutes for one of the integer types that can hold a pointer on the platform. The structure of these scalar objects is not exposed to C-code. The function :c:func:`PyArray_ScalarAsCtype` (..) can be used to extract the C-type @@ -1249,12 +1263,12 @@ PyArray_Dims The members of this structure are - .. c:member:: npy_intp *PyArray_Dims.ptr + .. c:member:: npy_intp *ptr A pointer to a list of (:c:type:`npy_intp`) integers which usually represent array shape or array strides. - .. c:member:: int PyArray_Dims.len + .. c:member:: int len The length of the list of integers. It is assumed safe to access *ptr* [0] to *ptr* [len-1]. @@ -1283,26 +1297,26 @@ PyArray_Chunk The members are - .. c:macro: PyArray_Chunk.PyObject_HEAD + .. c:macro: PyObject_HEAD Necessary for all Python objects. Included here so that the :c:type:`PyArray_Chunk` structure matches that of the buffer object (at least to the len member). - .. c:member:: PyObject *PyArray_Chunk.base + .. c:member:: PyObject *base The Python object this chunk of memory comes from. Needed so that memory can be accounted for properly. - .. c:member:: void *PyArray_Chunk.ptr + .. c:member:: void *ptr A pointer to the start of the single-segment chunk of memory. - .. c:member:: npy_intp PyArray_Chunk.len + .. c:member:: npy_intp len The length of the segment in bytes. - .. c:member:: int PyArray_Chunk.flags + .. c:member:: int flags Any data flags (*e.g.* :c:data:`NPY_ARRAY_WRITEABLE` ) that should be used to interpret the memory. @@ -1317,13 +1331,13 @@ PyArrayInterface The :c:type:`PyArrayInterface` structure is defined so that NumPy and other extension modules can use the rapid array interface - protocol. The :obj:`__array_struct__` method of an object that + protocol. The :obj:`~object.__array_struct__` method of an object that supports the rapid array interface protocol should return a - :c:type:`PyCObject` that contains a pointer to a :c:type:`PyArrayInterface` + :c:type:`PyCapsule` that contains a pointer to a :c:type:`PyArrayInterface` structure with the relevant details of the array. After the new array is created, the attribute should be ``DECREF``'d which will free the :c:type:`PyArrayInterface` structure. Remember to ``INCREF`` the - object (whose :obj:`__array_struct__` attribute was retrieved) and + object (whose :obj:`~object.__array_struct__` attribute was retrieved) and point the base member of the new :c:type:`PyArrayObject` to this same object. In this way the memory for the array will be managed correctly. @@ -1342,15 +1356,15 @@ PyArrayInterface PyObject *descr; } PyArrayInterface; - .. c:member:: int PyArrayInterface.two + .. c:member:: int two the integer 2 as a sanity check. - .. c:member:: int PyArrayInterface.nd + .. c:member:: int nd the number of dimensions in the array. - .. c:member:: char PyArrayInterface.typekind + .. c:member:: char typekind A character indicating what kind of array is present according to the typestring convention with 't' -> bitfield, 'b' -> Boolean, 'i' -> @@ -1358,11 +1372,11 @@ PyArrayInterface complex floating point, 'O' -> object, 'S' -> (byte-)string, 'U' -> unicode, 'V' -> void. - .. c:member:: int PyArrayInterface.itemsize + .. c:member:: int itemsize The number of bytes each item in the array requires. - .. c:member:: int PyArrayInterface.flags + .. c:member:: int flags Any of the bits :c:data:`NPY_ARRAY_C_CONTIGUOUS` (1), :c:data:`NPY_ARRAY_F_CONTIGUOUS` (2), :c:data:`NPY_ARRAY_ALIGNED` (0x100), @@ -1376,26 +1390,26 @@ PyArrayInterface structure is present (it will be ignored by objects consuming version 2 of the array interface). - .. c:member:: npy_intp *PyArrayInterface.shape + .. c:member:: npy_intp *shape An array containing the size of the array in each dimension. - .. c:member:: npy_intp *PyArrayInterface.strides + .. c:member:: npy_intp *strides An array containing the number of bytes to jump to get to the next element in each dimension. - .. c:member:: void *PyArrayInterface.data + .. c:member:: void *data A pointer *to* the first element of the array. - .. c:member:: PyObject *PyArrayInterface.descr + .. c:member:: PyObject *descr A Python object describing the data-type in more detail (same - as the *descr* key in :obj:`__array_interface__`). This can be + as the *descr* key in :obj:`~object.__array_interface__`). This can be ``NULL`` if *typekind* and *itemsize* provide enough information. This field is also ignored unless - :c:data:`ARR_HAS_DESCR` flag is on in *flags*. + :c:data:`NPY_ARR_HAS_DESCR` flag is on in *flags*. Internally used structures @@ -1433,7 +1447,7 @@ for completeness and assistance in understanding the code. Advanced indexing is handled with this Python type. It is simply a loose wrapper around the C-structure containing the variables needed for advanced array indexing. The associated C-structure, - :c:type:`PyArrayMapIterObject`, is useful if you are trying to + ``PyArrayMapIterObject``, is useful if you are trying to understand the advanced-index mapping code. It is defined in the ``arrayobject.h`` header. This type is not exposed to Python and could be replaced with a C-structure. As a Python type it takes diff --git a/doc/source/reference/c-api/ufunc.rst b/doc/source/reference/c-api/ufunc.rst index 50963c81f626..9eb70c3fbe34 100644 --- a/doc/source/reference/c-api/ufunc.rst +++ b/doc/source/reference/c-api/ufunc.rst @@ -10,17 +10,41 @@ UFunc API Constants --------- -.. c:var:: UFUNC_ERR_{HANDLER} +``UFUNC_ERR_{HANDLER}`` + .. c:macro:: UFUNC_ERR_IGNORE - ``{HANDLER}`` can be **IGNORE**, **WARN**, **RAISE**, or **CALL** + .. c:macro:: UFUNC_ERR_WARN -.. c:var:: UFUNC_{THING}_{ERR} + .. c:macro:: UFUNC_ERR_RAISE - ``{THING}`` can be **MASK**, **SHIFT**, or **FPE**, and ``{ERR}`` can - be **DIVIDEBYZERO**, **OVERFLOW**, **UNDERFLOW**, and **INVALID**. + .. c:macro:: UFUNC_ERR_CALL -.. c:var:: PyUFunc_{VALUE} +``UFUNC_{THING}_{ERR}`` + .. c:macro:: UFUNC_MASK_DIVIDEBYZERO + .. c:macro:: UFUNC_MASK_OVERFLOW + + .. c:macro:: UFUNC_MASK_UNDERFLOW + + .. c:macro:: UFUNC_MASK_INVALID + + .. c:macro:: UFUNC_SHIFT_DIVIDEBYZERO + + .. c:macro:: UFUNC_SHIFT_OVERFLOW + + .. c:macro:: UFUNC_SHIFT_UNDERFLOW + + .. c:macro:: UFUNC_SHIFT_INVALID + + .. c:macro:: UFUNC_FPE_DIVIDEBYZERO + + .. c:macro:: UFUNC_FPE_OVERFLOW + + .. c:macro:: UFUNC_FPE_UNDERFLOW + + .. c:macro:: UFUNC_FPE_INVALID + +``PyUFunc_{VALUE}`` .. c:macro:: PyUFunc_One .. c:macro:: PyUFunc_Zero @@ -50,6 +74,66 @@ Macros was released (because loop->obj was not true). +Types +----- + +.. c:type:: PyUFuncGenericFunction + + pointers to functions that actually implement the underlying + (element-by-element) function :math:`N` times with the following + signature: + + .. c:function:: void loopfunc(\ + char** args, npy_intp const *dimensions, npy_intp const *steps, void* data) + + *args* + + An array of pointers to the actual data for the input and output + arrays. The input arguments are given first followed by the output + arguments. + + *dimensions* + + A pointer to the size of the dimension over which this function is + looping. + + *steps* + + A pointer to the number of bytes to jump to get to the + next element in this dimension for each of the input and + output arguments. + + *data* + + Arbitrary data (extra arguments, function names, *etc.* ) + that can be stored with the ufunc and will be passed in + when it is called. + + This is an example of a func specialized for addition of doubles + returning doubles. + + .. code-block:: c + + static void + double_add(char **args, + npy_intp const *dimensions, + npy_intp const *steps, + void *extra) + { + npy_intp i; + npy_intp is1 = steps[0], is2 = steps[1]; + npy_intp os = steps[2], n = dimensions[0]; + char *i1 = args[0], *i2 = args[1], *op = args[2]; + for (i = 0; i < n; i++) { + *((double *)op) = *((double *)i1) + + *((double *)i2); + i1 += is1; + i2 += is2; + op += os; + } + } + + Functions --------- @@ -71,60 +155,7 @@ Functions :param func: Must to an array of length *ntypes* containing - :c:type:`PyUFuncGenericFunction` items. These items are pointers to - functions that actually implement the underlying - (element-by-element) function :math:`N` times with the following - signature: - - .. c:function:: void loopfunc( - char** args, npy_intp const *dimensions, npy_intp const *steps, void* data) - - *args* - - An array of pointers to the actual data for the input and output - arrays. The input arguments are given first followed by the output - arguments. - - *dimensions* - - A pointer to the size of the dimension over which this function is - looping. - - *steps* - - A pointer to the number of bytes to jump to get to the - next element in this dimension for each of the input and - output arguments. - - *data* - - Arbitrary data (extra arguments, function names, *etc.* ) - that can be stored with the ufunc and will be passed in - when it is called. - - This is an example of a func specialized for addition of doubles - returning doubles. - - .. code-block:: c - - static void - double_add(char **args, - npy_intp const *dimensions, - npy_intp const *steps, - void *extra) - { - npy_intp i; - npy_intp is1 = steps[0], is2 = steps[1]; - npy_intp os = steps[2], n = dimensions[0]; - char *i1 = args[0], *i2 = args[1], *op = args[2]; - for (i = 0; i < n; i++) { - *((double *)op) = *((double *)i1) + - *((double *)i2); - i1 += is1; - i2 += is2; - op += os; - } - } + :c:type:`PyUFuncGenericFunction` items. :param data: Should be ``NULL`` or a pointer to an array of size *ntypes* diff --git a/doc/source/reference/internals.code-explanations.rst b/doc/source/reference/internals.code-explanations.rst index 65553e07e6b2..e8e428f2ec6b 100644 --- a/doc/source/reference/internals.code-explanations.rst +++ b/doc/source/reference/internals.code-explanations.rst @@ -147,7 +147,8 @@ an iterator for each of the arrays being broadcast. The :c:func:`PyArray_Broadcast` function takes the iterators that have already been defined and uses them to determine the broadcast shape in each dimension (to create the iterators at the same time that broadcasting -occurs then use the :c:func:`PyMultiIter_New` function). Then, the iterators are +occurs then use the :c:func:`PyArray_MultiIterNew` function). +Then, the iterators are adjusted so that each iterator thinks it is iterating over an array with the broadcast size. This is done by adjusting the iterators number of dimensions, and the shape in each dimension. This works @@ -162,7 +163,7 @@ for the extended dimensions. It is done in exactly the same way in NumPy. The big difference is that now the array of strides is kept track of in a :c:type:`PyArrayIterObject`, the iterators involved in a broadcast result are kept track of in a :c:type:`PyArrayMultiIterObject`, -and the :c:func:`PyArray_BroadCast` call implements the broad-casting rules. +and the :c:func:`PyArray_Broadcast` call implements the broad-casting rules. Array Scalars @@ -368,7 +369,7 @@ The output arguments (if any) are then processed and any missing return arrays are constructed. If any provided output array doesn't have the correct type (or is mis-aligned) and is smaller than the buffer size, then a new output array is constructed with the special -:c:data:`WRITEBACKIFCOPY` flag set. At the end of the function, +:c:data:`NPY_ARRAY_WRITEBACKIFCOPY` flag set. At the end of the function, :c:func:`PyArray_ResolveWritebackIfCopy` is called so that its contents will be copied back into the output array. Iterators for the output arguments are then processed. diff --git a/doc/source/reference/maskedarray.baseclass.rst b/doc/source/reference/maskedarray.baseclass.rst index 5c1bdda239c6..5a0f99651c3f 100644 --- a/doc/source/reference/maskedarray.baseclass.rst +++ b/doc/source/reference/maskedarray.baseclass.rst @@ -242,8 +242,8 @@ Comparison operators: MaskedArray.__eq__ MaskedArray.__ne__ -Truth value of an array (:func:`bool()`): -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Truth value of an array (:class:`bool() `): +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .. autosummary:: :toctree: generated/ diff --git a/doc/source/reference/maskedarray.generic.rst b/doc/source/reference/maskedarray.generic.rst index 41c3ee564809..d3849c50deec 100644 --- a/doc/source/reference/maskedarray.generic.rst +++ b/doc/source/reference/maskedarray.generic.rst @@ -177,8 +177,8 @@ attribute. We must keep in mind that a ``True`` entry in the mask indicates an *invalid* data. Another possibility is to use the :func:`getmask` and :func:`getmaskarray` -functions. :func:`getmask(x)` outputs the mask of ``x`` if ``x`` is a masked -array, and the special value :data:`nomask` otherwise. :func:`getmaskarray(x)` +functions. ``getmask(x)`` outputs the mask of ``x`` if ``x`` is a masked +array, and the special value :data:`nomask` otherwise. ``getmaskarray(x)`` outputs the mask of ``x`` if ``x`` is a masked array. If ``x`` has no invalid entry or is not a masked array, the function outputs a boolean array of ``False`` with as many elements as ``x``. @@ -296,11 +296,11 @@ new valid values to them:: .. note:: Unmasking an entry by direct assignment will silently fail if the masked - array has a *hard* mask, as shown by the :attr:`hardmask` attribute. This - feature was introduced to prevent overwriting the mask. To force the - unmasking of an entry where the array has a hard mask, the mask must first - to be softened using the :meth:`soften_mask` method before the allocation. - It can be re-hardened with :meth:`harden_mask`:: + array has a *hard* mask, as shown by the :attr:`~MaskedArray.hardmask` + attribute. This feature was introduced to prevent overwriting the mask. + To force the unmasking of an entry where the array has a hard mask, + the mask must first to be softened using the :meth:`soften_mask` method + before the allocation. It can be re-hardened with :meth:`harden_mask`:: >>> x = ma.array([1, 2, 3], mask=[0, 0, 1], hard_mask=True) >>> x @@ -406,8 +406,8 @@ Operations on masked arrays Arithmetic and comparison operations are supported by masked arrays. As much as possible, invalid entries of a masked array are not processed, -meaning that the corresponding :attr:`data` entries *should* be the same -before and after the operation. +meaning that the corresponding :attr:`~MaskedArray.data` entries +*should* be the same before and after the operation. .. warning:: We need to stress that this behavior may not be systematic, that masked diff --git a/doc/source/reference/random/c-api.rst b/doc/source/reference/random/c-api.rst index 63b0fdc2b420..a79da7a492f7 100644 --- a/doc/source/reference/random/c-api.rst +++ b/doc/source/reference/random/c-api.rst @@ -181,6 +181,5 @@ Generate a single integer Generate random uint64 numbers in closed interval [off, off + rng]. -.. c:function:: npy_uint64 random_bounded_uint64(bitgen_t *bitgen_state, npy_uint64 off, npy_uint64 rng, npy_uint64 mask, bint use_masked) - +.. c:function:: npy_uint64 random_bounded_uint64(bitgen_t *bitgen_state, npy_uint64 off, npy_uint64 rng, npy_uint64 mask, bool use_masked) diff --git a/doc/source/reference/routines.array-manipulation.rst b/doc/source/reference/routines.array-manipulation.rst index 8d13a1800abf..1c96495d96f7 100644 --- a/doc/source/reference/routines.array-manipulation.rst +++ b/doc/source/reference/routines.array-manipulation.rst @@ -74,6 +74,7 @@ Joining arrays hstack dstack column_stack + row_stack Splitting arrays ================ diff --git a/doc/source/reference/routines.char.rst b/doc/source/reference/routines.char.rst index ed8393855763..90df14125b45 100644 --- a/doc/source/reference/routines.char.rst +++ b/doc/source/reference/routines.char.rst @@ -6,7 +6,7 @@ String operations .. module:: numpy.char The `numpy.char` module provides a set of vectorized string -operations for arrays of type `numpy.string_` or `numpy.unicode_`. +operations for arrays of type `numpy.str_` or `numpy.bytes_`. All of them are based on the string methods in the Python standard library. String operations diff --git a/doc/source/reference/routines.io.rst b/doc/source/reference/routines.io.rst index 2e119af9a111..3052ee1fbee8 100644 --- a/doc/source/reference/routines.io.rst +++ b/doc/source/reference/routines.io.rst @@ -88,4 +88,4 @@ Binary Format Description .. autosummary:: :toctree: generated/ - lib.format + lib.format diff --git a/doc/source/reference/routines.ma.rst b/doc/source/reference/routines.ma.rst index 97859ac67a24..18248fe099e5 100644 --- a/doc/source/reference/routines.ma.rst +++ b/doc/source/reference/routines.ma.rst @@ -272,7 +272,7 @@ Filling a masked array ma.common_fill_value ma.default_fill_value ma.maximum_fill_value - ma.maximum_fill_value + ma.minimum_fill_value ma.set_fill_value ma.MaskedArray.get_fill_value diff --git a/doc/source/reference/routines.other.rst b/doc/source/reference/routines.other.rst index def5b3e3c9f5..aefd680bbcd1 100644 --- a/doc/source/reference/routines.other.rst +++ b/doc/source/reference/routines.other.rst @@ -47,6 +47,7 @@ Utility show_config deprecate deprecate_with_doc + broadcast_shapes Matlab-like Functions --------------------- diff --git a/doc/source/reference/routines.set.rst b/doc/source/reference/routines.set.rst index b12d3d5f5e79..149c33a8b610 100644 --- a/doc/source/reference/routines.set.rst +++ b/doc/source/reference/routines.set.rst @@ -3,6 +3,11 @@ Set routines .. currentmodule:: numpy +.. autosummary:: + :toctree: generated/ + + lib.arraysetops + Making proper sets ------------------ .. autosummary:: diff --git a/doc/source/reference/ufuncs.rst b/doc/source/reference/ufuncs.rst index c729f8d37e08..06fbe28ddcba 100644 --- a/doc/source/reference/ufuncs.rst +++ b/doc/source/reference/ufuncs.rst @@ -1,5 +1,7 @@ .. sectionauthor:: adapted from "Guide to NumPy" by Travis E. Oliphant +.. currentmodule:: numpy + .. _ufuncs: ************************************ @@ -8,8 +10,6 @@ Universal functions (:class:`ufunc`) .. note: XXX: section might need to be made more reference-guideish... -.. currentmodule:: numpy - .. index: ufunc, universal function, arithmetic, operation A universal function (or :term:`ufunc` for short) is a function that diff --git a/doc/source/release.rst b/doc/source/release.rst index 1c77755fd1af..3ef1b06bd34a 100644 --- a/doc/source/release.rst +++ b/doc/source/release.rst @@ -6,6 +6,8 @@ Release Notes :maxdepth: 3 1.20.0 + 1.19.4 + 1.19.3 1.19.2 1.19.1 1.19.0 diff --git a/doc/source/release/1.16.0-notes.rst b/doc/source/release/1.16.0-notes.rst index e78e270f4a55..17d24160adb4 100644 --- a/doc/source/release/1.16.0-notes.rst +++ b/doc/source/release/1.16.0-notes.rst @@ -170,8 +170,8 @@ See the "accessing multiple fields" section of the C API changes ============= -The :c:data:`NPY_API_VERSION` was incremented to 0x0000D, due to the addition -of: +The :c:data:`NPY_FEATURE_VERSION` was incremented to 0x0000D, due to +the addition of: * :c:member:`PyUFuncObject.core_dim_flags` * :c:member:`PyUFuncObject.core_dim_sizes` diff --git a/doc/source/release/1.17.0-notes.rst b/doc/source/release/1.17.0-notes.rst index a93eb21863e5..4bdc6105fc1b 100644 --- a/doc/source/release/1.17.0-notes.rst +++ b/doc/source/release/1.17.0-notes.rst @@ -171,15 +171,15 @@ The functions `load`, and ``lib.format.read_array`` take an `CVE-2019-6446 `_. -.. currentmodule:: numpy.random.mtrand +.. currentmodule:: numpy.random Potential changes to the random stream in old random module ----------------------------------------------------------- Due to bugs in the application of ``log`` to random floating point numbers, the stream may change when sampling from `~RandomState.beta`, `~RandomState.binomial`, `~RandomState.laplace`, `~RandomState.logistic`, `~RandomState.logseries` or -`~RandomState.multinomial` if a ``0`` is generated in the underlying `MT19937 -<~numpy.random.mt11937.MT19937>` random stream. There is a ``1`` in +`~RandomState.multinomial` if a ``0`` is generated in the underlying `MT19937` +random stream. There is a ``1`` in :math:`10^{53}` chance of this occurring, so the probability that the stream changes for any given seed is extremely small. If a ``0`` is encountered in the underlying generator, then the incorrect value produced (either `numpy.inf` or @@ -559,4 +559,3 @@ Structured arrays indexed with non-existent fields raise ``KeyError`` not ``Valu ---------------------------------------------------------------------------------------- ``arr['bad_field']`` on a structured type raises ``KeyError``, for consistency with ``dict['bad_field']``. - diff --git a/doc/source/release/1.19.3-notes.rst b/doc/source/release/1.19.3-notes.rst new file mode 100644 index 000000000000..f1f1fd2b3e2f --- /dev/null +++ b/doc/source/release/1.19.3-notes.rst @@ -0,0 +1,46 @@ +.. currentmodule:: numpy + +========================== +NumPy 1.19.3 Release Notes +========================== + +NumPy 1.19.3 is a small maintenance release with two major improvements: + +- Python 3.9 binary wheels on all supported platforms. +- OpenBLAS fixes for Windows 10 version 2004 fmod bug. + +This release supports Python 3.6-3.9 and is linked with OpenBLAS 0.3.12 to avoid +some of the fmod problems on Windows 10 version 2004. Microsoft is aware of the +problem and users should upgrade when the fix becomes available, the fix here +is limited in scope. + +Contributors +============ + +A total of 8 people contributed to this release. People with a "+" by their +names contributed a patch for the first time. + +* Charles Harris +* Chris Brown + +* Daniel Vanzo + +* E. Madison Bray + +* Hugo van Kemenade + +* Ralf Gommers +* Sebastian Berg +* @danbeibei + + +Pull requests merged +==================== + +A total of 10 pull requests were merged for this release. + +* `#17298 `__: BLD: set upper versions for build dependencies +* `#17336 `__: BUG: Set deprecated fields to null in PyArray_InitArrFuncs +* `#17446 `__: ENH: Warn on unsupported Python 3.10+ +* `#17450 `__: MAINT: Update test_requirements.txt. +* `#17522 `__: ENH: Support for the NVIDIA HPC SDK nvfortran compiler +* `#17568 `__: BUG: Cygwin Workaround for #14787 on affected platforms +* `#17647 `__: BUG: Fix memory leak of buffer-info cache due to relaxed strides +* `#17652 `__: MAINT: Backport openblas_support from master. +* `#17653 `__: TST: Add Python 3.9 to the CI testing on Windows, Mac. +* `#17660 `__: TST: Simplify source path names in test_extending. diff --git a/doc/source/release/1.19.4-notes.rst b/doc/source/release/1.19.4-notes.rst new file mode 100644 index 000000000000..e7c0863f4510 --- /dev/null +++ b/doc/source/release/1.19.4-notes.rst @@ -0,0 +1,30 @@ +.. currentmodule:: numpy + +========================== +NumPy 1.19.4 Release Notes +========================== + +NumPy 1.19.4 is a quick release to revert the OpenBLAS library version. It was +hoped that the 0.3.12 OpenBLAS version used in 1.19.3 would work around the +Microsoft fmod bug, but problems in some docker environments turned up. Instead, +1.19.4 will use the older library and run a sanity check on import, raising an +error if the problem is detected. Microsoft is aware of the problem and has +promised a fix, users should upgrade when it becomes available. + +This release supports Python 3.6-3.9 + +Contributors +============ + +A total of 1 people contributed to this release. People with a "+" by their +names contributed a patch for the first time. + +* Charles Harris + +Pull requests merged +==================== + +A total of 2 pull requests were merged for this release. + +* `#17679 `__: MAINT: Add check for Windows 10 version 2004 bug. +* `#17680 `__: REV: Revert OpenBLAS to 1.19.2 version for 1.19.4 diff --git a/doc/source/user/absolute_beginners.rst b/doc/source/user/absolute_beginners.rst index 5873eb10862e..126f5f2a395e 100644 --- a/doc/source/user/absolute_beginners.rst +++ b/doc/source/user/absolute_beginners.rst @@ -1090,7 +1090,7 @@ To learn more about finding the unique elements in an array, see `unique`. Transposing and reshaping a matrix ---------------------------------- -*This section covers* ``arr.reshape()``, ``arr.transpose()``, ``arr.T()`` +*This section covers* ``arr.reshape()``, ``arr.transpose()``, ``arr.T`` ----- @@ -1114,7 +1114,7 @@ You simply need to pass in the new dimensions that you want for the matrix. :: .. image:: images/np_reshape.png -You can also use ``.transpose`` to reverse or change the axes of an array +You can also use ``.transpose()`` to reverse or change the axes of an array according to the values you specify. If you start with this array:: @@ -1131,6 +1131,13 @@ You can transpose your array with ``arr.transpose()``. :: [1, 4], [2, 5]]) +You can also use ``arr.T``:: + + >>> arr.T + array([[0, 3], + [1, 4], + [2, 5]]) + To learn more about transposing and reshaping arrays, see `transpose` and `reshape`. @@ -1138,12 +1145,12 @@ To learn more about transposing and reshaping arrays, see `transpose` and How to reverse an array ----------------------- -*This section covers* ``np.flip`` +*This section covers* ``np.flip()`` ----- NumPy's ``np.flip()`` function allows you to flip, or reverse, the contents of -an array along an axis. When using ``np.flip``, specify the array you would like +an array along an axis. When using ``np.flip()``, specify the array you would like to reverse and the axis. If you don't specify the axis, NumPy will reverse the contents along all of the axes of your input array. diff --git a/doc/source/user/basics.rec.rst b/doc/source/user/basics.rec.rst index f579b0d85141..0524fde8ef23 100644 --- a/doc/source/user/basics.rec.rst +++ b/doc/source/user/basics.rec.rst @@ -132,7 +132,7 @@ summary they are: Offsets may be chosen such that the fields overlap, though this will mean that assigning to one field may clobber any overlapping field's data. As - an exception, fields of :class:`numpy.object` type cannot overlap with + an exception, fields of :class:`numpy.object_` type cannot overlap with other fields, because of the risk of clobbering the internal object pointer and then dereferencing it. @@ -545,7 +545,7 @@ Viewing Structured Arrays Containing Objects -------------------------------------------- In order to prevent clobbering object pointers in fields of -:class:`numpy.object` type, numpy currently does not allow views of structured +:class:`object` type, numpy currently does not allow views of structured arrays containing objects. Structure Comparison @@ -575,11 +575,14 @@ Record Arrays ============= As an optional convenience numpy provides an ndarray subclass, -:class:`numpy.recarray`, and associated helper functions in the -:mod:`numpy.lib.recfunctions` submodule (aliased as ``numpy.rec``), that allows -access to fields of structured arrays by attribute instead of only by index. -Record arrays also use a special datatype, :class:`numpy.record`, that allows +:class:`numpy.recarray` that allows access to fields of structured arrays by +attribute instead of only by index. +Record arrays use a special datatype, :class:`numpy.record`, that allows field access by attribute on the structured scalars obtained from the array. +The :mod:`numpy.rec` module provides functions for creating recarrays from +various objects. +Additional helper functions for creating and manipulating structured arrays +can be found in :mod:`numpy.lib.recfunctions`. The simplest way to create a record array is with ``numpy.rec.array``:: @@ -616,8 +619,8 @@ appropriate `view `_:: >>> recordarr = arr.view(dtype=np.dtype((np.record, arr.dtype)), ... type=np.recarray) -For convenience, viewing an ndarray as type :class:`np.recarray` will -automatically convert to :class:`np.record` datatype, so the dtype can be left +For convenience, viewing an ndarray as type :class:`numpy.recarray` will +automatically convert to :class:`numpy.record` datatype, so the dtype can be left out of the view:: >>> recordarr = arr.view(np.recarray) diff --git a/doc/source/user/basics.subclassing.rst b/doc/source/user/basics.subclassing.rst index d8d104220e0d..8ffa31688531 100644 --- a/doc/source/user/basics.subclassing.rst +++ b/doc/source/user/basics.subclassing.rst @@ -92,8 +92,8 @@ Implications for subclassing If we subclass ndarray, we need to deal not only with explicit construction of our array type, but also :ref:`view-casting` or -:ref:`new-from-template`. NumPy has the machinery to do this, and this -machinery that makes subclassing slightly non-standard. +:ref:`new-from-template`. NumPy has the machinery to do this, and it is +this machinery that makes subclassing slightly non-standard. There are two aspects to the machinery that ndarray uses to support views and new-from-template in subclasses. diff --git a/doc/source/user/basics.types.rst b/doc/source/user/basics.types.rst index 3c39b35d0feb..ec2af409ae86 100644 --- a/doc/source/user/basics.types.rst +++ b/doc/source/user/basics.types.rst @@ -19,78 +19,78 @@ The primitive types supported are tied closely to those in C: - C type - Description - * - `np.bool_` + * - `numpy.bool_` - ``bool`` - Boolean (True or False) stored as a byte - * - `np.byte` + * - `numpy.byte` - ``signed char`` - Platform-defined - * - `np.ubyte` + * - `numpy.ubyte` - ``unsigned char`` - Platform-defined - * - `np.short` + * - `numpy.short` - ``short`` - Platform-defined - * - `np.ushort` + * - `numpy.ushort` - ``unsigned short`` - Platform-defined - * - `np.intc` + * - `numpy.intc` - ``int`` - Platform-defined - * - `np.uintc` + * - `numpy.uintc` - ``unsigned int`` - Platform-defined - * - `np.int_` + * - `numpy.int_` - ``long`` - Platform-defined - * - `np.uint` + * - `numpy.uint` - ``unsigned long`` - Platform-defined - * - `np.longlong` + * - `numpy.longlong` - ``long long`` - Platform-defined - * - `np.ulonglong` + * - `numpy.ulonglong` - ``unsigned long long`` - Platform-defined - * - `np.half` / `np.float16` + * - `numpy.half` / `numpy.float16` - - Half precision float: sign bit, 5 bits exponent, 10 bits mantissa - * - `np.single` + * - `numpy.single` - ``float`` - Platform-defined single precision float: typically sign bit, 8 bits exponent, 23 bits mantissa - * - `np.double` + * - `numpy.double` - ``double`` - Platform-defined double precision float: typically sign bit, 11 bits exponent, 52 bits mantissa. - * - `np.longdouble` + * - `numpy.longdouble` - ``long double`` - Platform-defined extended-precision float - * - `np.csingle` + * - `numpy.csingle` - ``float complex`` - Complex number, represented by two single-precision floats (real and imaginary components) - * - `np.cdouble` + * - `numpy.cdouble` - ``double complex`` - Complex number, represented by two double-precision floats (real and imaginary components). - * - `np.clongdouble` + * - `numpy.clongdouble` - ``long double complex`` - Complex number, represented by two extended-precision floats (real and imaginary components). @@ -105,59 +105,59 @@ aliases are provided: - C type - Description - * - `np.int8` + * - `numpy.int8` - ``int8_t`` - Byte (-128 to 127) - * - `np.int16` + * - `numpy.int16` - ``int16_t`` - Integer (-32768 to 32767) - * - `np.int32` + * - `numpy.int32` - ``int32_t`` - Integer (-2147483648 to 2147483647) - * - `np.int64` + * - `numpy.int64` - ``int64_t`` - Integer (-9223372036854775808 to 9223372036854775807) - * - `np.uint8` + * - `numpy.uint8` - ``uint8_t`` - Unsigned integer (0 to 255) - * - `np.uint16` + * - `numpy.uint16` - ``uint16_t`` - Unsigned integer (0 to 65535) - * - `np.uint32` + * - `numpy.uint32` - ``uint32_t`` - Unsigned integer (0 to 4294967295) - * - `np.uint64` + * - `numpy.uint64` - ``uint64_t`` - Unsigned integer (0 to 18446744073709551615) - * - `np.intp` + * - `numpy.intp` - ``intptr_t`` - Integer used for indexing, typically the same as ``ssize_t`` - * - `np.uintp` + * - `numpy.uintp` - ``uintptr_t`` - Integer large enough to hold a pointer - * - `np.float32` + * - `numpy.float32` - ``float`` - - * - `np.float64` / `np.float_` + * - `numpy.float64` / `numpy.float_` - ``double`` - Note that this matches the precision of the builtin python `float`. - * - `np.complex64` + * - `numpy.complex64` - ``float complex`` - Complex number, represented by two 32-bit floats (real and imaginary components) - * - `np.complex128` / `np.complex_` + * - `numpy.complex128` / `numpy.complex_` - ``double complex`` - Note that this matches the precision of the builtin python `complex`. diff --git a/doc/source/user/building.rst b/doc/source/user/building.rst index 54ece3da3749..47399139e6e8 100644 --- a/doc/source/user/building.rst +++ b/doc/source/user/building.rst @@ -142,6 +142,16 @@ will prefer to use ATLAS, then BLIS, then OpenBLAS and as a last resort MKL. If neither of these exists the build will fail (names are compared lower case). +Alternatively one may use ``!`` or ``^`` to negate all items:: + + NPY_BLAS_ORDER='^blas,atlas' python setup.py build + +will allow using anything **but** NetLIB BLAS and ATLAS libraries, the order of the above +list is retained. + +One cannot mix negation and positives, nor have multiple negations, such cases will +raise an error. + LAPACK ~~~~~~ @@ -165,6 +175,17 @@ will prefer to use ATLAS, then OpenBLAS and as a last resort MKL. If neither of these exists the build will fail (names are compared lower case). +Alternatively one may use ``!`` or ``^`` to negate all items:: + + NPY_LAPACK_ORDER='^lapack' python setup.py build + +will allow using anything **but** the NetLIB LAPACK library, the order of the above +list is retained. + +One cannot mix negation and positives, nor have multiple negations, such cases will +raise an error. + + .. deprecated:: 1.20 The native libraries on macOS, provided by Accelerate, are not fit for use in NumPy since they have bugs that cause wrong output under easily reproducible diff --git a/doc/source/user/c-info.beyond-basics.rst b/doc/source/user/c-info.beyond-basics.rst index 9e9cd30679a0..124162d6ce17 100644 --- a/doc/source/user/c-info.beyond-basics.rst +++ b/doc/source/user/c-info.beyond-basics.rst @@ -115,7 +115,7 @@ processors that use pipelining to enhance fundamental operations. The :c:func:`PyArray_IterAllButAxis` ( ``array``, ``&dim`` ) constructs an iterator object that is modified so that it will not iterate over the dimension indicated by dim. The only restriction on this iterator -object, is that the :c:func:`PyArray_Iter_GOTO1D` ( ``it``, ``ind`` ) macro +object, is that the :c:func:`PyArray_ITER_GOTO1D` ( ``it``, ``ind`` ) macro cannot be used (thus flat indexing won't work either if you pass this object back to Python --- so you shouldn't do this). Note that the returned object from this routine is still usually cast to diff --git a/doc/source/user/c-info.how-to-extend.rst b/doc/source/user/c-info.how-to-extend.rst index d7524209202a..845ce0a74c31 100644 --- a/doc/source/user/c-info.how-to-extend.rst +++ b/doc/source/user/c-info.how-to-extend.rst @@ -363,7 +363,6 @@ particular set of requirements ( *e.g.* contiguous, aligned, and writeable). The syntax is :c:func:`PyArray_FROM_OTF` - Return an ndarray from any Python object, *obj*, that can be converted to an array. The number of dimensions in the returned array is determined by the object. The desired data-type of the @@ -375,7 +374,6 @@ writeable). The syntax is exception is set. *obj* - The object can be any Python object convertible to an ndarray. If the object is already (a subclass of) the ndarray that satisfies the requirements then a new reference is returned. @@ -394,7 +392,6 @@ writeable). The syntax is to the requirements flag. *typenum* - One of the enumerated types or :c:data:`NPY_NOTYPE` if the data-type should be determined from the object itself. The C-based names can be used: @@ -422,7 +419,6 @@ writeable). The syntax is requirements flag to override this behavior. *requirements* - The memory model for an ndarray admits arbitrary strides in each dimension to advance to the next element of the array. Often, however, you need to interface with code that expects a @@ -446,13 +442,11 @@ writeable). The syntax is :c:data:`NPY_OUT_ARRAY`, and :c:data:`NPY_ARRAY_INOUT_ARRAY`: :c:data:`NPY_ARRAY_IN_ARRAY` - This flag is useful for arrays that must be in C-contiguous order and aligned. These kinds of arrays are usually input arrays for some algorithm. :c:data:`NPY_ARRAY_OUT_ARRAY` - This flag is useful to specify an array that is in C-contiguous order, is aligned, and can be written to as well. Such an array is usually returned as output @@ -460,7 +454,6 @@ writeable). The syntax is scratch). :c:data:`NPY_ARRAY_INOUT_ARRAY` - This flag is useful to specify an array that will be used for both input and output. :c:func:`PyArray_ResolveWritebackIfCopy` must be called before :c:func:`Py_DECREF` at @@ -479,16 +472,13 @@ writeable). The syntax is Other useful flags that can be OR'd as additional requirements are: :c:data:`NPY_ARRAY_FORCECAST` - Cast to the desired type, even if it can't be done without losing information. :c:data:`NPY_ARRAY_ENSURECOPY` - Make sure the resulting array is a copy of the original. :c:data:`NPY_ARRAY_ENSUREARRAY` - Make sure the resulting object is an actual ndarray and not a sub- class. diff --git a/doc/source/user/how-to-how-to.rst b/doc/source/user/how-to-how-to.rst new file mode 100644 index 000000000000..de8afc28ae9e --- /dev/null +++ b/doc/source/user/how-to-how-to.rst @@ -0,0 +1,118 @@ +.. _how-to-how-to: + +############################################################################## +How to write a NumPy how-to +############################################################################## + +How-tos get straight to the point -- they + + - answer a focused question, or + - narrow a broad question into focused questions that the user can + choose among. + +****************************************************************************** +A stranger has asked for directions... +****************************************************************************** + +**"I need to refuel my car."** + +****************************************************************************** +Give a brief but explicit answer +****************************************************************************** + + - `"Three kilometers/miles, take a right at Hayseed Road, it's on your left."` + +Add helpful details for newcomers ("Hayseed Road", even though it's the only +turnoff at three km/mi). But not irrelevant ones: + + - Don't also give directions from Route 7. + - Don't explain why the town has only one filling station. + +If there's related background (tutorial, explanation, reference, alternative +approach), bring it to the user's attention with a link ("Directions from Route 7," +"Why so few filling stations?"). + + +****************************************************************************** +Delegate +****************************************************************************** + + - `"Three km/mi, take a right at Hayseed Road, follow the signs."` + +If the information is already documented and succinct enough for a how-to, +just link to it, possibly after an introduction ("Three km/mi, take a right"). + +****************************************************************************** +If the question is broad, narrow and redirect it +****************************************************************************** + + **"I want to see the sights."** + +The `See the sights` how-to should link to a set of narrower how-tos: + +- Find historic buildings +- Find scenic lookouts +- Find the town center + +and these might in turn link to still narrower how-tos -- so the town center +page might link to + + - Find the court house + - Find city hall + +By organizing how-tos this way, you not only display the options for people +who need to narrow their question, you also have provided answers for users +who start with narrower questions ("I want to see historic buildings," "Which +way to city hall?"). + +****************************************************************************** +If there are many steps, break them up +****************************************************************************** + +If a how-to has many steps: + + - Consider breaking a step out into an individual how-to and linking to it. + - Include subheadings. They help readers grasp what's coming and return + where they left off. + +****************************************************************************** +Why write how-tos when there's Stack Overflow, Reddit, Gitter...? +****************************************************************************** + + - We have authoritative answers. + - How-tos make the site less forbidding to non-experts. + - How-tos bring people into the site and help them discover other information + that's here . + - Creating how-tos helps us see NumPy usability through new eyes. + +****************************************************************************** +Aren't how-tos and tutorials the same thing? +****************************************************************************** + +People use the terms "how-to" and "tutorial" interchangeably, but we draw a +distinction, following Daniele Procida's `taxonomy of documentation`_. + + .. _`taxonomy of documentation`: https://documentation.divio.com/ + +Documentation needs to meet users where they are. `How-tos` offer get-it-done +information; the user wants steps to copy and doesn't necessarily want to +understand NumPy. `Tutorials` are warm-fuzzy information; the user wants a +feel for some aspect of NumPy (and again, may or may not care about deeper +knowledge). + +We distinguish both tutorials and how-tos from `Explanations`, which are +deep dives intended to give understanding rather than immediate assistance, +and `References`, which give complete, autoritative data on some concrete +part of NumPy (like its API) but aren't obligated to paint a broader picture. + +For more on tutorials, see the `tutorial how-to`_. + +.. _`tutorial how-to`: https://github.com/numpy/numpy-tutorials/blob/master/tutorial_style.ipynb + + +****************************************************************************** +Is this page an example of a how-to? +****************************************************************************** + +Yes -- until the sections with question-mark headings; they explain rather +than giving directions. In a how-to, those would be links. \ No newline at end of file diff --git a/doc/source/user/how-to-io.rst b/doc/source/user/how-to-io.rst new file mode 100644 index 000000000000..ca9fc41f07f8 --- /dev/null +++ b/doc/source/user/how-to-io.rst @@ -0,0 +1,328 @@ +.. _how-to-io: + +############################################################################## +Reading and writing files +############################################################################## + +This page tackles common applications; for the full collection of I/O +routines, see :ref:`routines.io`. + + +****************************************************************************** +Reading text and CSV_ files +****************************************************************************** + +.. _CSV: https://en.wikipedia.org/wiki/Comma-separated_values + +With no missing values +============================================================================== + +Use :func:`numpy.loadtxt`. + +With missing values +============================================================================== + +Use :func:`numpy.genfromtxt`. + +:func:`numpy.genfromtxt` will either + + - return a :ref:`masked array` + **masking out missing values** (if ``usemask=True``), or + + - **fill in the missing value** with the value specified in + ``filling_values`` (default is ``np.nan`` for float, -1 for int). + +With non-whitespace delimiters +------------------------------------------------------------------------------ +:: + + >>> print(open("csv.txt").read()) # doctest: +SKIP + 1, 2, 3 + 4,, 6 + 7, 8, 9 + + +Masked-array output +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +:: + + >>> np.genfromtxt("csv.txt", delimiter=",", usemask=True) # doctest: +SKIP + masked_array( + data=[[1.0, 2.0, 3.0], + [4.0, --, 6.0], + [7.0, 8.0, 9.0]], + mask=[[False, False, False], + [False, True, False], + [False, False, False]], + fill_value=1e+20) + +Array output +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +:: + + >>> np.genfromtxt("csv.txt", delimiter=",") # doctest: +SKIP + array([[ 1., 2., 3.], + [ 4., nan, 6.], + [ 7., 8., 9.]]) + +Array output, specified fill-in value +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +:: + + >>> np.genfromtxt("csv.txt", delimiter=",", dtype=np.int8, filling_values=99) # doctest: +SKIP + array([[ 1, 2, 3], + [ 4, 99, 6], + [ 7, 8, 9]], dtype=int8) + +Whitespace-delimited +------------------------------------------------------------------------------- + +:func:`numpy.genfromtxt` can also parse whitespace-delimited data files +that have missing values if + +* **Each field has a fixed width**: Use the width as the `delimiter` argument. + :: + + # File with width=4. The data does not have to be justified (for example, + # the 2 in row 1), the last column can be less than width (for example, the 6 + # in row 2), and no delimiting character is required (for instance 8888 and 9 + # in row 3) + + >>> f = open("fixedwidth.txt").read() # doctest: +SKIP + >>> print(f) # doctest: +SKIP + 1 2 3 + 44 6 + 7 88889 + + # Showing spaces as ^ + >>> print(f.replace(" ","^")) # doctest: +SKIP + 1^^^2^^^^^^3 + 44^^^^^^6 + 7^^^88889 + + >>> np.genfromtxt("fixedwidth.txt", delimiter=4) # doctest: +SKIP + array([[1.000e+00, 2.000e+00, 3.000e+00], + [4.400e+01, nan, 6.000e+00], + [7.000e+00, 8.888e+03, 9.000e+00]]) + +* **A special value (e.g. "x") indicates a missing field**: Use it as the + `missing_values` argument. + :: + + >>> print(open("nan.txt").read()) # doctest: +SKIP + 1 2 3 + 44 x 6 + 7 8888 9 + + >>> np.genfromtxt("nan.txt", missing_values="x") # doctest: +SKIP + array([[1.000e+00, 2.000e+00, 3.000e+00], + [4.400e+01, nan, 6.000e+00], + [7.000e+00, 8.888e+03, 9.000e+00]]) + +* **You want to skip the rows with missing values**: Set + `invalid_raise=False`. + :: + + >>> print(open("skip.txt").read()) # doctest: +SKIP + 1 2 3 + 44 6 + 7 888 9 + + >>> np.genfromtxt("skip.txt", invalid_raise=False) # doctest: +SKIP + __main__:1: ConversionWarning: Some errors were detected ! + Line #2 (got 2 columns instead of 3) + array([[ 1., 2., 3.], + [ 7., 888., 9.]]) + + +* **The delimiter whitespace character is different from the whitespace that + indicates missing data**. For instance, if columns are delimited by ``\t``, + then missing data will be recognized if it consists of one + or more spaces. + :: + + >>> f = open("tabs.txt").read() # doctest: +SKIP + >>> print(f) # doctest: +SKIP + 1 2 3 + 44 6 + 7 888 9 + + # Tabs vs. spaces + >>> print(f.replace("\t","^")) # doctest: +SKIP + 1^2^3 + 44^ ^6 + 7^888^9 + + >>> np.genfromtxt("tabs.txt", delimiter="\t", missing_values=" +") # doctest: +SKIP + array([[ 1., 2., 3.], + [ 44., nan, 6.], + [ 7., 888., 9.]]) + +****************************************************************************** +Read a file in .npy or .npz format +****************************************************************************** + +Choices: + + - Use :func:`numpy.load`. It can read files generated by any of + :func:`numpy.save`, :func:`numpy.savez`, or :func:`numpy.savez_compressed`. + + - Use memory mapping. See `numpy.lib.format.open_memmap`. + +****************************************************************************** +Write to a file to be read back by NumPy +****************************************************************************** + +Binary +=============================================================================== + +Use +:func:`numpy.save`, or to store multiple arrays :func:`numpy.savez` +or :func:`numpy.savez_compressed`. + +For :ref:`security and portability `, set +``allow_pickle=False`` unless the dtype contains Python objects, which +requires pickling. + +Masked arrays :any:`can't currently be saved `, +nor can other arbitrary array subclasses. + +Human-readable +============================================================================== + +:func:`numpy.save` and :func:`numpy.savez` create binary files. To **write a +human-readable file**, use :func:`numpy.savetxt`. The array can only be 1- or +2-dimensional, and there's no ` savetxtz` for multiple files. + +Large arrays +============================================================================== + +See :ref:`how-to-io-large-arrays`. + +****************************************************************************** +Read an arbitrarily formatted binary file ("binary blob") +****************************************************************************** + +Use a :doc:`structured array `. + +**Example:** + +The ``.wav`` file header is a 44-byte block preceding ``data_size`` bytes of the +actual sound data:: + + chunk_id "RIFF" + chunk_size 4-byte unsigned little-endian integer + format "WAVE" + fmt_id "fmt " + fmt_size 4-byte unsigned little-endian integer + audio_fmt 2-byte unsigned little-endian integer + num_channels 2-byte unsigned little-endian integer + sample_rate 4-byte unsigned little-endian integer + byte_rate 4-byte unsigned little-endian integer + block_align 2-byte unsigned little-endian integer + bits_per_sample 2-byte unsigned little-endian integer + data_id "data" + data_size 4-byte unsigned little-endian integer + +The ``.wav`` file header as a NumPy structured dtype:: + + wav_header_dtype = np.dtype([ + ("chunk_id", (bytes, 4)), # flexible-sized scalar type, item size 4 + ("chunk_size", "`_.) + +.. _how-to-io-large-arrays: + +****************************************************************************** +Write or read large arrays +****************************************************************************** + +**Arrays too large to fit in memory** can be treated like ordinary in-memory +arrays using memory mapping. + +- Raw array data written with :func:`numpy.ndarray.tofile` or + :func:`numpy.ndarray.tobytes` can be read with :func:`numpy.memmap`:: + + array = numpy.memmap("mydata/myarray.arr", mode="r", dtype=np.int16, shape=(1024, 1024)) + +- Files output by :func:`numpy.save` (that is, using the numpy format) can be read + using :func:`numpy.load` with the ``mmap_mode`` keyword argument:: + + large_array[some_slice] = np.load("path/to/small_array", mmap_mode="r") + +Memory mapping lacks features like data chunking and compression; more +full-featured formats and libraries usable with NumPy include: + +* **HDF5**: `h5py `_ or `PyTables `_. +* **Zarr**: `here `_. +* **NetCDF**: :class:`scipy.io.netcdf_file`. + +For tradeoffs among memmap, Zarr, and HDF5, see +`pythonspeed.com `_. + +****************************************************************************** +Write files for reading by other (non-NumPy) tools +****************************************************************************** + +Formats for **exchanging data** with other tools include HDF5, Zarr, and +NetCDF (see :ref:`how-to-io-large-arrays`). + +****************************************************************************** +Write or read a JSON file +****************************************************************************** + +NumPy arrays are **not** directly +`JSON serializable `_. + + +.. _how-to-io-pickle-file: + +****************************************************************************** +Save/restore using a pickle file +****************************************************************************** + +Avoid when possible; :doc:`pickles ` are not secure +against erroneous or maliciously constructed data. + +Use :func:`numpy.save` and :func:`numpy.load`. Set ``allow_pickle=False``, +unless the array dtype includes Python objects, in which case pickling is +required. + +****************************************************************************** +Convert from a pandas DataFrame to a NumPy array +****************************************************************************** + +See :meth:`pandas.DataFrame.to_numpy`. + +****************************************************************************** + Save/restore using `~numpy.ndarray.tofile` and `~numpy.fromfile` +****************************************************************************** + +In general, prefer :func:`numpy.save` and :func:`numpy.load`. + +:func:`numpy.ndarray.tofile` and :func:`numpy.fromfile` lose information on +endianness and precision and so are unsuitable for anything but scratch +storage. + diff --git a/doc/source/user/howtos_index.rst b/doc/source/user/howtos_index.rst index c052286b9be8..89a6f54e791c 100644 --- a/doc/source/user/howtos_index.rst +++ b/doc/source/user/howtos_index.rst @@ -11,4 +11,5 @@ the package, see the :ref:`API reference `. .. toctree:: :maxdepth: 1 - ionumpy + how-to-how-to + how-to-io diff --git a/doc/source/user/index.rst b/doc/source/user/index.rst index 3a79f0f2e9a5..28297d9eaa9b 100644 --- a/doc/source/user/index.rst +++ b/doc/source/user/index.rst @@ -3,18 +3,17 @@ .. _user: ################ -NumPy User Guide +NumPy user guide ################ -This guide is intended as an introductory overview of NumPy and -explains how to install and make use of the most important features of -NumPy. For detailed reference documentation of the functions and -classes contained in the package, see the :ref:`reference`. +This guide is an overview and explains the important features; +details are found in :ref:`reference`. .. toctree:: :maxdepth: 1 - setting-up + whatisnumpy + Installation quickstart absolute_beginners basics @@ -26,7 +25,10 @@ classes contained in the package, see the :ref:`reference`. howtos_index -.. These are stuck here to avoid the "WARNING: document isn't included in any +.. Links to these files are placed directly in the top-level html + (doc/source/_templates/indexcontent.html, which appears for the URLs + numpy.org/devdocs and numpy.org/doc/XX) and are not in any toctree, so + we include them here to avoid a "WARNING: document isn't included in any toctree" message .. toctree:: @@ -39,5 +41,5 @@ classes contained in the package, see the :ref:`reference`. ../docs/index ../bugs ../release - ../about + ../doc_conventions ../license diff --git a/doc/source/user/install.rst b/doc/source/user/install.rst index 52586f3d7d6a..e05cee2f130d 100644 --- a/doc/source/user/install.rst +++ b/doc/source/user/install.rst @@ -1,10 +1,7 @@ -**************** -Installing NumPy -**************** - -In most use cases the best way to install NumPy on your system is by using a -pre-built package for your operating system. Please see -https://scipy.org/install.html for links to available options. - -For instructions on building for source package, see -:doc:`building`. This information is useful mainly for advanced users. +:orphan: + +**************** +Installing NumPy +**************** + +See `Installing NumPy `_. \ No newline at end of file diff --git a/doc/source/user/ionumpy.rst b/doc/source/user/ionumpy.rst deleted file mode 100644 index a31720322a77..000000000000 --- a/doc/source/user/ionumpy.rst +++ /dev/null @@ -1,20 +0,0 @@ -================================================ -How to read and write data using NumPy -================================================ - -.. currentmodule:: numpy - -.. testsetup:: - - import numpy as np - np.random.seed(1) - -**Objectives** - -- Writing NumPy arrays to files -- Reading NumPy arrays from files -- Dealing with encoding and dtype issues - -**Content** - -To be completed. diff --git a/doc/source/user/numpy-for-matlab-users.rst b/doc/source/user/numpy-for-matlab-users.rst index 547d5b2a0767..ed0be82a0851 100644 --- a/doc/source/user/numpy-for-matlab-users.rst +++ b/doc/source/user/numpy-for-matlab-users.rst @@ -7,12 +7,9 @@ NumPy for MATLAB users Introduction ============ -MATLAB® and NumPy/SciPy have a lot in common. But there are many -differences. NumPy and SciPy were created to do numerical and scientific -computing in the most natural way with Python, not to be MATLAB clones. -This page is intended to be a place to collect wisdom about the -differences, mostly for the purpose of helping proficient MATLAB users -become proficient NumPy and SciPy users. +MATLAB® and NumPy have a lot in common, but NumPy was created to work with +Python, not to be a MATLAB clone. This guide will help MATLAB users get started +with NumPy. .. raw:: html @@ -20,234 +17,184 @@ become proficient NumPy and SciPy users. table.docutils td { border: solid 1px #ccc; } -Some Key Differences +Some key differences ==================== .. list-table:: - - * - In MATLAB, the basic data type is a multidimensional array of - double precision floating point numbers. Most expressions take such - arrays and return such arrays. Operations on the 2-D instances of - these arrays are designed to act more or less like matrix operations - in linear algebra. - - In NumPy the basic type is a multidimensional ``array``. Operations - on these arrays in all dimensionalities including 2D are element-wise - operations. One needs to use specific functions for linear algebra - (though for matrix multiplication, one can use the ``@`` operator - in python 3.5 and above). - - * - MATLAB uses 1 (one) based indexing. The initial element of a - sequence is found using a(1). + :class: docutils + + * - In MATLAB, the basic type, even for scalars, is a + multidimensional array. Array assignments in MATLAB are stored as + 2D arrays of double precision floating point numbers, unless you + specify the number of dimensions and type. Operations on the 2D + instances of these arrays are modeled on matrix operations in + linear algebra. + + - In NumPy, the basic type is a multidimensional ``array``. Array + assignments in NumPy are usually stored as :ref:`n-dimensional arrays` with the + minimum type required to hold the objects in sequence, unless you + specify the number of dimensions and type. NumPy performs + operations element-by-element, so multiplying 2D arrays with + ``*`` is not a matrix multiplication -- it's an + element-by-element multiplication. (The ``@`` operator, available + since Python 3.5, can be used for conventional matrix + multiplication.) + + * - MATLAB numbers indices from 1; ``a(1)`` is the first element. :ref:`See note INDEXING ` - - Python uses 0 (zero) based indexing. The initial element of a - sequence is found using a[0]. - - * - MATLAB's scripting language was created for doing linear algebra. - The syntax for basic matrix operations is nice and clean, but the API - for adding GUIs and making full-fledged applications is more or less - an afterthought. - - NumPy is based on Python, which was designed from the outset to be - an excellent general-purpose programming language. While MATLAB's - syntax for some array manipulations is more compact than - NumPy's, NumPy (by virtue of being an add-on to Python) can do many - things that MATLAB just cannot, for instance dealing properly with - stacks of matrices. - - * - In MATLAB, arrays have pass-by-value semantics, with a lazy - copy-on-write scheme to prevent actually creating copies until they - are actually needed. Slice operations copy parts of the array. - - In NumPy arrays have pass-by-reference semantics. Slice operations - are views into an array. - - -'array' or 'matrix'? Which should I use? -======================================== - -Historically, NumPy has provided a special matrix type, `np.matrix`, which -is a subclass of ndarray which makes binary operations linear algebra -operations. You may see it used in some existing code instead of `np.array`. -So, which one to use? - -Short answer ------------- - -**Use arrays**. - -- They are the standard vector/matrix/tensor type of numpy. Many numpy - functions return arrays, not matrices. -- There is a clear distinction between element-wise operations and - linear algebra operations. -- You can have standard vectors or row/column vectors if you like. - -Until Python 3.5 the only disadvantage of using the array type was that you -had to use ``dot`` instead of ``*`` to multiply (reduce) two tensors -(scalar product, matrix vector multiplication etc.). Since Python 3.5 you -can use the matrix multiplication ``@`` operator. - -Given the above, we intend to deprecate ``matrix`` eventually. - -Long answer ------------ - -NumPy contains both an ``array`` class and a ``matrix`` class. The -``array`` class is intended to be a general-purpose n-dimensional array -for many kinds of numerical computing, while ``matrix`` is intended to -facilitate linear algebra computations specifically. In practice there -are only a handful of key differences between the two. - -- Operators ``*`` and ``@``, functions ``dot()``, and ``multiply()``: - - - For ``array``, **``*`` means element-wise multiplication**, while - **``@`` means matrix multiplication**; they have associated functions - ``multiply()`` and ``dot()``. (Before python 3.5, ``@`` did not exist - and one had to use ``dot()`` for matrix multiplication). - - For ``matrix``, **``*`` means matrix multiplication**, and for - element-wise multiplication one has to use the ``multiply()`` function. - -- Handling of vectors (one-dimensional arrays) - - - For ``array``, the **vector shapes 1xN, Nx1, and N are all different - things**. Operations like ``A[:,1]`` return a one-dimensional array of - shape N, not a two-dimensional array of shape Nx1. Transpose on a - one-dimensional ``array`` does nothing. - - For ``matrix``, **one-dimensional arrays are always upconverted to 1xN - or Nx1 matrices** (row or column vectors). ``A[:,1]`` returns a - two-dimensional matrix of shape Nx1. - -- Handling of higher-dimensional arrays (ndim > 2) - - - ``array`` objects **can have number of dimensions > 2**; - - ``matrix`` objects **always have exactly two dimensions**. - -- Convenience attributes - - - ``array`` **has a .T attribute**, which returns the transpose of - the data. - - ``matrix`` **also has .H, .I, and .A attributes**, which return - the conjugate transpose, inverse, and ``asarray()`` of the matrix, - respectively. - -- Convenience constructor + - NumPy, like Python, numbers indices from 0; ``a[0]`` is the first + element. - - The ``array`` constructor **takes (nested) Python sequences as - initializers**. As in, ``array([[1,2,3],[4,5,6]])``. - - The ``matrix`` constructor additionally **takes a convenient - string initializer**. As in ``matrix("[1 2 3; 4 5 6]")``. - -There are pros and cons to using both: - -- ``array`` - - - ``:)`` Element-wise multiplication is easy: ``A*B``. - - ``:(`` You have to remember that matrix multiplication has its own - operator, ``@``. - - ``:)`` You can treat one-dimensional arrays as *either* row or column - vectors. ``A @ v`` treats ``v`` as a column vector, while - ``v @ A`` treats ``v`` as a row vector. This can save you having to - type a lot of transposes. - - ``:)`` ``array`` is the "default" NumPy type, so it gets the most - testing, and is the type most likely to be returned by 3rd party - code that uses NumPy. - - ``:)`` Is quite at home handling data of any number of dimensions. - - ``:)`` Closer in semantics to tensor algebra, if you are familiar - with that. - - ``:)`` *All* operations (``*``, ``/``, ``+``, ``-`` etc.) are - element-wise. - - ``:(`` Sparse matrices from ``scipy.sparse`` do not interact as well - with arrays. - -- ``matrix`` - - - ``:\\`` Behavior is more like that of MATLAB matrices. - - ``<:(`` Maximum of two-dimensional. To hold three-dimensional data you - need ``array`` or perhaps a Python list of ``matrix``. - - ``<:(`` Minimum of two-dimensional. You cannot have vectors. They must be - cast as single-column or single-row matrices. - - ``<:(`` Since ``array`` is the default in NumPy, some functions may - return an ``array`` even if you give them a ``matrix`` as an - argument. This shouldn't happen with NumPy functions (if it does - it's a bug), but 3rd party code based on NumPy may not honor type - preservation like NumPy does. - - ``:)`` ``A*B`` is matrix multiplication, so it looks just like you write - it in linear algebra (For Python >= 3.5 plain arrays have the same - convenience with the ``@`` operator). - - ``<:(`` Element-wise multiplication requires calling a function, - ``multiply(A,B)``. - - ``<:(`` The use of operator overloading is a bit illogical: ``*`` - does not work element-wise but ``/`` does. - - Interaction with ``scipy.sparse`` is a bit cleaner. - -The ``array`` is thus much more advisable to use. Indeed, we intend to -deprecate ``matrix`` eventually. - -Table of Rough MATLAB-NumPy Equivalents + * - MATLAB's scripting language was created for linear algebra so the + syntax for some array manipulations is more compact than + NumPy's. On the other hand, the API for adding GUIs and creating + full-fledged applications is more or less an afterthought. + - NumPy is based on Python, a + general-purpose language. The advantage to NumPy + is access to Python libraries including: `SciPy + `_, `Matplotlib `_, + `Pandas `_, `OpenCV `_, + and more. In addition, Python is often `embedded as a scripting language + `_ + in other software, allowing NumPy to be used there too. + + * - MATLAB array slicing uses pass-by-value semantics, with a lazy + copy-on-write scheme to prevent creating copies until they are + needed. Slicing operations copy parts of the array. + - NumPy array slicing uses pass-by-reference, that does not copy + the arguments. Slicing operations are views into an array. + + +Rough equivalents ======================================= The table below gives rough equivalents for some common MATLAB -expressions. **These are not exact equivalents**, but rather should be -taken as hints to get you going in the right direction. For more detail -read the built-in documentation on the NumPy functions. +expressions. These are similar expressions, not equivalents. For +details, see the :ref:`documentation`. In the table below, it is assumed that you have executed the following commands in Python: :: - from numpy import * - import scipy.linalg + import numpy as np + from scipy import io, integrate, linalg, signal + from scipy.sparse.linalg import eigs Also assume below that if the Notes talk about "matrix" that the arguments are two-dimensional entities. -General Purpose Equivalents +General purpose equivalents --------------------------- .. list-table:: :header-rows: 1 - * - **MATLAB** - - **numpy** - - **Notes** + * - MATLAB + - NumPy + - Notes * - ``help func`` - - ``info(func)`` or ``help(func)`` or ``func?`` (in Ipython) + - ``info(func)`` or ``help(func)`` or ``func?`` (in IPython) - get help on the function *func* * - ``which func`` - - `see note HELP `__ + - :ref:`see note HELP ` - find out where *func* is defined * - ``type func`` - - ``source(func)`` or ``func??`` (in Ipython) + - ``np.source(func)`` or ``func??`` (in IPython) - print source for *func* (if not a native function) + * - ``% comment`` + - ``# comment`` + - comment a line of code with the text ``comment`` + + * - :: + + for i=1:3 + fprintf('%i\n',i) + end + + - :: + + for i in range(1, 4): + print(i) + + - use a for-loop to print the numbers 1, 2, and 3 using :py:class:`range ` + * - ``a && b`` - ``a and b`` - - short-circuiting logical AND operator (Python native operator); + - short-circuiting logical AND operator (:ref:`Python native operator `); scalar arguments only * - ``a || b`` - ``a or b`` - - short-circuiting logical OR operator (Python native operator); + - short-circuiting logical OR operator (:ref:`Python native operator `); scalar arguments only + * - .. code:: matlab + + >> 4 == 4 + ans = 1 + >> 4 == 5 + ans = 0 + + - :: + + >>> 4 == 4 + True + >>> 4 == 5 + False + + - The :ref:`boolean objects ` + in Python are ``True`` and ``False``, as opposed to MATLAB + logical types of ``1`` and ``0``. + + * - .. code:: matlab + + a=4 + if a==4 + fprintf('a = 4\n') + elseif a==5 + fprintf('a = 5\n') + end + + - :: + + a = 4 + if a == 4: + print('a = 4') + elif a == 5: + print('a = 5') + + - create an if-else statement to check if ``a`` is 4 or 5 and print result + * - ``1*i``, ``1*j``, ``1i``, ``1j`` - ``1j`` - complex numbers * - ``eps`` - - ``np.spacing(1)`` - - Distance between 1 and the nearest floating point number. + - ``np.finfo(float).eps`` or ``np.spacing(1)`` + - Upper bound to relative error due to rounding in 64-bit floating point + arithmetic. + + * - ``load data.mat`` + - ``io.loadmat('data.mat')`` + - Load MATLAB variables saved to the file ``data.mat``. (Note: When saving arrays to + ``data.mat`` in MATLAB/Octave, use a recent binary format. :func:`scipy.io.loadmat` + will create a dictionary with the saved arrays and further information.) * - ``ode45`` - - ``scipy.integrate.solve_ivp(f)`` + - ``integrate.solve_ivp(f)`` - integrate an ODE with Runge-Kutta 4,5 * - ``ode15s`` - - ``scipy.integrate.solve_ivp(f, method='BDF')`` + - ``integrate.solve_ivp(f, method='BDF')`` - integrate an ODE with BDF method -Linear Algebra Equivalents + +Linear algebra equivalents -------------------------- .. list-table:: @@ -258,16 +205,16 @@ Linear Algebra Equivalents - Notes * - ``ndims(a)`` - - ``ndim(a)`` or ``a.ndim`` - - get the number of dimensions of an array + - ``np.ndim(a)`` or ``a.ndim`` + - number of dimensions of array ``a`` * - ``numel(a)`` - - ``size(a)`` or ``a.size`` - - get the number of elements of an array + - ``np.size(a)`` or ``a.size`` + - number of elements of array ``a`` * - ``size(a)`` - - ``shape(a)`` or ``a.shape`` - - get the "size" of the matrix + - ``np.shape(a)`` or ``a.shape`` + - "size" of array ``a`` * - ``size(a,n)`` - ``a.shape[n-1]`` @@ -276,45 +223,45 @@ Linear Algebra Equivalents See note :ref:`INDEXING `) * - ``[ 1 2 3; 4 5 6 ]`` - - ``array([[1.,2.,3.], [4.,5.,6.]])`` - - 2x3 matrix literal + - ``np.array([[1. ,2. ,3.], [4. ,5. ,6.]])`` + - define a 2x3 2D array * - ``[ a b; c d ]`` - - ``block([[a,b], [c,d]])`` + - ``np.block([[a, b], [c, d]])`` - construct a matrix from blocks ``a``, ``b``, ``c``, and ``d`` * - ``a(end)`` - ``a[-1]`` - - access last element in the 1xn matrix ``a`` + - access last element in MATLAB vector (1xn or nx1) or 1D NumPy array + ``a`` (length n) * - ``a(2,5)`` - - ``a[1,4]`` - - access element in second row, fifth column + - ``a[1, 4]`` + - access element in second row, fifth column in 2D array ``a`` * - ``a(2,:)`` - - ``a[1]`` or ``a[1,:]`` - - entire second row of ``a`` + - ``a[1]`` or ``a[1, :]`` + - entire second row of 2D array ``a`` * - ``a(1:5,:)`` - - ``a[0:5]`` or ``a[:5]`` or ``a[0:5,:]`` - - the first five rows of ``a`` + - ``a[0:5]`` or ``a[:5]`` or ``a[0:5, :]`` + - first 5 rows of 2D array ``a`` * - ``a(end-4:end,:)`` - ``a[-5:]`` - - the last five rows of ``a`` + - last 5 rows of 2D array ``a`` * - ``a(1:3,5:9)`` - - ``a[0:3][:,4:9]`` - - rows one to three and columns five to nine of ``a``. This gives - read-only access. + - ``a[0:3, 4:9]`` + - The first through third rows and fifth through ninth columns of a 2D array, ``a``. * - ``a([2,4,5],[1,3])`` - - ``a[ix_([1,3,4],[0,2])]`` + - ``a[np.ix_([1, 3, 4], [0, 2])]`` - rows 2,4 and 5 and columns 1 and 3. This allows the matrix to be modified, and doesn't require a regular slice. * - ``a(3:2:21,:)`` - - ``a[ 2:21:2,:]`` + - ``a[2:21:2,:]`` - every other row of ``a``, starting with the third and going to the twenty-first @@ -323,11 +270,11 @@ Linear Algebra Equivalents - every other row of ``a``, starting with the first * - ``a(end:-1:1,:)`` or ``flipud(a)`` - - ``a[ ::-1,:]`` + - ``a[::-1,:]`` - ``a`` with rows in reverse order * - ``a([1:end 1],:)`` - - ``a[r_[:len(a),0]]`` + - ``a[np.r_[:len(a),0]]`` - ``a`` with copy of the first row appended to the end * - ``a.'`` @@ -354,30 +301,30 @@ Linear Algebra Equivalents - ``a**3`` - element-wise exponentiation - * - ``(a>0.5)`` - - ``(a>0.5)`` + * - ``(a > 0.5)`` + - ``(a > 0.5)`` - matrix whose i,jth element is (a_ij > 0.5). The MATLAB result is an - array of 0s and 1s. The NumPy result is an array of the boolean + array of logical values 0 and 1. The NumPy result is an array of the boolean values ``False`` and ``True``. - * - ``find(a>0.5)`` - - ``nonzero(a>0.5)`` + * - ``find(a > 0.5)`` + - ``np.nonzero(a > 0.5)`` - find the indices where (``a`` > 0.5) - * - ``a(:,find(v>0.5))`` - - ``a[:,nonzero(v>0.5)[0]]`` + * - ``a(:,find(v > 0.5))`` + - ``a[:,np.nonzero(v > 0.5)[0]]`` - extract the columms of ``a`` where vector v > 0.5 * - ``a(:,find(v>0.5))`` - - ``a[:,v.T>0.5]`` + - ``a[:, v.T > 0.5]`` - extract the columms of ``a`` where column vector v > 0.5 * - ``a(a<0.5)=0`` - - ``a[a<0.5]=0`` + - ``a[a < 0.5]=0`` - ``a`` with elements less than 0.5 zeroed out * - ``a .* (a>0.5)`` - - ``a * (a>0.5)`` + - ``a * (a > 0.5)`` - ``a`` with elements less than 0.5 zeroed out * - ``a(:) = 3`` @@ -386,11 +333,11 @@ Linear Algebra Equivalents * - ``y=x`` - ``y = x.copy()`` - - numpy assigns by reference + - NumPy assigns by reference * - ``y=x(2,:)`` - - ``y = x[1,:].copy()`` - - numpy slices are by reference + - ``y = x[1, :].copy()`` + - NumPy slices are by reference * - ``y=x(:)`` - ``y = x.flatten()`` @@ -398,62 +345,74 @@ Linear Algebra Equivalents same data ordering as in MATLAB, use ``x.flatten('F')``. * - ``1:10`` - - ``arange(1.,11.)`` or ``r_[1.:11.]`` or ``r_[1:10:10j]`` + - ``np.arange(1., 11.)`` or ``np.r_[1.:11.]`` or ``np.r_[1:10:10j]`` - create an increasing vector (see note :ref:`RANGES `) * - ``0:9`` - - ``arange(10.)`` or ``r_[:10.]`` or ``r_[:9:10j]`` + - ``np.arange(10.)`` or ``np.r_[:10.]`` or ``np.r_[:9:10j]`` - create an increasing vector (see note :ref:`RANGES `) * - ``[1:10]'`` - - ``arange(1.,11.)[:, newaxis]`` + - ``np.arange(1.,11.)[:, np.newaxis]`` - create a column vector * - ``zeros(3,4)`` - - ``zeros((3,4))`` + - ``np.zeros((3, 4))`` - 3x4 two-dimensional array full of 64-bit floating point zeros * - ``zeros(3,4,5)`` - - ``zeros((3,4,5))`` + - ``np.zeros((3, 4, 5))`` - 3x4x5 three-dimensional array full of 64-bit floating point zeros * - ``ones(3,4)`` - - ``ones((3,4))`` + - ``np.ones((3, 4))`` - 3x4 two-dimensional array full of 64-bit floating point ones * - ``eye(3)`` - - ``eye(3)`` + - ``np.eye(3)`` - 3x3 identity matrix * - ``diag(a)`` - - ``diag(a)`` - - vector of diagonal elements of ``a`` + - ``np.diag(a)`` + - returns a vector of the diagonal elements of 2D array, ``a`` + + * - ``diag(v,0)`` + - ``np.diag(v, 0)`` + - returns a square diagonal matrix whose nonzero values are the elements of + vector, ``v`` - * - ``diag(a,0)`` - - ``diag(a,0)`` - - square diagonal matrix whose nonzero values are the elements of - ``a`` + * - .. code:: matlab + + rng(42,'twister') + rand(3,4) - * - ``rand(3,4)`` - - ``random.rand(3,4)`` or ``random.random_sample((3, 4))`` - - random 3x4 matrix + - :: + + from numpy.random import default_rng + rng = default_rng(42) + rng.random(3, 4) + + or older version: ``random.rand((3, 4))`` + + - generate a random 3x4 array with default random number generator and + seed = 42 * - ``linspace(1,3,4)`` - - ``linspace(1,3,4)`` + - ``np.linspace(1,3,4)`` - 4 equally spaced samples between 1 and 3, inclusive * - ``[x,y]=meshgrid(0:8,0:5)`` - - ``mgrid[0:9.,0:6.]`` or ``meshgrid(r_[0:9.],r_[0:6.]`` + - ``np.mgrid[0:9.,0:6.]`` or ``np.meshgrid(r_[0:9.],r_[0:6.]`` - two 2D arrays: one of x values, the other of y values * - - - ``ogrid[0:9.,0:6.]`` or ``ix_(r_[0:9.],r_[0:6.]`` + - ``ogrid[0:9.,0:6.]`` or ``np.ix_(np.r_[0:9.],np.r_[0:6.]`` - the best way to eval functions on a grid * - ``[x,y]=meshgrid([1,2,4],[2,4,5])`` - - ``meshgrid([1,2,4],[2,4,5])`` + - ``np.meshgrid([1,2,4],[2,4,5])`` - * - @@ -461,37 +420,38 @@ Linear Algebra Equivalents - the best way to eval functions on a grid * - ``repmat(a, m, n)`` - - ``tile(a, (m, n))`` + - ``np.tile(a, (m, n))`` - create m by n copies of ``a`` * - ``[a b]`` - - ``concatenate((a,b),1)`` or ``hstack((a,b))`` or - ``column_stack((a,b))`` or ``c_[a,b]`` + - ``np.concatenate((a,b),1)`` or ``np.hstack((a,b))`` or + ``np.column_stack((a,b))`` or ``np.c_[a,b]`` - concatenate columns of ``a`` and ``b`` * - ``[a; b]`` - - ``concatenate((a,b))`` or ``vstack((a,b))`` or ``r_[a,b]`` + - ``np.concatenate((a,b))`` or ``np.vstack((a,b))`` or ``np.r_[a,b]`` - concatenate rows of ``a`` and ``b`` * - ``max(max(a))`` - - ``a.max()`` - - maximum element of ``a`` (with ndims(a)<=2 for MATLAB) + - ``a.max()`` or ``np.nanmax(a)`` + - maximum element of ``a`` (with ndims(a)<=2 for MATLAB, if there are + NaN's, ``nanmax`` will ignore these and return largest value) * - ``max(a)`` - ``a.max(0)`` - - maximum element of each column of matrix ``a`` + - maximum element of each column of array ``a`` * - ``max(a,[],2)`` - ``a.max(1)`` - - maximum element of each row of matrix ``a`` + - maximum element of each row of array ``a`` * - ``max(a,b)`` - - ``maximum(a, b)`` + - ``np.maximum(a, b)`` - compares ``a`` and ``b`` element-wise, and returns the maximum value from each pair * - ``norm(v)`` - - ``sqrt(v @ v)`` or ``np.linalg.norm(v)`` + - ``np.sqrt(v @ v)`` or ``np.linalg.norm(v)`` - L2 norm of vector ``v`` * - ``a & b`` @@ -500,7 +460,7 @@ Linear Algebra Equivalents LOGICOPS ` * - ``a | b`` - - ``logical_or(a,b)`` + - ``np.logical_or(a,b)`` - element-by-element OR operator (NumPy ufunc) :ref:`See note LOGICOPS ` @@ -514,90 +474,99 @@ Linear Algebra Equivalents * - ``inv(a)`` - ``linalg.inv(a)`` - - inverse of square matrix ``a`` + - inverse of square 2D array ``a`` * - ``pinv(a)`` - ``linalg.pinv(a)`` - - pseudo-inverse of matrix ``a`` + - pseudo-inverse of 2D array ``a`` * - ``rank(a)`` - ``linalg.matrix_rank(a)`` - - matrix rank of a 2D array / matrix ``a`` + - matrix rank of a 2D array ``a`` * - ``a\b`` - - ``linalg.solve(a,b)`` if ``a`` is square; ``linalg.lstsq(a,b)`` + - ``linalg.solve(a, b)`` if ``a`` is square; ``linalg.lstsq(a, b)`` otherwise - solution of a x = b for x * - ``b/a`` - - Solve a.T x.T = b.T instead + - Solve ``a.T x.T = b.T`` instead - solution of x a = b for x * - ``[U,S,V]=svd(a)`` - ``U, S, Vh = linalg.svd(a), V = Vh.T`` - singular value decomposition of ``a`` - * - ``chol(a)`` - - ``linalg.cholesky(a).T`` - - cholesky factorization of a matrix (``chol(a)`` in MATLAB returns an - upper triangular matrix, but ``linalg.cholesky(a)`` returns a lower - triangular matrix) + * - ``c=chol(a)`` where ``a==c'*c`` + - ``c = linalg.cholesky(a)`` where ``a == c@c.T`` + - Cholesky factorization of a 2D array (``chol(a)`` in MATLAB returns an + upper triangular 2D array, but :func:`~scipy.linalg.cholesky` returns a lower + triangular 2D array) * - ``[V,D]=eig(a)`` - ``D,V = linalg.eig(a)`` - - eigenvalues and eigenvectors of ``a`` + - eigenvalues :math:`\lambda` and eigenvectors :math:`\bar{v}` of ``a``, + where :math:`\lambda\bar{v}=\mathbf{a}\bar{v}` * - ``[V,D]=eig(a,b)`` - - ``D,V = scipy.linalg.eig(a,b)`` - - eigenvalues and eigenvectors of ``a``, ``b`` + - ``D,V = linalg.eig(a, b)`` + - eigenvalues :math:`\lambda` and eigenvectors :math:`\bar{v}` of + ``a``, ``b`` + where :math:`\lambda\mathbf{b}\bar{v}=\mathbf{a}\bar{v}` - * - ``[V,D]=eigs(a,k)`` - - - - find the ``k`` largest eigenvalues and eigenvectors of ``a`` + * - ``[V,D]=eigs(a,3)`` + - ``D,V = eigs(a, k = 3)`` + - find the ``k=3`` largest eigenvalues and eigenvectors of 2D array, ``a`` * - ``[Q,R,P]=qr(a,0)`` - - ``Q,R = scipy.linalg.qr(a)`` + - ``Q,R = linalg.qr(a)`` - QR decomposition - * - ``[L,U,P]=lu(a)`` - - ``L,U = scipy.linalg.lu(a)`` or ``LU,P=scipy.linalg.lu_factor(a)`` - - LU decomposition (note: P(MATLAB) == transpose(P(numpy)) ) + * - ``[L,U,P]=lu(a)`` where ``a==P'*L*U`` + - ``P,L,U = linalg.lu(a)`` where ``a == P@L@U`` + - LU decomposition (note: P(MATLAB) == transpose(P(NumPy))) * - ``conjgrad`` - - ``scipy.sparse.linalg.cg`` + - ``cg`` - Conjugate gradients solver * - ``fft(a)`` - - ``fft(a)`` + - ``np.fft(a)`` - Fourier transform of ``a`` * - ``ifft(a)`` - - ``ifft(a)`` + - ``np.ifft(a)`` - inverse Fourier transform of ``a`` * - ``sort(a)`` - - ``sort(a)`` or ``a.sort()`` - - sort the matrix + - ``np.sort(a)`` or ``a.sort(axis=0)`` + - sort each column of a 2D array, ``a`` - * - ``[b,I] = sortrows(a,i)`` - - ``I = argsort(a[:,i]), b=a[I,:]`` - - sort the rows of the matrix + * - ``sort(a, 2)`` + - ``np.sort(a, axis = 1)`` or ``a.sort(axis = 1)`` + - sort the each row of 2D array, ``a`` - * - ``regress(y,X)`` - - ``linalg.lstsq(X,y)`` - - multilinear regression + * - ``[b,I]=sortrows(a,1)`` + - ``I = np.argsort(a[:, 0]); b = a[I,:]`` + - save the array ``a`` as array ``b`` with rows sorted by the first column + + * - ``x = Z\y`` + - ``x = linalg.lstsq(Z, y)`` + - perform a linear regression of the form :math:`\mathbf{Zx}=\mathbf{y}` * - ``decimate(x, q)`` - - ``scipy.signal.resample(x, len(x)/q)`` + - ``signal.resample(x, np.ceil(len(x)/q))`` - downsample with low-pass filtering * - ``unique(a)`` - - ``unique(a)`` - - + - ``np.unique(a)`` + - a vector of unique values in array ``a`` * - ``squeeze(a)`` - ``a.squeeze()`` - - + - remove singleton dimensions of array ``a``. Note that MATLAB will always + return arrays of 2D or higher while NumPy will return arrays of 0D or + higher .. _numpy-for-matlab-users.notes: @@ -605,11 +574,11 @@ Notes ===== \ **Submatrix**: Assignment to a submatrix can be done with lists of -indexes using the ``ix_`` command. E.g., for 2d array ``a``, one might -do: ``ind=[1,3]; a[np.ix_(ind,ind)]+=100``. +indices using the ``ix_`` command. E.g., for 2D array ``a``, one might +do: ``ind=[1, 3]; a[np.ix_(ind, ind)] += 100``. \ **HELP**: There is no direct equivalent of MATLAB's ``which`` command, -but the commands ``help`` and ``source`` will usually list the filename +but the commands :func:`help` and :func:`numpy.source` will usually list the filename where the function is located. Python also has an ``inspect`` module (do ``import inspect``) which provides a ``getfile`` that often works. @@ -627,35 +596,35 @@ Dijkstra ; MATLAB's is the reverse. + operators like ``<`` and ``>``; MATLAB's is the reverse. If you know you have boolean arguments, you can get away with using -NumPy's bitwise operators, but be careful with parentheses, like this: z -= (x > 1) & (x < 2). The absence of NumPy operator forms of logical\_and -and logical\_or is an unfortunate consequence of Python's design. +NumPy's bitwise operators, but be careful with parentheses, like this: ``z += (x > 1) & (x < 2)``. The absence of NumPy operator forms of ``logical_and`` +and ``logical_or`` is an unfortunate consequence of Python's design. **RESHAPE and LINEAR INDEXING**: MATLAB always allows multi-dimensional arrays to be accessed using scalar or linear indices, NumPy does not. -Linear indices are common in MATLAB programs, e.g. find() on a matrix +Linear indices are common in MATLAB programs, e.g. ``find()`` on a matrix returns them, whereas NumPy's find behaves differently. When converting MATLAB code it might be necessary to first reshape a matrix to a linear sequence, perform some indexing operations and then reshape back. As @@ -664,11 +633,132 @@ possible to do this fairly efficiently. Note that the scan order used by reshape in NumPy defaults to the 'C' order, whereas MATLAB uses the Fortran order. If you are simply converting to a linear sequence and back this doesn't matter. But if you are converting reshapes from MATLAB -code which relies on the scan order, then this MATLAB code: z = -reshape(x,3,4); should become z = x.reshape(3,4,order='F').copy() in +code which relies on the scan order, then this MATLAB code: ``z = +reshape(x,3,4);`` should become ``z = x.reshape(3,4,order='F').copy()`` in NumPy. -Customizing Your Environment +'array' or 'matrix'? Which should I use? +======================================== + +Historically, NumPy has provided a special matrix type, `np.matrix`, which +is a subclass of ndarray which makes binary operations linear algebra +operations. You may see it used in some existing code instead of `np.array`. +So, which one to use? + +Short answer +------------ + +**Use arrays**. + +- They support multidimensional array algebra that is supported in MATLAB +- They are the standard vector/matrix/tensor type of NumPy. Many NumPy + functions return arrays, not matrices. +- There is a clear distinction between element-wise operations and + linear algebra operations. +- You can have standard vectors or row/column vectors if you like. + +Until Python 3.5 the only disadvantage of using the array type was that you +had to use ``dot`` instead of ``*`` to multiply (reduce) two tensors +(scalar product, matrix vector multiplication etc.). Since Python 3.5 you +can use the matrix multiplication ``@`` operator. + +Given the above, we intend to deprecate ``matrix`` eventually. + +Long answer +----------- + +NumPy contains both an ``array`` class and a ``matrix`` class. The +``array`` class is intended to be a general-purpose n-dimensional array +for many kinds of numerical computing, while ``matrix`` is intended to +facilitate linear algebra computations specifically. In practice there +are only a handful of key differences between the two. + +- Operators ``*`` and ``@``, functions ``dot()``, and ``multiply()``: + + - For ``array``, **``*`` means element-wise multiplication**, while + **``@`` means matrix multiplication**; they have associated functions + ``multiply()`` and ``dot()``. (Before Python 3.5, ``@`` did not exist + and one had to use ``dot()`` for matrix multiplication). + - For ``matrix``, **``*`` means matrix multiplication**, and for + element-wise multiplication one has to use the ``multiply()`` function. + +- Handling of vectors (one-dimensional arrays) + + - For ``array``, the **vector shapes 1xN, Nx1, and N are all different + things**. Operations like ``A[:,1]`` return a one-dimensional array of + shape N, not a two-dimensional array of shape Nx1. Transpose on a + one-dimensional ``array`` does nothing. + - For ``matrix``, **one-dimensional arrays are always upconverted to 1xN + or Nx1 matrices** (row or column vectors). ``A[:,1]`` returns a + two-dimensional matrix of shape Nx1. + +- Handling of higher-dimensional arrays (ndim > 2) + + - ``array`` objects **can have number of dimensions > 2**; + - ``matrix`` objects **always have exactly two dimensions**. + +- Convenience attributes + + - ``array`` **has a .T attribute**, which returns the transpose of + the data. + - ``matrix`` **also has .H, .I, and .A attributes**, which return + the conjugate transpose, inverse, and ``asarray()`` of the matrix, + respectively. + +- Convenience constructor + + - The ``array`` constructor **takes (nested) Python sequences as + initializers**. As in, ``array([[1,2,3],[4,5,6]])``. + - The ``matrix`` constructor additionally **takes a convenient + string initializer**. As in ``matrix("[1 2 3; 4 5 6]")``. + +There are pros and cons to using both: + +- ``array`` + + - ``:)`` Element-wise multiplication is easy: ``A*B``. + - ``:(`` You have to remember that matrix multiplication has its own + operator, ``@``. + - ``:)`` You can treat one-dimensional arrays as *either* row or column + vectors. ``A @ v`` treats ``v`` as a column vector, while + ``v @ A`` treats ``v`` as a row vector. This can save you having to + type a lot of transposes. + - ``:)`` ``array`` is the "default" NumPy type, so it gets the most + testing, and is the type most likely to be returned by 3rd party + code that uses NumPy. + - ``:)`` Is quite at home handling data of any number of dimensions. + - ``:)`` Closer in semantics to tensor algebra, if you are familiar + with that. + - ``:)`` *All* operations (``*``, ``/``, ``+``, ``-`` etc.) are + element-wise. + - ``:(`` Sparse matrices from ``scipy.sparse`` do not interact as well + with arrays. + +- ``matrix`` + + - ``:\\`` Behavior is more like that of MATLAB matrices. + - ``<:(`` Maximum of two-dimensional. To hold three-dimensional data you + need ``array`` or perhaps a Python list of ``matrix``. + - ``<:(`` Minimum of two-dimensional. You cannot have vectors. They must be + cast as single-column or single-row matrices. + - ``<:(`` Since ``array`` is the default in NumPy, some functions may + return an ``array`` even if you give them a ``matrix`` as an + argument. This shouldn't happen with NumPy functions (if it does + it's a bug), but 3rd party code based on NumPy may not honor type + preservation like NumPy does. + - ``:)`` ``A*B`` is matrix multiplication, so it looks just like you write + it in linear algebra (For Python >= 3.5 plain arrays have the same + convenience with the ``@`` operator). + - ``<:(`` Element-wise multiplication requires calling a function, + ``multiply(A,B)``. + - ``<:(`` The use of operator overloading is a bit illogical: ``*`` + does not work element-wise but ``/`` does. + - Interaction with ``scipy.sparse`` is a bit cleaner. + +The ``array`` is thus much more advisable to use. Indeed, we intend to +deprecate ``matrix`` eventually. + +Customizing your environment ============================ In MATLAB the main tool available to you for customizing the @@ -696,26 +786,39 @@ this is just an example, not a statement of "best practices"): # Make all numpy available via shorter 'np' prefix import numpy as np - # Make all matlib functions accessible at the top level via M.func() - import numpy.matlib as M - # Make some matlib functions accessible directly at the top level via, e.g. rand(3,3) - from numpy.matlib import rand,zeros,ones,empty,eye + # + # Make the SciPy linear algebra functions available as linalg.func() + # e.g. linalg.lu, linalg.eig (for general l*B@u==A@u solution) + from scipy import linalg + # # Define a Hermitian function def hermitian(A, **kwargs): - return np.transpose(A,**kwargs).conj() - # Make some shortcuts for transpose,hermitian: - # np.transpose(A) --> T(A) + return np.conj(A,**kwargs).T + # Make a shortcut for hermitian: # hermitian(A) --> H(A) - T = np.transpose H = hermitian +To use the deprecated `matrix` and other `matlib` functions: + +:: + + # Make all matlib functions accessible at the top level via M.func() + import numpy.matlib as M + # Make some matlib functions accessible directly at the top level via, e.g. rand(3,3) + from numpy.matlib import matrix,rand,zeros,ones,empty,eye + Links ===== -See http://mathesaurus.sf.net/ for another MATLAB/NumPy -cross-reference. +Another somewhat outdated MATLAB/NumPy cross-reference can be found at +http://mathesaurus.sf.net/ -An extensive list of tools for scientific work with python can be +An extensive list of tools for scientific work with Python can be found in the `topical software page `__. -MATLAB® and SimuLink® are registered trademarks of The MathWorks. +See +`List of Python software: scripting +`_ +for a list of softwares that use Python as a scripting language + +MATLAB® and SimuLink® are registered trademarks of The MathWorks, Inc. diff --git a/doc/source/user/quickstart.rst b/doc/source/user/quickstart.rst index b1af8188675e..387a7f7151c5 100644 --- a/doc/source/user/quickstart.rst +++ b/doc/source/user/quickstart.rst @@ -1,5 +1,5 @@ =================== -Quickstart tutorial +NumPy quickstart =================== .. currentmodule:: numpy @@ -12,26 +12,24 @@ Quickstart tutorial Prerequisites ============= -Before reading this tutorial you should know a bit of Python. If you -would like to refresh your memory, take a look at the `Python +You'll need to know a bit of Python. For a refresher, see the `Python tutorial `__. -If you wish to work the examples in this tutorial, you must also have -some software installed on your computer. Please see -https://scipy.org/install.html for instructions. +To work the examples, you'll need ``matplotlib`` installed +in addition to NumPy. **Learner profile** -This tutorial is intended as a quick overview of -algebra and arrays in NumPy and want to understand how n-dimensional +This is a quick overview of +algebra and arrays in NumPy. It demonstrates how n-dimensional (:math:`n>=2`) arrays are represented and can be manipulated. In particular, if you don't know how to apply common functions to n-dimensional arrays (without using for-loops), or if you want to understand axis and shape properties for -n-dimensional arrays, this tutorial might be of help. +n-dimensional arrays, this article might be of help. **Learning Objectives** -After this tutorial, you should be able to: +After reading, you should be able to: - Understand the difference between one-, two- and n-dimensional arrays in NumPy; diff --git a/doc/source/user/setting-up.rst b/doc/source/user/setting-up.rst deleted file mode 100644 index 7ca3a365c4e7..000000000000 --- a/doc/source/user/setting-up.rst +++ /dev/null @@ -1,10 +0,0 @@ -********** -Setting up -********** - -.. toctree:: - :maxdepth: 1 - - whatisnumpy - install - troubleshooting-importerror diff --git a/doc/source/user/troubleshooting-importerror.rst b/doc/source/user/troubleshooting-importerror.rst index 7d4846f7794f..1f99491a1e95 100644 --- a/doc/source/user/troubleshooting-importerror.rst +++ b/doc/source/user/troubleshooting-importerror.rst @@ -1,3 +1,11 @@ +:orphan: + +.. Reason for orphan: This page is referenced by the installation + instructions, which have moved from Sphinx to https://numpy.org/install. + All install links in Sphinx now point there, leaving no Sphinx references + to this page. + + *************************** Troubleshooting ImportError *************************** @@ -69,7 +77,7 @@ or conda. Using Eclipse/PyDev with Anaconda/conda Python (or environments) ---------------------------------------------------------------- -Please see the +Please see the `Anaconda Documentation `_ on how to properly configure Eclipse/PyDev to use Anaconda Python with specific conda environments. diff --git a/doc/sphinxext b/doc/sphinxext deleted file mode 160000 index b4c5fd17e2b8..000000000000 --- a/doc/sphinxext +++ /dev/null @@ -1 +0,0 @@ -Subproject commit b4c5fd17e2b85c2416a5e586933eee353b58bf7c diff --git a/doc_requirements.txt b/doc_requirements.txt index e2694ba12808..f69cc5dcb406 100644 --- a/doc_requirements.txt +++ b/doc_requirements.txt @@ -1,6 +1,7 @@ sphinx>=2.2.0,<3.0 +numpydoc==1.1.0 ipython scipy matplotlib pandas -pydata-sphinx-theme==0.4.0 +pydata-sphinx-theme==0.4.1 diff --git a/numpy/__init__.pyi b/numpy/__init__.pyi index bf54207a4637..75021634289a 100644 --- a/numpy/__init__.pyi +++ b/numpy/__init__.pyi @@ -4,7 +4,48 @@ import datetime as dt from abc import abstractmethod from numpy.core._internal import _ctypes -from numpy.typing import ArrayLike, DtypeLike, _Shape, _ShapeLike +from numpy.typing import ( + ArrayLike, + DTypeLike, + _Shape, + _ShapeLike, + _CharLike, + _BoolLike, + _IntLike, + _FloatLike, + _ComplexLike, + _NumberLike, + _SupportsDType, + _VoidDTypeLike, + NBitBase, + _64Bit, + _32Bit, + _16Bit, + _8Bit, +) +from numpy.typing._callable import ( + _BoolOp, + _BoolBitOp, + _BoolSub, + _BoolTrueDiv, + _BoolMod, + _BoolDivMod, + _TD64Div, + _IntTrueDiv, + _UnsignedIntOp, + _UnsignedIntBitOp, + _UnsignedIntMod, + _UnsignedIntDivMod, + _SignedIntOp, + _SignedIntBitOp, + _SignedIntMod, + _SignedIntDivMod, + _FloatOp, + _FloatMod, + _FloatDivMod, + _ComplexOp, + _NumberOp, +) from typing import ( Any, @@ -22,8 +63,6 @@ from typing import ( overload, Sequence, Sized, - SupportsAbs, - SupportsBytes, SupportsComplex, SupportsFloat, SupportsInt, @@ -35,17 +74,15 @@ from typing import ( ) if sys.version_info >= (3, 8): - from typing import Literal, Protocol, SupportsIndex + from typing import Literal, Protocol, SupportsIndex, Final else: - from typing_extensions import Literal, Protocol + from typing_extensions import Literal, Protocol, Final class SupportsIndex(Protocol): def __index__(self) -> int: ... # Ensures that the stubs are picked up -from . import ( +from numpy import ( char, - compat, - core, ctypeslib, emath, fft, @@ -66,31 +103,760 @@ from numpy.core.function_base import ( geomspace, ) +from numpy.core.fromnumeric import ( + take, + reshape, + choose, + repeat, + put, + swapaxes, + transpose, + partition, + argpartition, + sort, + argsort, + argmax, + argmin, + searchsorted, + resize, + squeeze, + diagonal, + trace, + ravel, + nonzero, + shape, + compress, + clip, + sum, + all, + any, + cumsum, + ptp, + amax, + amin, + prod, + cumprod, + ndim, + size, + around, + mean, + std, + var, +) + +from numpy.core._asarray import ( + asarray as asarray, + asanyarray as asanyarray, + ascontiguousarray as ascontiguousarray, + asfortranarray as asfortranarray, + require as require, +) + +from numpy.core._type_aliases import ( + sctypes as sctypes, + sctypeDict as sctypeDict, +) + +from numpy.core.numeric import ( + zeros_like as zeros_like, + ones as ones, + ones_like as ones_like, + empty_like as empty_like, + full as full, + full_like as full_like, + count_nonzero as count_nonzero, + isfortran as isfortran, + argwhere as argwhere, + flatnonzero as flatnonzero, + correlate as correlate, + convolve as convolve, + outer as outer, + tensordot as tensordot, + roll as roll, + rollaxis as rollaxis, + moveaxis as moveaxis, + cross as cross, + indices as indices, + fromfunction as fromfunction, + isscalar as isscalar, + binary_repr as binary_repr, + base_repr as base_repr, + identity as identity, + allclose as allclose, + isclose as isclose, + array_equal as array_equal, + array_equiv as array_equiv, +) + +from numpy.core.numerictypes import ( + maximum_sctype as maximum_sctype, + issctype as issctype, + obj2sctype as obj2sctype, + issubclass_ as issubclass_, + issubsctype as issubsctype, + issubdtype as issubdtype, + sctype2char as sctype2char, + find_common_type as find_common_type, +) + +from numpy.core.shape_base import ( + atleast_1d as atleast_1d, + atleast_2d as atleast_2d, + atleast_3d as atleast_3d, + block as block, + hstack as hstack, + stack as stack, + vstack as vstack, +) + # Add an object to `__all__` if their stubs are defined in an external file; # their stubs will not be recognized otherwise. # NOTE: This is redundant for objects defined within this file. -__all__ = ["linspace", "logspace", "geomspace"] +__all__ = [ + "linspace", + "logspace", + "geomspace", + "take", + "reshape", + "choose", + "repeat", + "put", + "swapaxes", + "transpose", + "partition", + "argpartition", + "sort", + "argsort", + "argmax", + "argmin", + "searchsorted", + "resize", + "squeeze", + "diagonal", + "trace", + "ravel", + "nonzero", + "shape", + "compress", + "clip", + "sum", + "all", + "any", + "cumsum", + "ptp", + "amax", + "amin", + "prod", + "cumprod", + "ndim", + "size", + "around", + "mean", + "std", + "var", +] -# TODO: remove when the full numpy namespace is defined -def __getattr__(name: str) -> Any: ... +DataSource: Any +MachAr: Any +ScalarType: Any +angle: Any +append: Any +apply_along_axis: Any +apply_over_axes: Any +arange: Any +array2string: Any +array_repr: Any +array_split: Any +array_str: Any +asarray_chkfinite: Any +asfarray: Any +asmatrix: Any +asscalar: Any +average: Any +bartlett: Any +bincount: Any +bitwise_not: Any +blackman: Any +bmat: Any +bool8: Any +broadcast: Any +broadcast_arrays: Any +broadcast_to: Any +busday_count: Any +busday_offset: Any +busdaycalendar: Any +byte: Any +byte_bounds: Any +bytes0: Any +c_: Any +can_cast: Any +cast: Any +cdouble: Any +cfloat: Any +chararray: Any +clongdouble: Any +clongfloat: Any +column_stack: Any +common_type: Any +compare_chararrays: Any +complex256: Any +complex_: Any +concatenate: Any +conj: Any +copy: Any +copyto: Any +corrcoef: Any +cov: Any +csingle: Any +cumproduct: Any +datetime_as_string: Any +datetime_data: Any +delete: Any +deprecate: Any +deprecate_with_doc: Any +diag: Any +diag_indices: Any +diag_indices_from: Any +diagflat: Any +diff: Any +digitize: Any +disp: Any +divide: Any +dot: Any +double: Any +dsplit: Any +dstack: Any +ediff1d: Any +einsum: Any +einsum_path: Any +errstate: Any +expand_dims: Any +extract: Any +eye: Any +fill_diagonal: Any +finfo: Any +fix: Any +flip: Any +fliplr: Any +flipud: Any +float128: Any +float_: Any +format_float_positional: Any +format_float_scientific: Any +format_parser: Any +frombuffer: Any +fromfile: Any +fromiter: Any +frompyfunc: Any +fromregex: Any +fromstring: Any +genfromtxt: Any +get_include: Any +get_printoptions: Any +getbufsize: Any +geterr: Any +geterrcall: Any +geterrobj: Any +gradient: Any +half: Any +hamming: Any +hanning: Any +histogram: Any +histogram2d: Any +histogram_bin_edges: Any +histogramdd: Any +hsplit: Any +i0: Any +iinfo: Any +imag: Any +in1d: Any +index_exp: Any +info: Any +inner: Any +insert: Any +int0: Any +int_: Any +intc: Any +interp: Any +intersect1d: Any +intp: Any +is_busday: Any +iscomplex: Any +iscomplexobj: Any +isin: Any +isneginf: Any +isposinf: Any +isreal: Any +isrealobj: Any +iterable: Any +ix_: Any +kaiser: Any +kron: Any +lexsort: Any +load: Any +loads: Any +loadtxt: Any +longcomplex: Any +longdouble: Any +longfloat: Any +longlong: Any +lookfor: Any +mafromtxt: Any +mask_indices: Any +mat: Any +matrix: Any +max: Any +may_share_memory: Any +median: Any +memmap: Any +meshgrid: Any +mgrid: Any +min: Any +min_scalar_type: Any +mintypecode: Any +mod: Any +msort: Any +nan_to_num: Any +nanargmax: Any +nanargmin: Any +nancumprod: Any +nancumsum: Any +nanmax: Any +nanmean: Any +nanmedian: Any +nanmin: Any +nanpercentile: Any +nanprod: Any +nanquantile: Any +nanstd: Any +nansum: Any +nanvar: Any +nbytes: Any +ndenumerate: Any +ndfromtxt: Any +ndindex: Any +nditer: Any +nested_iters: Any +newaxis: Any +numarray: Any +object0: Any +ogrid: Any +packbits: Any +pad: Any +percentile: Any +piecewise: Any +place: Any +poly: Any +poly1d: Any +polyadd: Any +polyder: Any +polydiv: Any +polyfit: Any +polyint: Any +polymul: Any +polysub: Any +polyval: Any +printoptions: Any +product: Any +promote_types: Any +put_along_axis: Any +putmask: Any +quantile: Any +r_: Any +ravel_multi_index: Any +real: Any +real_if_close: Any +recarray: Any +recfromcsv: Any +recfromtxt: Any +record: Any +result_type: Any +roots: Any +rot90: Any +round: Any +round_: Any +row_stack: Any +s_: Any +save: Any +savetxt: Any +savez: Any +savez_compressed: Any +select: Any +set_printoptions: Any +set_string_function: Any +setbufsize: Any +setdiff1d: Any +seterr: Any +seterrcall: Any +seterrobj: Any +setxor1d: Any +shares_memory: Any +short: Any +show_config: Any +sinc: Any +single: Any +singlecomplex: Any +sort_complex: Any +source: Any +split: Any +string_: Any +take_along_axis: Any +tile: Any +trapz: Any +tri: Any +tril: Any +tril_indices: Any +tril_indices_from: Any +trim_zeros: Any +triu: Any +triu_indices: Any +triu_indices_from: Any +typeDict: Any +typecodes: Any +typename: Any +ubyte: Any +uint: Any +uint0: Any +uintc: Any +uintp: Any +ulonglong: Any +union1d: Any +unique: Any +unpackbits: Any +unravel_index: Any +unwrap: Any +ushort: Any +vander: Any +vdot: Any +vectorize: Any +void0: Any +vsplit: Any +where: Any +who: Any _NdArraySubClass = TypeVar("_NdArraySubClass", bound=ndarray) +_DTypeScalar = TypeVar("_DTypeScalar", bound=generic) _ByteOrder = Literal["S", "<", ">", "=", "|", "L", "B", "N", "I"] -class dtype: +class dtype(Generic[_DTypeScalar]): names: Optional[Tuple[str, ...]] - def __init__( - self, - dtype: DtypeLike, + # Overload for subclass of generic + @overload + def __new__( + cls, + dtype: Type[_DTypeScalar], align: bool = ..., copy: bool = ..., - ) -> None: ... - def __eq__(self, other: DtypeLike) -> bool: ... - def __ne__(self, other: DtypeLike) -> bool: ... - def __gt__(self, other: DtypeLike) -> bool: ... - def __ge__(self, other: DtypeLike) -> bool: ... - def __lt__(self, other: DtypeLike) -> bool: ... - def __le__(self, other: DtypeLike) -> bool: ... + ) -> dtype[_DTypeScalar]: ... + # Overloads for string aliases, Python types, and some assorted + # other special cases. Order is sometimes important because of the + # subtype relationships + # + # bool < int < float < complex + # + # so we have to make sure the overloads for the narrowest type is + # first. + @overload + def __new__( + cls, + dtype: Union[ + Type[bool], + Literal[ + "?", + "=?", + "?", + "bool", + "bool_", + ], + ], + align: bool = ..., + copy: bool = ..., + ) -> dtype[bool_]: ... + @overload + def __new__( + cls, + dtype: Literal[ + "uint8", + "u1", + "=u1", + "u1", + ], + align: bool = ..., + copy: bool = ..., + ) -> dtype[uint8]: ... + @overload + def __new__( + cls, + dtype: Literal[ + "uint16", + "u2", + "=u2", + "u2", + ], + align: bool = ..., + copy: bool = ..., + ) -> dtype[uint16]: ... + @overload + def __new__( + cls, + dtype: Literal[ + "uint32", + "u4", + "=u4", + "u4", + ], + align: bool = ..., + copy: bool = ..., + ) -> dtype[uint32]: ... + @overload + def __new__( + cls, + dtype: Literal[ + "uint64", + "u8", + "=u8", + "u8", + ], + align: bool = ..., + copy: bool = ..., + ) -> dtype[uint64]: ... + @overload + def __new__( + cls, + dtype: Literal[ + "int8", + "i1", + "=i1", + "i1", + ], + align: bool = ..., + copy: bool = ..., + ) -> dtype[int8]: ... + @overload + def __new__( + cls, + dtype: Literal[ + "int16", + "i2", + "=i2", + "i2", + ], + align: bool = ..., + copy: bool = ..., + ) -> dtype[int16]: ... + @overload + def __new__( + cls, + dtype: Literal[ + "int32", + "i4", + "=i4", + "i4", + ], + align: bool = ..., + copy: bool = ..., + ) -> dtype[int32]: ... + @overload + def __new__( + cls, + dtype: Literal[ + "int64", + "i8", + "=i8", + "i8", + ], + align: bool = ..., + copy: bool = ..., + ) -> dtype[int64]: ... + # "int"/int resolve to int_, which is system dependent and as of + # now untyped. Long-term we'll do something fancier here. + @overload + def __new__( + cls, + dtype: Union[Type[int], Literal["int"]], + align: bool = ..., + copy: bool = ..., + ) -> dtype: ... + @overload + def __new__( + cls, + dtype: Literal[ + "float16", + "f4", + "=f4", + "f4", + "e", + "=e", + "e", + "half", + ], + align: bool = ..., + copy: bool = ..., + ) -> dtype[float16]: ... + @overload + def __new__( + cls, + dtype: Literal[ + "float32", + "f4", + "=f4", + "f4", + "f", + "=f", + "f", + "single", + ], + align: bool = ..., + copy: bool = ..., + ) -> dtype[float32]: ... + @overload + def __new__( + cls, + dtype: Union[ + None, + Type[float], + Literal[ + "float64", + "f8", + "=f8", + "f8", + "d", + "d", + "float", + "double", + "float_", + ], + ], + align: bool = ..., + copy: bool = ..., + ) -> dtype[float64]: ... + @overload + def __new__( + cls, + dtype: Literal[ + "complex64", + "c8", + "=c8", + "c8", + "F", + "=F", + "F", + ], + align: bool = ..., + copy: bool = ..., + ) -> dtype[complex64]: ... + @overload + def __new__( + cls, + dtype: Union[ + Type[complex], + Literal[ + "complex128", + "c16", + "=c16", + "c16", + "D", + "=D", + "D", + ], + ], + align: bool = ..., + copy: bool = ..., + ) -> dtype[complex128]: ... + @overload + def __new__( + cls, + dtype: Union[ + Type[bytes], + Literal[ + "S", + "=S", + "S", + "bytes", + "bytes_", + "bytes0", + ], + ], + align: bool = ..., + copy: bool = ..., + ) -> dtype[bytes_]: ... + @overload + def __new__( + cls, + dtype: Union[ + Type[str], + Literal[ + "U", + "=U", + # U intentionally not included; they are not + # the same dtype and which one dtype("U") translates + # to is platform-dependent. + "str", + "str_", + "str0", + ], + ], + align: bool = ..., + copy: bool = ..., + ) -> dtype[str_]: ... + # dtype of a dtype is the same dtype + @overload + def __new__( + cls, + dtype: dtype[_DTypeScalar], + align: bool = ..., + copy: bool = ..., + ) -> dtype[_DTypeScalar]: ... + # TODO: handle _SupportsDType better + @overload + def __new__( + cls, + dtype: _SupportsDType, + align: bool = ..., + copy: bool = ..., + ) -> dtype[Any]: ... + # Handle strings that can't be expressed as literals; i.e. s1, s2, ... + @overload + def __new__( + cls, + dtype: str, + align: bool = ..., + copy: bool = ..., + ) -> dtype[Any]: ... + # Catchall overload + @overload + def __new__( + cls, + dtype: _VoidDTypeLike, + align: bool = ..., + copy: bool = ..., + ) -> dtype[void]: ... + def __eq__(self, other: DTypeLike) -> bool: ... + def __ne__(self, other: DTypeLike) -> bool: ... + def __gt__(self, other: DTypeLike) -> bool: ... + def __ge__(self, other: DTypeLike) -> bool: ... + def __lt__(self, other: DTypeLike) -> bool: ... + def __le__(self, other: DTypeLike) -> bool: ... @property def alignment(self) -> int: ... @property @@ -139,7 +905,7 @@ class dtype: @property def type(self) -> Type[generic]: ... -_Dtype = dtype # to avoid name conflicts with ndarray.dtype +_DType = dtype # to avoid name conflicts with ndarray.dtype class _flagsobj: aligned: bool @@ -198,23 +964,36 @@ class flatiter(Generic[_ArraySelf]): def __getitem__( self, key: Union[_ArrayLikeInt, slice, ellipsis], ) -> _ArraySelf: ... - def __array__(self, __dtype: DtypeLike = ...) -> ndarray: ... + def __array__(self, __dtype: DTypeLike = ...) -> ndarray: ... _OrderKACF = Optional[Literal["K", "A", "C", "F"]] _OrderACF = Optional[Literal["A", "C", "F"]] _OrderCF = Optional[Literal["C", "F"]] +_ModeKind = Literal["raise", "wrap", "clip"] +_PartitionKind = Literal["introselect"] +_SortKind = Literal["quicksort", "mergesort", "heapsort", "stable"] +_SortSide = Literal["left", "right"] + +_ArrayLikeBool = Union[_BoolLike, Sequence[_BoolLike], ndarray] +_ArrayLikeIntOrBool = Union[ + _IntLike, + _BoolLike, + ndarray, + Sequence[_IntLike], + Sequence[_BoolLike], + Sequence[Sequence[Any]], # TODO: wait for support for recursive types +] + _ArraySelf = TypeVar("_ArraySelf", bound=_ArrayOrScalarCommon) -class _ArrayOrScalarCommon( - SupportsInt, SupportsFloat, SupportsComplex, SupportsBytes, SupportsAbs[Any] -): +class _ArrayOrScalarCommon: @property def T(self: _ArraySelf) -> _ArraySelf: ... @property def base(self) -> Optional[ndarray]: ... @property - def dtype(self) -> _Dtype: ... + def dtype(self) -> _DType: ... @property def data(self) -> memoryview: ... @property @@ -231,10 +1010,7 @@ class _ArrayOrScalarCommon( def shape(self) -> _Shape: ... @property def strides(self) -> _Shape: ... - def __array__(self, __dtype: DtypeLike = ...) -> ndarray: ... - def __int__(self) -> int: ... - def __float__(self) -> float: ... - def __complex__(self) -> complex: ... + def __array__(self, __dtype: DTypeLike = ...) -> ndarray: ... def __bool__(self) -> bool: ... def __bytes__(self) -> bytes: ... def __str__(self) -> str: ... @@ -247,40 +1023,9 @@ class _ArrayOrScalarCommon( def __ne__(self, other): ... def __gt__(self, other): ... def __ge__(self, other): ... - def __add__(self, other): ... - def __radd__(self, other): ... - def __sub__(self, other): ... - def __rsub__(self, other): ... - def __mul__(self, other): ... - def __rmul__(self, other): ... - def __truediv__(self, other): ... - def __rtruediv__(self, other): ... - def __floordiv__(self, other): ... - def __rfloordiv__(self, other): ... - def __mod__(self, other): ... - def __rmod__(self, other): ... - def __divmod__(self, other): ... - def __rdivmod__(self, other): ... - # NumPy's __pow__ doesn't handle a third argument - def __pow__(self, other): ... - def __rpow__(self, other): ... - def __lshift__(self, other): ... - def __rlshift__(self, other): ... - def __rshift__(self, other): ... - def __rrshift__(self, other): ... - def __and__(self, other): ... - def __rand__(self, other): ... - def __xor__(self, other): ... - def __rxor__(self, other): ... - def __or__(self, other): ... - def __ror__(self, other): ... - def __neg__(self: _ArraySelf) -> _ArraySelf: ... - def __pos__(self: _ArraySelf) -> _ArraySelf: ... - def __abs__(self: _ArraySelf) -> _ArraySelf: ... - def __invert__(self: _ArraySelf) -> _ArraySelf: ... def astype( self: _ArraySelf, - dtype: DtypeLike, + dtype: DTypeLike, order: _OrderKACF = ..., casting: _Casting = ..., subok: bool = ..., @@ -295,7 +1040,7 @@ class _ArrayOrScalarCommon( def flat(self: _ArraySelf) -> flatiter[_ArraySelf]: ... def flatten(self: _ArraySelf, order: _OrderKACF = ...) -> _ArraySelf: ... def getfield( - self: _ArraySelf, dtype: DtypeLike, offset: int = ... + self: _ArraySelf, dtype: DTypeLike, offset: int = ... ) -> _ArraySelf: ... @overload def item(self, *args: int) -> Any: ... @@ -326,6 +1071,8 @@ class _ArrayOrScalarCommon( ) -> _ArraySelf: ... def swapaxes(self: _ArraySelf, axis1: int, axis2: int) -> _ArraySelf: ... def tobytes(self, order: _OrderKACF = ...) -> bytes: ... + # NOTE: `tostring()` is deprecated and therefore excluded + # def tostring(self, order=...): ... def tofile( self, fid: Union[IO[bytes], str], sep: str = ..., format: str = ... ) -> None: ... @@ -338,10 +1085,10 @@ class _ArrayOrScalarCommon( @overload def view(self, type: Type[_NdArraySubClass]) -> _NdArraySubClass: ... @overload - def view(self: _ArraySelf, dtype: DtypeLike = ...) -> _ArraySelf: ... + def view(self: _ArraySelf, dtype: DTypeLike = ...) -> _ArraySelf: ... @overload def view( - self, dtype: DtypeLike, type: Type[_NdArraySubClass] + self, dtype: DTypeLike, type: Type[_NdArraySubClass] ) -> _NdArraySubClass: ... # TODO: Add proper signatures @@ -354,42 +1101,376 @@ class _ArrayOrScalarCommon( def __array_struct__(self): ... def __array_wrap__(array, context=...): ... def __setstate__(self, __state): ... - def all(self, axis=..., out=..., keepdims=...): ... - def any(self, axis=..., out=..., keepdims=...): ... - def argmax(self, axis=..., out=...): ... - def argmin(self, axis=..., out=...): ... - def argpartition(self, kth, axis=..., kind=..., order=...): ... - def argsort(self, axis=..., kind=..., order=...): ... - def choose(self, choices, out=..., mode=...): ... - def clip(self, min=..., max=..., out=..., **kwargs): ... - def compress(self, condition, axis=..., out=...): ... - def conj(self): ... - def conjugate(self): ... - def cumprod(self, axis=..., dtype=..., out=...): ... - def cumsum(self, axis=..., dtype=..., out=...): ... - def diagonal(self, offset=..., axis1=..., axis2=...): ... - def dot(self, b, out=...): ... - def max(self, axis=..., out=..., keepdims=..., initial=..., where=...): ... - def mean(self, axis=..., dtype=..., out=..., keepdims=...): ... - def min(self, axis=..., out=..., keepdims=..., initial=..., where=...): ... - def newbyteorder(self, new_order=...): ... - def nonzero(self): ... - def partition(self, kth, axis=..., kind=..., order=...): ... - def prod(self, axis=..., dtype=..., out=..., keepdims=..., initial=..., where=...): ... - def ptp(self, axis=..., out=..., keepdims=...): ... - def put(self, indices, values, mode=...): ... - def repeat(self, repeats, axis=...): ... - def round(self, decimals=..., out=...): ... - def searchsorted(self, v, side=..., sorter=...): ... - def setfield(self, val, dtype, offset=...): ... - def sort(self, axis=..., kind=..., order=...): ... - def std(self, axis=..., dtype=..., out=..., ddof=..., keepdims=...): ... - def sum(self, axis=..., dtype=..., out=..., keepdims=..., initial=..., where=...): ... - def take(self, indices, axis=..., out=..., mode=...): ... - # NOTE: `tostring()` is deprecated and therefore excluded - # def tostring(self, order=...): ... - def trace(self, offset=..., axis1=..., axis2=..., dtype=..., out=...): ... - def var(self, axis=..., dtype=..., out=..., ddof=..., keepdims=...): ... + # a `bool_` is returned when `keepdims=True` and `self` is a 0d array + @overload + def all( + self, axis: None = ..., out: None = ..., keepdims: Literal[False] = ... + ) -> bool_: ... + @overload + def all( + self, axis: Optional[_ShapeLike] = ..., out: None = ..., keepdims: bool = ... + ) -> Union[bool_, ndarray]: ... + @overload + def all( + self, + axis: Optional[_ShapeLike] = ..., + out: _NdArraySubClass = ..., + keepdims: bool = ..., + ) -> _NdArraySubClass: ... + @overload + def any( + self, axis: None = ..., out: None = ..., keepdims: Literal[False] = ... + ) -> bool_: ... + @overload + def any( + self, axis: Optional[_ShapeLike] = ..., out: None = ..., keepdims: bool = ... + ) -> Union[bool_, ndarray]: ... + @overload + def any( + self, + axis: Optional[_ShapeLike] = ..., + out: _NdArraySubClass = ..., + keepdims: bool = ..., + ) -> _NdArraySubClass: ... + @overload + def argmax(self, axis: None = ..., out: None = ...) -> signedinteger: ... + @overload + def argmax( + self, axis: _ShapeLike = ..., out: None = ... + ) -> Union[signedinteger, ndarray]: ... + @overload + def argmax( + self, axis: Optional[_ShapeLike] = ..., out: _NdArraySubClass = ... + ) -> _NdArraySubClass: ... + @overload + def argmin(self, axis: None = ..., out: None = ...) -> signedinteger: ... + @overload + def argmin( + self, axis: _ShapeLike = ..., out: None = ... + ) -> Union[signedinteger, ndarray]: ... + @overload + def argmin( + self, axis: Optional[_ShapeLike] = ..., out: _NdArraySubClass = ... + ) -> _NdArraySubClass: ... + def argsort( + self, + axis: Optional[int] = ..., + kind: Optional[_SortKind] = ..., + order: Union[None, str, Sequence[str]] = ..., + ) -> ndarray: ... + @overload + def choose( + self, choices: ArrayLike, out: None = ..., mode: _ModeKind = ..., + ) -> ndarray: ... + @overload + def choose( + self, choices: ArrayLike, out: _NdArraySubClass = ..., mode: _ModeKind = ..., + ) -> _NdArraySubClass: ... + @overload + def clip( + self, + min: ArrayLike = ..., + max: Optional[ArrayLike] = ..., + out: None = ..., + **kwargs: Any, + ) -> Union[number, ndarray]: ... + @overload + def clip( + self, + min: None = ..., + max: ArrayLike = ..., + out: None = ..., + **kwargs: Any, + ) -> Union[number, ndarray]: ... + @overload + def clip( + self, + min: ArrayLike = ..., + max: Optional[ArrayLike] = ..., + out: _NdArraySubClass = ..., + **kwargs: Any, + ) -> _NdArraySubClass: ... + @overload + def clip( + self, + min: None = ..., + max: ArrayLike = ..., + out: _NdArraySubClass = ..., + **kwargs: Any, + ) -> _NdArraySubClass: ... + @overload + def compress( + self, a: ArrayLike, axis: Optional[int] = ..., out: None = ..., + ) -> ndarray: ... + @overload + def compress( + self, a: ArrayLike, axis: Optional[int] = ..., out: _NdArraySubClass = ..., + ) -> _NdArraySubClass: ... + def conj(self: _ArraySelf) -> _ArraySelf: ... + def conjugate(self: _ArraySelf) -> _ArraySelf: ... + @overload + def cumprod( + self, axis: Optional[int] = ..., dtype: DTypeLike = ..., out: None = ..., + ) -> ndarray: ... + @overload + def cumprod( + self, + axis: Optional[int] = ..., + dtype: DTypeLike = ..., + out: _NdArraySubClass = ..., + ) -> _NdArraySubClass: ... + @overload + def cumsum( + self, axis: Optional[int] = ..., dtype: DTypeLike = ..., out: None = ..., + ) -> ndarray: ... + @overload + def cumsum( + self, + axis: Optional[int] = ..., + dtype: DTypeLike = ..., + out: _NdArraySubClass = ..., + ) -> _NdArraySubClass: ... + @overload + def max( + self, + axis: None = ..., + out: None = ..., + keepdims: Literal[False] = ..., + initial: _NumberLike = ..., + where: _ArrayLikeBool = ..., + ) -> number: ... + @overload + def max( + self, + axis: Optional[_ShapeLike] = ..., + out: None = ..., + keepdims: bool = ..., + initial: _NumberLike = ..., + where: _ArrayLikeBool = ..., + ) -> Union[number, ndarray]: ... + @overload + def max( + self, + axis: Optional[_ShapeLike] = ..., + out: _NdArraySubClass = ..., + keepdims: bool = ..., + initial: _NumberLike = ..., + where: _ArrayLikeBool = ..., + ) -> _NdArraySubClass: ... + @overload + def mean( + self, + axis: None = ..., + dtype: DTypeLike = ..., + out: None = ..., + keepdims: Literal[False] = ..., + ) -> number: ... + @overload + def mean( + self, + axis: Optional[_ShapeLike] = ..., + dtype: DTypeLike = ..., + out: None = ..., + keepdims: bool = ..., + ) -> Union[number, ndarray]: ... + @overload + def mean( + self, + axis: Optional[_ShapeLike] = ..., + dtype: DTypeLike = ..., + out: _NdArraySubClass = ..., + keepdims: bool = ..., + ) -> _NdArraySubClass: ... + @overload + def min( + self, + axis: None = ..., + out: None = ..., + keepdims: Literal[False] = ..., + initial: _NumberLike = ..., + where: _ArrayLikeBool = ..., + ) -> number: ... + @overload + def min( + self, + axis: Optional[_ShapeLike] = ..., + out: None = ..., + keepdims: bool = ..., + initial: _NumberLike = ..., + where: _ArrayLikeBool = ..., + ) -> Union[number, ndarray]: ... + @overload + def min( + self, + axis: Optional[_ShapeLike] = ..., + out: _NdArraySubClass = ..., + keepdims: bool = ..., + initial: _NumberLike = ..., + where: _ArrayLikeBool = ..., + ) -> _NdArraySubClass: ... + def newbyteorder(self: _ArraySelf, __new_order: _ByteOrder = ...) -> _ArraySelf: ... + @overload + def prod( + self, + axis: None = ..., + dtype: DTypeLike = ..., + out: None = ..., + keepdims: Literal[False] = ..., + initial: _NumberLike = ..., + where: _ArrayLikeBool = ..., + ) -> number: ... + @overload + def prod( + self, + axis: Optional[_ShapeLike] = ..., + dtype: DTypeLike = ..., + out: None = ..., + keepdims: bool = ..., + initial: _NumberLike = ..., + where: _ArrayLikeBool = ..., + ) -> Union[number, ndarray]: ... + @overload + def prod( + self, + axis: Optional[_ShapeLike] = ..., + dtype: DTypeLike = ..., + out: _NdArraySubClass = ..., + keepdims: bool = ..., + initial: _NumberLike = ..., + where: _ArrayLikeBool = ..., + ) -> _NdArraySubClass: ... + @overload + def ptp( + self, axis: None = ..., out: None = ..., keepdims: Literal[False] = ..., + ) -> number: ... + @overload + def ptp( + self, axis: Optional[_ShapeLike] = ..., out: None = ..., keepdims: bool = ..., + ) -> Union[number, ndarray]: ... + @overload + def ptp( + self, + axis: Optional[_ShapeLike] = ..., + out: _NdArraySubClass = ..., + keepdims: bool = ..., + ) -> _NdArraySubClass: ... + def repeat( + self, repeats: _ArrayLikeIntOrBool, axis: Optional[int] = ... + ) -> ndarray: ... + @overload + def round(self: _ArraySelf, decimals: int = ..., out: None = ...) -> _ArraySelf: ... + @overload + def round( + self, decimals: int = ..., out: _NdArraySubClass = ... + ) -> _NdArraySubClass: ... + @overload + def std( + self, + axis: None = ..., + dtype: DTypeLike = ..., + out: None = ..., + ddof: int = ..., + keepdims: Literal[False] = ..., + ) -> number: ... + @overload + def std( + self, + axis: Optional[_ShapeLike] = ..., + dtype: DTypeLike = ..., + out: None = ..., + ddof: int = ..., + keepdims: bool = ..., + ) -> Union[number, ndarray]: ... + @overload + def std( + self, + axis: Optional[_ShapeLike] = ..., + dtype: DTypeLike = ..., + out: _NdArraySubClass = ..., + ddof: int = ..., + keepdims: bool = ..., + ) -> _NdArraySubClass: ... + @overload + def sum( + self, + axis: None = ..., + dtype: DTypeLike = ..., + out: None = ..., + keepdims: Literal[False] = ..., + initial: _NumberLike = ..., + where: _ArrayLikeBool = ..., + ) -> number: ... + @overload + def sum( + self, + axis: Optional[_ShapeLike] = ..., + dtype: DTypeLike = ..., + out: None = ..., + keepdims: bool = ..., + initial: _NumberLike = ..., + where: _ArrayLikeBool = ..., + ) -> Union[number, ndarray]: ... + @overload + def sum( + self, + axis: Optional[_ShapeLike] = ..., + dtype: DTypeLike = ..., + out: _NdArraySubClass = ..., + keepdims: bool = ..., + initial: _NumberLike = ..., + where: _ArrayLikeBool = ..., + ) -> _NdArraySubClass: ... + @overload + def take( + self, + indices: Union[_IntLike, _BoolLike], + axis: Optional[int] = ..., + out: None = ..., + mode: _ModeKind = ..., + ) -> generic: ... + @overload + def take( + self, + indices: _ArrayLikeIntOrBool, + axis: Optional[int] = ..., + out: None = ..., + mode: _ModeKind = ..., + ) -> ndarray: ... + @overload + def take( + self, + indices: _ArrayLikeIntOrBool, + axis: Optional[int] = ..., + out: _NdArraySubClass = ..., + mode: _ModeKind = ..., + ) -> _NdArraySubClass: ... + @overload + def var( + self, + axis: None = ..., + dtype: DTypeLike = ..., + out: None = ..., + ddof: int = ..., + keepdims: Literal[False] = ..., + ) -> number: ... + @overload + def var( + self, + axis: Optional[_ShapeLike] = ..., + dtype: DTypeLike = ..., + out: None = ..., + ddof: int = ..., + keepdims: bool = ..., + ) -> Union[number, ndarray]: ... + @overload + def var( + self, + axis: Optional[_ShapeLike] = ..., + dtype: DTypeLike = ..., + out: _NdArraySubClass = ..., + ddof: int = ..., + keepdims: bool = ..., + ) -> _NdArraySubClass: ... _BufferType = Union[ndarray, bytes, bytearray, memoryview] _Casting = Literal["no", "equiv", "safe", "same_kind", "unsafe"] @@ -406,14 +1487,14 @@ class ndarray(_ArrayOrScalarCommon, Iterable, Sized, Container): def __new__( cls: Type[_ArraySelf], shape: Sequence[int], - dtype: DtypeLike = ..., + dtype: DTypeLike = ..., buffer: _BufferType = ..., offset: int = ..., strides: _ShapeLike = ..., order: _OrderKACF = ..., ) -> _ArraySelf: ... @property - def dtype(self) -> _Dtype: ... + def dtype(self) -> _DType: ... @property def ctypes(self) -> _ctypes: ... @property @@ -424,30 +1505,128 @@ class ndarray(_ArrayOrScalarCommon, Iterable, Sized, Container): def strides(self) -> _Shape: ... @strides.setter def strides(self, value: _ShapeLike): ... + def argpartition( + self, + kth: _ArrayLikeIntOrBool, + axis: Optional[int] = ..., + kind: _PartitionKind = ..., + order: Union[None, str, Sequence[str]] = ..., + ) -> ndarray: ... + def diagonal( + self: _ArraySelf, offset: int = ..., axis1: int = ..., axis2: int = ... + ) -> _ArraySelf: ... + @overload + def dot(self, b: ArrayLike, out: None = ...) -> Union[number, ndarray]: ... + @overload + def dot(self, b: ArrayLike, out: _NdArraySubClass = ...) -> _NdArraySubClass: ... + # `nonzero()` is deprecated for 0d arrays/generics + def nonzero(self) -> Tuple[ndarray, ...]: ... + def partition( + self, + kth: _ArrayLikeIntOrBool, + axis: int = ..., + kind: _PartitionKind = ..., + order: Union[None, str, Sequence[str]] = ..., + ) -> None: ... + # `put` is technically available to `generic`, + # but is pointless as `generic`s are immutable + def put( + self, ind: _ArrayLikeIntOrBool, v: ArrayLike, mode: _ModeKind = ... + ) -> None: ... + def searchsorted( + self, # >= 1D array + v: ArrayLike, + side: _SortSide = ..., + sorter: Optional[_ArrayLikeIntOrBool] = ..., # 1D int array + ) -> ndarray: ... + def setfield( + self, val: ArrayLike, dtype: DTypeLike, offset: int = ... + ) -> None: ... + def sort( + self, + axis: int = ..., + kind: Optional[_SortKind] = ..., + order: Union[None, str, Sequence[str]] = ..., + ) -> None: ... + @overload + def trace( + self, # >= 2D array + offset: int = ..., + axis1: int = ..., + axis2: int = ..., + dtype: DTypeLike = ..., + out: None = ..., + ) -> Union[number, ndarray]: ... + @overload + def trace( + self, # >= 2D array + offset: int = ..., + axis1: int = ..., + axis2: int = ..., + dtype: DTypeLike = ..., + out: _NdArraySubClass = ..., + ) -> _NdArraySubClass: ... # Many of these special methods are irrelevant currently, since protocols # aren't supported yet. That said, I'm adding them for completeness. # https://docs.python.org/3/reference/datamodel.html + def __int__(self) -> int: ... + def __float__(self) -> float: ... + def __complex__(self) -> complex: ... def __len__(self) -> int: ... def __setitem__(self, key, value): ... def __iter__(self) -> Any: ... def __contains__(self, key) -> bool: ... def __index__(self) -> int: ... - def __matmul__(self, other): ... - def __imatmul__(self, other): ... - def __rmatmul__(self, other): ... + def __matmul__(self, other: ArrayLike) -> Union[ndarray, generic]: ... + # NOTE: `ndarray` does not implement `__imatmul__` + def __rmatmul__(self, other: ArrayLike) -> Union[ndarray, generic]: ... + def __neg__(self: _ArraySelf) -> Union[_ArraySelf, generic]: ... + def __pos__(self: _ArraySelf) -> Union[_ArraySelf, generic]: ... + def __abs__(self: _ArraySelf) -> Union[_ArraySelf, generic]: ... + def __mod__(self, other: ArrayLike) -> Union[ndarray, generic]: ... + def __rmod__(self, other: ArrayLike) -> Union[ndarray, generic]: ... + def __divmod__( + self, other: ArrayLike + ) -> Union[Tuple[ndarray, ndarray], Tuple[generic, generic]]: ... + def __rdivmod__( + self, other: ArrayLike + ) -> Union[Tuple[ndarray, ndarray], Tuple[generic, generic]]: ... + def __add__(self, other: ArrayLike) -> Union[ndarray, generic]: ... + def __radd__(self, other: ArrayLike) -> Union[ndarray, generic]: ... + def __sub__(self, other: ArrayLike) -> Union[ndarray, generic]: ... + def __rsub__(self, other: ArrayLike) -> Union[ndarray, generic]: ... + def __mul__(self, other: ArrayLike) -> Union[ndarray, generic]: ... + def __rmul__(self, other: ArrayLike) -> Union[ndarray, generic]: ... + def __floordiv__(self, other: ArrayLike) -> Union[ndarray, generic]: ... + def __rfloordiv__(self, other: ArrayLike) -> Union[ndarray, generic]: ... + def __pow__(self, other: ArrayLike) -> Union[ndarray, generic]: ... + def __rpow__(self, other: ArrayLike) -> Union[ndarray, generic]: ... + def __truediv__(self, other: ArrayLike) -> Union[ndarray, generic]: ... + def __rtruediv__(self, other: ArrayLike) -> Union[ndarray, generic]: ... + def __invert__(self: _ArraySelf) -> Union[_ArraySelf, integer, bool_]: ... + def __lshift__(self, other: ArrayLike) -> Union[ndarray, integer]: ... + def __rlshift__(self, other: ArrayLike) -> Union[ndarray, integer]: ... + def __rshift__(self, other: ArrayLike) -> Union[ndarray, integer]: ... + def __rrshift__(self, other: ArrayLike) -> Union[ndarray, integer]: ... + def __and__(self, other: ArrayLike) -> Union[ndarray, integer, bool_]: ... + def __rand__(self, other: ArrayLike) -> Union[ndarray, integer, bool_]: ... + def __xor__(self, other: ArrayLike) -> Union[ndarray, integer, bool_]: ... + def __rxor__(self, other: ArrayLike) -> Union[ndarray, integer, bool_]: ... + def __or__(self, other: ArrayLike) -> Union[ndarray, integer, bool_]: ... + def __ror__(self, other: ArrayLike) -> Union[ndarray, integer, bool_]: ... # `np.generic` does not support inplace operations - def __iadd__(self, other): ... - def __isub__(self, other): ... - def __imul__(self, other): ... - def __itruediv__(self, other): ... - def __ifloordiv__(self, other): ... - def __imod__(self, other): ... - def __ipow__(self, other): ... - def __ilshift__(self, other): ... - def __irshift__(self, other): ... - def __iand__(self, other): ... - def __ixor__(self, other): ... - def __ior__(self, other): ... + def __iadd__(self: _ArraySelf, other: ArrayLike) -> _ArraySelf: ... + def __isub__(self: _ArraySelf, other: ArrayLike) -> _ArraySelf: ... + def __imul__(self: _ArraySelf, other: ArrayLike) -> _ArraySelf: ... + def __itruediv__(self: _ArraySelf, other: ArrayLike) -> _ArraySelf: ... + def __ifloordiv__(self: _ArraySelf, other: ArrayLike) -> _ArraySelf: ... + def __ipow__(self: _ArraySelf, other: ArrayLike) -> _ArraySelf: ... + def __imod__(self: _ArraySelf, other: ArrayLike) -> _ArraySelf: ... + def __ilshift__(self: _ArraySelf, other: ArrayLike) -> _ArraySelf: ... + def __irshift__(self: _ArraySelf, other: ArrayLike) -> _ArraySelf: ... + def __iand__(self: _ArraySelf, other: ArrayLike) -> _ArraySelf: ... + def __ixor__(self: _ArraySelf, other: ArrayLike) -> _ArraySelf: ... + def __ior__(self: _ArraySelf, other: ArrayLike) -> _ArraySelf: ... # NOTE: while `np.generic` is not technically an instance of `ABCMeta`, # the `@abstractmethod` decorator is herein used to (forcefully) deny @@ -457,7 +1636,8 @@ class ndarray(_ArrayOrScalarCommon, Iterable, Sized, Container): # See https://github.com/numpy/numpy-stubs/pull/80 for more details. -_CharLike = Union[str, bytes] +_NBit_co = TypeVar("_NBit_co", covariant=True, bound=NBitBase) +_NBit_co2 = TypeVar("_NBit_co2", covariant=True, bound=NBitBase) class generic(_ArrayOrScalarCommon): @abstractmethod @@ -465,11 +1645,30 @@ class generic(_ArrayOrScalarCommon): @property def base(self) -> None: ... -class number(generic): # type: ignore +class number(generic, Generic[_NBit_co]): # type: ignore @property def real(self: _ArraySelf) -> _ArraySelf: ... @property def imag(self: _ArraySelf) -> _ArraySelf: ... + def __int__(self) -> int: ... + def __float__(self) -> float: ... + def __complex__(self) -> complex: ... + def __neg__(self: _ArraySelf) -> _ArraySelf: ... + def __pos__(self: _ArraySelf) -> _ArraySelf: ... + def __abs__(self: _ArraySelf) -> _ArraySelf: ... + # Ensure that objects annotated as `number` support arithmetic operations + __add__: _NumberOp + __radd__: _NumberOp + __sub__: _NumberOp + __rsub__: _NumberOp + __mul__: _NumberOp + __rmul__: _NumberOp + __floordiv__: _NumberOp + __rfloordiv__: _NumberOp + __pow__: _NumberOp + __rpow__: _NumberOp + __truediv__: _NumberOp + __rtruediv__: _NumberOp class bool_(generic): def __init__(self, __value: object = ...) -> None: ... @@ -477,6 +1676,37 @@ class bool_(generic): def real(self: _ArraySelf) -> _ArraySelf: ... @property def imag(self: _ArraySelf) -> _ArraySelf: ... + def __int__(self) -> int: ... + def __float__(self) -> float: ... + def __complex__(self) -> complex: ... + def __abs__(self: _ArraySelf) -> _ArraySelf: ... + __add__: _BoolOp[bool_] + __radd__: _BoolOp[bool_] + __sub__: _BoolSub + __rsub__: _BoolSub + __mul__: _BoolOp[bool_] + __rmul__: _BoolOp[bool_] + __floordiv__: _BoolOp[int8] + __rfloordiv__: _BoolOp[int8] + __pow__: _BoolOp[int8] + __rpow__: _BoolOp[int8] + __truediv__: _BoolTrueDiv + __rtruediv__: _BoolTrueDiv + def __invert__(self) -> bool_: ... + __lshift__: _BoolBitOp[int8] + __rlshift__: _BoolBitOp[int8] + __rshift__: _BoolBitOp[int8] + __rrshift__: _BoolBitOp[int8] + __and__: _BoolBitOp[bool_] + __rand__: _BoolBitOp[bool_] + __xor__: _BoolBitOp[bool_] + __rxor__: _BoolBitOp[bool_] + __or__: _BoolBitOp[bool_] + __ror__: _BoolBitOp[bool_] + __mod__: _BoolMod + __rmod__: _BoolMod + __divmod__: _BoolDivMod + __rdivmod__: _BoolDivMod class object_(generic): def __init__(self, __value: object = ...) -> None: ... @@ -493,10 +1723,18 @@ class datetime64(generic): __format: Union[_CharLike, Tuple[_CharLike, _IntLike]] = ..., ) -> None: ... @overload - def __init__(self, __value: int, __format: Union[_CharLike, Tuple[_CharLike, _IntLike]]) -> None: ... - def __add__(self, other: Union[timedelta64, int]) -> datetime64: ... - def __sub__(self, other: Union[timedelta64, datetime64, int]) -> timedelta64: ... - def __rsub__(self, other: Union[datetime64, int]) -> timedelta64: ... + def __init__( + self, + __value: int, + __format: Union[_CharLike, Tuple[_CharLike, _IntLike]] + ) -> None: ... + def __add__(self, other: Union[timedelta64, _IntLike, _BoolLike]) -> datetime64: ... + def __radd__(self, other: Union[timedelta64, _IntLike, _BoolLike]) -> datetime64: ... + @overload + def __sub__(self, other: datetime64) -> timedelta64: ... + @overload + def __sub__(self, other: Union[timedelta64, _IntLike, _BoolLike]) -> datetime64: ... + def __rsub__(self, other: datetime64) -> timedelta64: ... # Support for `__index__` was added in python 3.8 (bpo-20092) if sys.version_info >= (3, 8): @@ -508,93 +1746,192 @@ else: _FloatValue = Union[None, _CharLike, SupportsFloat] _ComplexValue = Union[None, _CharLike, SupportsFloat, SupportsComplex] -class integer(number): # type: ignore +class integer(number[_NBit_co]): # type: ignore # NOTE: `__index__` is technically defined in the bottom-most # sub-classes (`int64`, `uint32`, etc) def __index__(self) -> int: ... - -class signedinteger(integer): ... # type: ignore - -class int8(signedinteger): - def __init__(self, __value: _IntValue = ...) -> None: ... - -class int16(signedinteger): - def __init__(self, __value: _IntValue = ...) -> None: ... - -class int32(signedinteger): - def __init__(self, __value: _IntValue = ...) -> None: ... - -class int64(signedinteger): + __truediv__: _IntTrueDiv[_NBit_co] + __rtruediv__: _IntTrueDiv[_NBit_co] + def __mod__(self, value: Union[_IntLike, integer]) -> integer: ... + def __rmod__(self, value: Union[_IntLike, integer]) -> integer: ... + def __invert__(self: _IntType) -> _IntType: ... + # Ensure that objects annotated as `integer` support bit-wise operations + def __lshift__(self, other: Union[_IntLike, _BoolLike]) -> integer: ... + def __rlshift__(self, other: Union[_IntLike, _BoolLike]) -> integer: ... + def __rshift__(self, other: Union[_IntLike, _BoolLike]) -> integer: ... + def __rrshift__(self, other: Union[_IntLike, _BoolLike]) -> integer: ... + def __and__(self, other: Union[_IntLike, _BoolLike]) -> integer: ... + def __rand__(self, other: Union[_IntLike, _BoolLike]) -> integer: ... + def __or__(self, other: Union[_IntLike, _BoolLike]) -> integer: ... + def __ror__(self, other: Union[_IntLike, _BoolLike]) -> integer: ... + def __xor__(self, other: Union[_IntLike, _BoolLike]) -> integer: ... + def __rxor__(self, other: Union[_IntLike, _BoolLike]) -> integer: ... + +class signedinteger(integer[_NBit_co]): def __init__(self, __value: _IntValue = ...) -> None: ... - -class timedelta64(signedinteger): + __add__: _SignedIntOp[_NBit_co] + __radd__: _SignedIntOp[_NBit_co] + __sub__: _SignedIntOp[_NBit_co] + __rsub__: _SignedIntOp[_NBit_co] + __mul__: _SignedIntOp[_NBit_co] + __rmul__: _SignedIntOp[_NBit_co] + __floordiv__: _SignedIntOp[_NBit_co] + __rfloordiv__: _SignedIntOp[_NBit_co] + __pow__: _SignedIntOp[_NBit_co] + __rpow__: _SignedIntOp[_NBit_co] + __lshift__: _SignedIntBitOp[_NBit_co] + __rlshift__: _SignedIntBitOp[_NBit_co] + __rshift__: _SignedIntBitOp[_NBit_co] + __rrshift__: _SignedIntBitOp[_NBit_co] + __and__: _SignedIntBitOp[_NBit_co] + __rand__: _SignedIntBitOp[_NBit_co] + __xor__: _SignedIntBitOp[_NBit_co] + __rxor__: _SignedIntBitOp[_NBit_co] + __or__: _SignedIntBitOp[_NBit_co] + __ror__: _SignedIntBitOp[_NBit_co] + __mod__: _SignedIntMod[_NBit_co] + __rmod__: _SignedIntMod[_NBit_co] + __divmod__: _SignedIntDivMod[_NBit_co] + __rdivmod__: _SignedIntDivMod[_NBit_co] + +int8 = signedinteger[_8Bit] +int16 = signedinteger[_16Bit] +int32 = signedinteger[_32Bit] +int64 = signedinteger[_64Bit] + +class timedelta64(generic): def __init__( self, __value: Union[None, int, _CharLike, dt.timedelta, timedelta64] = ..., __format: Union[_CharLike, Tuple[_CharLike, _IntLike]] = ..., ) -> None: ... - @overload - def __add__(self, other: Union[timedelta64, int]) -> timedelta64: ... - @overload - def __add__(self, other: datetime64) -> datetime64: ... - def __sub__(self, other: Union[timedelta64, int]) -> timedelta64: ... - @overload - def __truediv__(self, other: timedelta64) -> float: ... - @overload - def __truediv__(self, other: float) -> timedelta64: ... + def __int__(self) -> int: ... + def __float__(self) -> float: ... + def __complex__(self) -> complex: ... + def __neg__(self: _ArraySelf) -> _ArraySelf: ... + def __pos__(self: _ArraySelf) -> _ArraySelf: ... + def __abs__(self: _ArraySelf) -> _ArraySelf: ... + def __add__(self, other: Union[timedelta64, _IntLike, _BoolLike]) -> timedelta64: ... + def __radd__(self, other: Union[timedelta64, _IntLike, _BoolLike]) -> timedelta64: ... + def __sub__(self, other: Union[timedelta64, _IntLike, _BoolLike]) -> timedelta64: ... + def __rsub__(self, other: Union[timedelta64, _IntLike, _BoolLike]) -> timedelta64: ... + def __mul__(self, other: Union[_FloatLike, _BoolLike]) -> timedelta64: ... + def __rmul__(self, other: Union[_FloatLike, _BoolLike]) -> timedelta64: ... + __truediv__: _TD64Div[float64] + __floordiv__: _TD64Div[int64] + def __rtruediv__(self, other: timedelta64) -> float64: ... + def __rfloordiv__(self, other: timedelta64) -> int64: ... def __mod__(self, other: timedelta64) -> timedelta64: ... + def __rmod__(self, other: timedelta64) -> timedelta64: ... + def __divmod__(self, other: timedelta64) -> Tuple[int64, timedelta64]: ... + def __rdivmod__(self, other: timedelta64) -> Tuple[int64, timedelta64]: ... -class unsignedinteger(integer): ... # type: ignore - -class uint8(unsignedinteger): - def __init__(self, __value: _IntValue = ...) -> None: ... - -class uint16(unsignedinteger): - def __init__(self, __value: _IntValue = ...) -> None: ... - -class uint32(unsignedinteger): +class unsignedinteger(integer[_NBit_co]): + # NOTE: `uint64 + signedinteger -> float64` def __init__(self, __value: _IntValue = ...) -> None: ... - -class uint64(unsignedinteger): - def __init__(self, __value: _IntValue = ...) -> None: ... - -class inexact(number): ... # type: ignore -class floating(inexact): ... # type: ignore - + __add__: _UnsignedIntOp[_NBit_co] + __radd__: _UnsignedIntOp[_NBit_co] + __sub__: _UnsignedIntOp[_NBit_co] + __rsub__: _UnsignedIntOp[_NBit_co] + __mul__: _UnsignedIntOp[_NBit_co] + __rmul__: _UnsignedIntOp[_NBit_co] + __floordiv__: _UnsignedIntOp[_NBit_co] + __rfloordiv__: _UnsignedIntOp[_NBit_co] + __pow__: _UnsignedIntOp[_NBit_co] + __rpow__: _UnsignedIntOp[_NBit_co] + __lshift__: _UnsignedIntBitOp[_NBit_co] + __rlshift__: _UnsignedIntBitOp[_NBit_co] + __rshift__: _UnsignedIntBitOp[_NBit_co] + __rrshift__: _UnsignedIntBitOp[_NBit_co] + __and__: _UnsignedIntBitOp[_NBit_co] + __rand__: _UnsignedIntBitOp[_NBit_co] + __xor__: _UnsignedIntBitOp[_NBit_co] + __rxor__: _UnsignedIntBitOp[_NBit_co] + __or__: _UnsignedIntBitOp[_NBit_co] + __ror__: _UnsignedIntBitOp[_NBit_co] + __mod__: _UnsignedIntMod[_NBit_co] + __rmod__: _UnsignedIntMod[_NBit_co] + __divmod__: _UnsignedIntDivMod[_NBit_co] + __rdivmod__: _UnsignedIntDivMod[_NBit_co] + +uint8 = unsignedinteger[_8Bit] +uint16 = unsignedinteger[_16Bit] +uint32 = unsignedinteger[_32Bit] +uint64 = unsignedinteger[_64Bit] + +class inexact(number[_NBit_co]): ... # type: ignore + +_IntType = TypeVar("_IntType", bound=integer) _FloatType = TypeVar('_FloatType', bound=floating) -class float16(floating): - def __init__(self, __value: _FloatValue = ...) -> None: ... - -class float32(floating): +class floating(inexact[_NBit_co]): def __init__(self, __value: _FloatValue = ...) -> None: ... - -class float64(floating, float): - def __init__(self, __value: _FloatValue = ...) -> None: ... - -class complexfloating(inexact, Generic[_FloatType]): # type: ignore - @property - def real(self) -> _FloatType: ... # type: ignore[override] - @property - def imag(self) -> _FloatType: ... # type: ignore[override] - def __abs__(self) -> _FloatType: ... # type: ignore[override] - -class complex64(complexfloating[float32]): - def __init__(self, __value: _ComplexValue = ...) -> None: ... - -class complex128(complexfloating[float64], complex): + __add__: _FloatOp[_NBit_co] + __radd__: _FloatOp[_NBit_co] + __sub__: _FloatOp[_NBit_co] + __rsub__: _FloatOp[_NBit_co] + __mul__: _FloatOp[_NBit_co] + __rmul__: _FloatOp[_NBit_co] + __truediv__: _FloatOp[_NBit_co] + __rtruediv__: _FloatOp[_NBit_co] + __floordiv__: _FloatOp[_NBit_co] + __rfloordiv__: _FloatOp[_NBit_co] + __pow__: _FloatOp[_NBit_co] + __rpow__: _FloatOp[_NBit_co] + __mod__: _FloatMod[_NBit_co] + __rmod__: _FloatMod[_NBit_co] + __divmod__: _FloatDivMod[_NBit_co] + __rdivmod__: _FloatDivMod[_NBit_co] + +float16 = floating[_16Bit] +float32 = floating[_32Bit] +float64 = floating[_64Bit] + +# The main reason for `complexfloating` having two typevars is cosmetic. +# It is used to clarify why `complex128`s precision is `_64Bit`, the latter +# describing the two 64 bit floats representing its real and imaginary component + +class complexfloating(inexact[_NBit_co], Generic[_NBit_co, _NBit_co2]): def __init__(self, __value: _ComplexValue = ...) -> None: ... + @property + def real(self) -> floating[_NBit_co]: ... # type: ignore[override] + @property + def imag(self) -> floating[_NBit_co2]: ... # type: ignore[override] + def __abs__(self) -> floating[_NBit_co]: ... # type: ignore[override] + __add__: _ComplexOp[_NBit_co] + __radd__: _ComplexOp[_NBit_co] + __sub__: _ComplexOp[_NBit_co] + __rsub__: _ComplexOp[_NBit_co] + __mul__: _ComplexOp[_NBit_co] + __rmul__: _ComplexOp[_NBit_co] + __truediv__: _ComplexOp[_NBit_co] + __rtruediv__: _ComplexOp[_NBit_co] + __floordiv__: _ComplexOp[_NBit_co] + __rfloordiv__: _ComplexOp[_NBit_co] + __pow__: _ComplexOp[_NBit_co] + __rpow__: _ComplexOp[_NBit_co] + +complex64 = complexfloating[_32Bit, _32Bit] +complex128 = complexfloating[_64Bit, _64Bit] class flexible(generic): ... # type: ignore class void(flexible): - def __init__(self, __value: Union[int, integer, bool_, bytes]): ... + def __init__(self, __value: Union[_IntLike, _BoolLike, bytes]): ... @property def real(self: _ArraySelf) -> _ArraySelf: ... @property def imag(self: _ArraySelf) -> _ArraySelf: ... + def setfield( + self, val: ArrayLike, dtype: DTypeLike, offset: int = ... + ) -> None: ... + +class character(flexible): # type: ignore + def __int__(self) -> int: ... + def __float__(self) -> float: ... -class character(flexible): ... # type: ignore +# NOTE: Most `np.bytes_` / `np.str_` methods return their +# builtin `bytes` / `str` counterpart class bytes_(character, bytes): @overload @@ -612,6 +1949,8 @@ class str_(character, str): self, __value: bytes, encoding: str = ..., errors: str = ... ) -> None: ... +unicode_ = str0 = str_ + # TODO(alan): Platform dependent types # longcomplex, longdouble, longfloat # bytes, short, intc, intp, longlong @@ -622,7 +1961,7 @@ class str_(character, str): def array( object: object, - dtype: DtypeLike = ..., + dtype: DTypeLike = ..., *, copy: bool = ..., order: _OrderKACF = ..., @@ -632,177 +1971,71 @@ def array( ) -> ndarray: ... def zeros( shape: _ShapeLike, - dtype: DtypeLike = ..., - order: _OrderCF = ..., - *, - like: ArrayLike = ..., -) -> ndarray: ... -def ones( - shape: _ShapeLike, - dtype: DtypeLike = ..., + dtype: DTypeLike = ..., order: _OrderCF = ..., *, like: ArrayLike = ..., ) -> ndarray: ... def empty( shape: _ShapeLike, - dtype: DtypeLike = ..., + dtype: DTypeLike = ..., order: _OrderCF = ..., *, like: ArrayLike = ..., ) -> ndarray: ... -def zeros_like( - a: ArrayLike, - dtype: DtypeLike = ..., - order: _OrderKACF = ..., - subok: bool = ..., - shape: Optional[Union[int, Sequence[int]]] = ..., -) -> ndarray: ... -def ones_like( - a: ArrayLike, - dtype: DtypeLike = ..., - order: _OrderKACF = ..., - subok: bool = ..., - shape: Optional[_ShapeLike] = ..., -) -> ndarray: ... -def empty_like( - a: ArrayLike, - dtype: DtypeLike = ..., - order: _OrderKACF = ..., - subok: bool = ..., - shape: Optional[_ShapeLike] = ..., -) -> ndarray: ... -def full( - shape: _ShapeLike, - fill_value: Any, - dtype: DtypeLike = ..., - order: _OrderCF = ..., - *, - like: ArrayLike = ..., -) -> ndarray: ... -def full_like( - a: ArrayLike, - fill_value: Any, - dtype: DtypeLike = ..., - order: _OrderKACF = ..., - subok: bool = ..., - shape: Optional[_ShapeLike] = ..., -) -> ndarray: ... -def count_nonzero( - a: ArrayLike, axis: Optional[Union[int, Tuple[int], Tuple[int, int]]] = ... -) -> Union[int, ndarray]: ... -def isfortran(a: ndarray) -> bool: ... -def argwhere(a: ArrayLike) -> ndarray: ... -def flatnonzero(a: ArrayLike) -> ndarray: ... - -_CorrelateMode = Literal["valid", "same", "full"] - -def correlate(a: ArrayLike, v: ArrayLike, mode: _CorrelateMode = ...) -> ndarray: ... -def convolve(a: ArrayLike, v: ArrayLike, mode: _CorrelateMode = ...) -> ndarray: ... -def outer(a: ArrayLike, b: ArrayLike, out: ndarray = ...) -> ndarray: ... -def tensordot( - a: ArrayLike, - b: ArrayLike, - axes: Union[ - int, Tuple[int, int], Tuple[Tuple[int, int], ...], Tuple[List[int, int], ...] - ] = ..., -) -> ndarray: ... -def roll( - a: ArrayLike, - shift: Union[int, Tuple[int, ...]], - axis: Optional[Union[int, Tuple[int, ...]]] = ..., -) -> ndarray: ... -def rollaxis(a: ArrayLike, axis: int, start: int = ...) -> ndarray: ... -def moveaxis( - a: ndarray, - source: Union[int, Sequence[int]], - destination: Union[int, Sequence[int]], -) -> ndarray: ... -def cross( - a: ArrayLike, - b: ArrayLike, - axisa: int = ..., - axisb: int = ..., - axisc: int = ..., - axis: Optional[int] = ..., -) -> ndarray: ... -def indices( - dimensions: Sequence[int], dtype: dtype = ..., sparse: bool = ... -) -> Union[ndarray, Tuple[ndarray, ...]]: ... -def fromfunction( - function: Callable, - shape: Tuple[int, int], - *, - like: ArrayLike = ..., - **kwargs, -) -> Any: ... -def isscalar(element: Any) -> bool: ... -def binary_repr(num: int, width: Optional[int] = ...) -> str: ... -def base_repr(number: int, base: int = ..., padding: int = ...) -> str: ... -def identity(n: int, dtype: DtypeLike = ..., *, like: ArrayLike = ...) -> ndarray: ... -def allclose( - a: ArrayLike, - b: ArrayLike, - rtol: float = ..., - atol: float = ..., - equal_nan: bool = ..., -) -> bool: ... -def isclose( - a: ArrayLike, - b: ArrayLike, - rtol: float = ..., - atol: float = ..., - equal_nan: bool = ..., -) -> Union[bool_, ndarray]: ... -def array_equal(a1: ArrayLike, a2: ArrayLike) -> bool: ... -def array_equiv(a1: ArrayLike, a2: ArrayLike) -> bool: ... + +def broadcast_shapes(*args: _ShapeLike) -> _Shape: ... # # Constants # -Inf: float -Infinity: float -NAN: float -NINF: float -NZERO: float -NaN: float -PINF: float -PZERO: float -e: float -euler_gamma: float -inf: float -infty: float -nan: float -pi: float - -ALLOW_THREADS: int -BUFSIZE: int -CLIP: int -ERR_CALL: int -ERR_DEFAULT: int -ERR_IGNORE: int -ERR_LOG: int -ERR_PRINT: int -ERR_RAISE: int -ERR_WARN: int -FLOATING_POINT_SUPPORT: int -FPE_DIVIDEBYZERO: int -FPE_INVALID: int -FPE_OVERFLOW: int -FPE_UNDERFLOW: int -MAXDIMS: int -MAY_SHARE_BOUNDS: int -MAY_SHARE_EXACT: int -RAISE: int -SHIFT_DIVIDEBYZERO: int -SHIFT_INVALID: int -SHIFT_OVERFLOW: int -SHIFT_UNDERFLOW: int -UFUNC_BUFSIZE_DEFAULT: int -WRAP: int -little_endian: int -tracemalloc_domain: int +Inf: Final[float] +Infinity: Final[float] +NAN: Final[float] +NINF: Final[float] +NZERO: Final[float] +NaN: Final[float] +PINF: Final[float] +PZERO: Final[float] +e: Final[float] +euler_gamma: Final[float] +inf: Final[float] +infty: Final[float] +nan: Final[float] +pi: Final[float] +ALLOW_THREADS: Final[int] +BUFSIZE: Final[int] +CLIP: Final[int] +ERR_CALL: Final[int] +ERR_DEFAULT: Final[int] +ERR_IGNORE: Final[int] +ERR_LOG: Final[int] +ERR_PRINT: Final[int] +ERR_RAISE: Final[int] +ERR_WARN: Final[int] +FLOATING_POINT_SUPPORT: Final[int] +FPE_DIVIDEBYZERO: Final[int] +FPE_INVALID: Final[int] +FPE_OVERFLOW: Final[int] +FPE_UNDERFLOW: Final[int] +MAXDIMS: Final[int] +MAY_SHARE_BOUNDS: Final[int] +MAY_SHARE_EXACT: Final[int] +RAISE: Final[int] +SHIFT_DIVIDEBYZERO: Final[int] +SHIFT_INVALID: Final[int] +SHIFT_OVERFLOW: Final[int] +SHIFT_UNDERFLOW: Final[int] +UFUNC_BUFSIZE_DEFAULT: Final[int] +WRAP: Final[int] +tracemalloc_domain: Final[int] + +little_endian: Final[bool] +True_: Final[bool_] +False_: Final[bool_] + +UFUNC_PYVALS_NAME: Final[str] class ufunc: @property @@ -822,7 +2055,7 @@ class ufunc: keepdims: bool = ..., casting: _Casting = ..., order: _OrderKACF = ..., - dtype: DtypeLike = ..., + dtype: DTypeLike = ..., subok: bool = ..., signature: Union[str, Tuple[str]] = ..., # In reality this should be a length of list 3 containing an @@ -974,505 +2207,3 @@ class AxisError(ValueError, IndexError): def __init__( self, axis: int, ndim: Optional[int] = ..., msg_prefix: Optional[str] = ... ) -> None: ... - -# Functions from np.core.numerictypes -_DefaultType = TypeVar("_DefaultType") - -def maximum_sctype(t: DtypeLike) -> dtype: ... -def issctype(rep: object) -> bool: ... -@overload -def obj2sctype(rep: object) -> Optional[generic]: ... -@overload -def obj2sctype(rep: object, default: None) -> Optional[generic]: ... -@overload -def obj2sctype( - rep: object, default: Type[_DefaultType] -) -> Union[generic, Type[_DefaultType]]: ... -def issubclass_(arg1: object, arg2: Union[object, Tuple[object, ...]]) -> bool: ... -def issubsctype( - arg1: Union[ndarray, DtypeLike], arg2: Union[ndarray, DtypeLike] -) -> bool: ... -def issubdtype(arg1: DtypeLike, arg2: DtypeLike) -> bool: ... -def sctype2char(sctype: object) -> str: ... -def find_common_type( - array_types: Sequence[DtypeLike], scalar_types: Sequence[DtypeLike] -) -> dtype: ... - -# Functions from np.core.fromnumeric -_Mode = Literal["raise", "wrap", "clip"] -_PartitionKind = Literal["introselect"] -_SortKind = Literal["quicksort", "mergesort", "heapsort", "stable"] -_Side = Literal["left", "right"] - -# Various annotations for scalars - -# While dt.datetime and dt.timedelta are not technically part of NumPy, -# they are one of the rare few builtin scalars which serve as valid return types. -# See https://github.com/numpy/numpy-stubs/pull/67#discussion_r412604113. -_ScalarNumpy = Union[generic, dt.datetime, dt.timedelta] -_ScalarBuiltin = Union[str, bytes, dt.date, dt.timedelta, bool, int, float, complex] -_Scalar = Union[_ScalarBuiltin, _ScalarNumpy] - -# Integers and booleans can generally be used interchangeably -_ScalarIntOrBool = TypeVar("_ScalarIntOrBool", bound=Union[integer, bool_]) -_ScalarGeneric = TypeVar("_ScalarGeneric", bound=generic) -_ScalarGenericDT = TypeVar( - "_ScalarGenericDT", bound=Union[dt.datetime, dt.timedelta, generic] -) - -_Number = TypeVar('_Number', bound=number) -_NumberLike = Union[int, float, complex, number, bool_] - -# An array-like object consisting of integers -_IntLike = Union[int, integer] -_BoolLike = Union[bool, bool_] -_IntOrBool = Union[_IntLike, _BoolLike] -_ArrayLikeIntNested = ArrayLike # TODO: wait for support for recursive types -_ArrayLikeBoolNested = ArrayLike # TODO: wait for support for recursive types - -# Integers and booleans can generally be used interchangeably -_ArrayLikeIntOrBool = Union[ - _IntOrBool, - ndarray, - Sequence[_IntOrBool], - Sequence[_ArrayLikeIntNested], - Sequence[_ArrayLikeBoolNested], -] -_ArrayLikeBool = Union[ - _BoolLike, - Sequence[_BoolLike], - ndarray -] - -# The signature of take() follows a common theme with its overloads: -# 1. A generic comes in; the same generic comes out -# 2. A scalar comes in; a generic comes out -# 3. An array-like object comes in; some keyword ensures that a generic comes out -# 4. An array-like object comes in; an ndarray or generic comes out -@overload -def take( - a: _ScalarGenericDT, - indices: int, - axis: Optional[int] = ..., - out: Optional[ndarray] = ..., - mode: _Mode = ..., -) -> _ScalarGenericDT: ... -@overload -def take( - a: _Scalar, - indices: int, - axis: Optional[int] = ..., - out: Optional[ndarray] = ..., - mode: _Mode = ..., -) -> _ScalarNumpy: ... -@overload -def take( - a: ArrayLike, - indices: int, - axis: Optional[int] = ..., - out: Optional[ndarray] = ..., - mode: _Mode = ..., -) -> _ScalarNumpy: ... -@overload -def take( - a: ArrayLike, - indices: _ArrayLikeIntOrBool, - axis: Optional[int] = ..., - out: Optional[ndarray] = ..., - mode: _Mode = ..., -) -> Union[_ScalarNumpy, ndarray]: ... -def reshape(a: ArrayLike, newshape: _ShapeLike, order: _OrderACF = ...) -> ndarray: ... -@overload -def choose( - a: _ScalarIntOrBool, - choices: ArrayLike, - out: Optional[ndarray] = ..., - mode: _Mode = ..., -) -> _ScalarIntOrBool: ... -@overload -def choose( - a: _IntOrBool, choices: ArrayLike, out: Optional[ndarray] = ..., mode: _Mode = ... -) -> Union[integer, bool_]: ... -@overload -def choose( - a: _ArrayLikeIntOrBool, - choices: ArrayLike, - out: Optional[ndarray] = ..., - mode: _Mode = ..., -) -> ndarray: ... -def repeat( - a: ArrayLike, repeats: _ArrayLikeIntOrBool, axis: Optional[int] = ... -) -> ndarray: ... -def put( - a: ndarray, ind: _ArrayLikeIntOrBool, v: ArrayLike, mode: _Mode = ... -) -> None: ... -def swapaxes(a: ArrayLike, axis1: int, axis2: int) -> ndarray: ... -def transpose( - a: ArrayLike, axes: Union[None, Sequence[int], ndarray] = ... -) -> ndarray: ... -def partition( - a: ArrayLike, - kth: _ArrayLikeIntOrBool, - axis: Optional[int] = ..., - kind: _PartitionKind = ..., - order: Union[None, str, Sequence[str]] = ..., -) -> ndarray: ... -@overload -def argpartition( - a: generic, - kth: _ArrayLikeIntOrBool, - axis: Optional[int] = ..., - kind: _PartitionKind = ..., - order: Union[None, str, Sequence[str]] = ..., -) -> integer: ... -@overload -def argpartition( - a: _ScalarBuiltin, - kth: _ArrayLikeIntOrBool, - axis: Optional[int] = ..., - kind: _PartitionKind = ..., - order: Union[None, str, Sequence[str]] = ..., -) -> ndarray: ... -@overload -def argpartition( - a: ArrayLike, - kth: _ArrayLikeIntOrBool, - axis: Optional[int] = ..., - kind: _PartitionKind = ..., - order: Union[None, str, Sequence[str]] = ..., -) -> ndarray: ... -def sort( - a: ArrayLike, - axis: Optional[int] = ..., - kind: Optional[_SortKind] = ..., - order: Union[None, str, Sequence[str]] = ..., -) -> ndarray: ... -def argsort( - a: ArrayLike, - axis: Optional[int] = ..., - kind: Optional[_SortKind] = ..., - order: Union[None, str, Sequence[str]] = ..., -) -> ndarray: ... -@overload -def argmax(a: ArrayLike, axis: None = ..., out: Optional[ndarray] = ...) -> integer: ... -@overload -def argmax( - a: ArrayLike, axis: int = ..., out: Optional[ndarray] = ... -) -> Union[integer, ndarray]: ... -@overload -def argmin(a: ArrayLike, axis: None = ..., out: Optional[ndarray] = ...) -> integer: ... -@overload -def argmin( - a: ArrayLike, axis: int = ..., out: Optional[ndarray] = ... -) -> Union[integer, ndarray]: ... -@overload -def searchsorted( - a: ArrayLike, - v: _Scalar, - side: _Side = ..., - sorter: Optional[_ArrayLikeIntOrBool] = ..., # 1D int array -) -> integer: ... -@overload -def searchsorted( - a: ArrayLike, - v: ArrayLike, - side: _Side = ..., - sorter: Optional[_ArrayLikeIntOrBool] = ..., # 1D int array -) -> ndarray: ... -def resize(a: ArrayLike, new_shape: _ShapeLike) -> ndarray: ... -@overload -def squeeze(a: _ScalarGeneric, axis: Optional[_ShapeLike] = ...) -> _ScalarGeneric: ... -@overload -def squeeze(a: ArrayLike, axis: Optional[_ShapeLike] = ...) -> ndarray: ... -def diagonal( - a: ArrayLike, offset: int = ..., axis1: int = ..., axis2: int = ... # >= 2D array -) -> ndarray: ... -def trace( - a: ArrayLike, # >= 2D array - offset: int = ..., - axis1: int = ..., - axis2: int = ..., - dtype: DtypeLike = ..., - out: Optional[ndarray] = ..., -) -> Union[number, ndarray]: ... -def ravel(a: ArrayLike, order: _OrderKACF = ...) -> ndarray: ... -def nonzero(a: ArrayLike) -> Tuple[ndarray, ...]: ... -def shape(a: ArrayLike) -> _Shape: ... -def compress( - condition: ArrayLike, # 1D bool array - a: ArrayLike, - axis: Optional[int] = ..., - out: Optional[ndarray] = ..., -) -> ndarray: ... -@overload -def clip( - a: _Number, - a_min: ArrayLike, - a_max: Optional[ArrayLike], - out: Optional[ndarray] = ..., - **kwargs: Any, -) -> _Number: ... -@overload -def clip( - a: _Number, - a_min: None, - a_max: ArrayLike, - out: Optional[ndarray] = ..., - **kwargs: Any, -) -> _Number: ... -@overload -def clip( - a: ArrayLike, - a_min: ArrayLike, - a_max: Optional[ArrayLike], - out: Optional[ndarray] = ..., - **kwargs: Any, -) -> Union[number, ndarray]: ... -@overload -def clip( - a: ArrayLike, - a_min: None, - a_max: ArrayLike, - out: Optional[ndarray] = ..., - **kwargs: Any, -) -> Union[number, ndarray]: ... -@overload -def sum( - a: _Number, - axis: Optional[_ShapeLike] = ..., - dtype: DtypeLike = ..., - out: Optional[ndarray] = ..., - keepdims: bool = ..., - initial: _NumberLike = ..., - where: _ArrayLikeBool = ..., -) -> _Number: ... -@overload -def sum( - a: ArrayLike, - axis: _ShapeLike = ..., - dtype: DtypeLike = ..., - out: Optional[ndarray] = ..., - keepdims: bool = ..., - initial: _NumberLike = ..., - where: _ArrayLikeBool = ..., -) -> Union[number, ndarray]: ... -@overload -def all( - a: ArrayLike, - axis: None = ..., - out: Optional[ndarray] = ..., - keepdims: Literal[False] = ..., -) -> bool_: ... -@overload -def all( - a: ArrayLike, - axis: Optional[_ShapeLike] = ..., - out: Optional[ndarray] = ..., - keepdims: bool = ..., -) -> Union[bool_, ndarray]: ... -@overload -def any( - a: ArrayLike, - axis: None = ..., - out: Optional[ndarray] = ..., - keepdims: Literal[False] = ..., -) -> bool_: ... -@overload -def any( - a: ArrayLike, - axis: Optional[_ShapeLike] = ..., - out: Optional[ndarray] = ..., - keepdims: bool = ..., -) -> Union[bool_, ndarray]: ... -def cumsum( - a: ArrayLike, - axis: Optional[int] = ..., - dtype: DtypeLike = ..., - out: Optional[ndarray] = ..., -) -> ndarray: ... -@overload -def ptp( - a: _Number, - axis: Optional[_ShapeLike] = ..., - out: Optional[ndarray] = ..., - keepdims: bool = ..., -) -> _Number: ... -@overload -def ptp( - a: ArrayLike, - axis: None = ..., - out: Optional[ndarray] = ..., - keepdims: Literal[False] = ..., -) -> number: ... -@overload -def ptp( - a: ArrayLike, - axis: Optional[_ShapeLike] = ..., - out: Optional[ndarray] = ..., - keepdims: bool = ..., -) -> Union[number, ndarray]: ... -@overload -def amax( - a: _Number, - axis: Optional[_ShapeLike] = ..., - out: Optional[ndarray] = ..., - keepdims: bool = ..., - initial: _NumberLike = ..., - where: _ArrayLikeBool = ..., -) -> _Number: ... -@overload -def amax( - a: ArrayLike, - axis: None = ..., - out: Optional[ndarray] = ..., - keepdims: Literal[False] = ..., - initial: _NumberLike = ..., - where: _ArrayLikeBool = ..., -) -> number: ... -@overload -def amax( - a: ArrayLike, - axis: Optional[_ShapeLike] = ..., - out: Optional[ndarray] = ..., - keepdims: bool = ..., - initial: _NumberLike = ..., - where: _ArrayLikeBool = ..., -) -> Union[number, ndarray]: ... -@overload -def amin( - a: _Number, - axis: Optional[_ShapeLike] = ..., - out: Optional[ndarray] = ..., - keepdims: bool = ..., - initial: _NumberLike = ..., - where: _ArrayLikeBool = ..., -) -> _Number: ... -@overload -def amin( - a: ArrayLike, - axis: None = ..., - out: Optional[ndarray] = ..., - keepdims: Literal[False] = ..., - initial: _NumberLike = ..., - where: _ArrayLikeBool = ..., -) -> number: ... -@overload -def amin( - a: ArrayLike, - axis: Optional[_ShapeLike] = ..., - out: Optional[ndarray] = ..., - keepdims: bool = ..., - initial: _NumberLike = ..., - where: _ArrayLikeBool = ..., -) -> Union[number, ndarray]: ... - -# TODO: `np.prod()``: For object arrays `initial` does not necessarily -# have to be a numerical scalar. -# The only requirement is that it is compatible -# with the `.__mul__()` method(s) of the passed array's elements. - -# Note that the same situation holds for all wrappers around -# `np.ufunc.reduce`, e.g. `np.sum()` (`.__add__()`). - -@overload -def prod( - a: _Number, - axis: Optional[_ShapeLike] = ..., - dtype: DtypeLike = ..., - out: None = ..., - keepdims: bool = ..., - initial: _NumberLike = ..., - where: _ArrayLikeBool = ..., -) -> _Number: ... -@overload -def prod( - a: ArrayLike, - axis: None = ..., - dtype: DtypeLike = ..., - out: None = ..., - keepdims: Literal[False] = ..., - initial: _NumberLike = ..., - where: _ArrayLikeBool = ..., -) -> number: ... -@overload -def prod( - a: ArrayLike, - axis: Optional[_ShapeLike] = ..., - dtype: DtypeLike = ..., - out: Optional[ndarray] = ..., - keepdims: bool = ..., - initial: _NumberLike = ..., - where: _ArrayLikeBool = ..., -) -> Union[number, ndarray]: ... -def cumprod( - a: ArrayLike, - axis: Optional[int] = ..., - dtype: DtypeLike = ..., - out: Optional[ndarray] = ..., -) -> ndarray: ... -def ndim(a: ArrayLike) -> int: ... -def size(a: ArrayLike, axis: Optional[int] = ...) -> int: ... -@overload -def around( - a: _Number, decimals: int = ..., out: Optional[ndarray] = ... -) -> _Number: ... -@overload -def around( - a: _NumberLike, decimals: int = ..., out: Optional[ndarray] = ... -) -> number: ... -@overload -def around( - a: ArrayLike, decimals: int = ..., out: Optional[ndarray] = ... -) -> ndarray: ... -@overload -def mean( - a: ArrayLike, - axis: None = ..., - dtype: DtypeLike = ..., - out: None = ..., - keepdims: Literal[False] = ..., -) -> number: ... -@overload -def mean( - a: ArrayLike, - axis: Optional[_ShapeLike] = ..., - dtype: DtypeLike = ..., - out: Optional[ndarray] = ..., - keepdims: bool = ..., -) -> Union[number, ndarray]: ... -@overload -def std( - a: ArrayLike, - axis: None = ..., - dtype: DtypeLike = ..., - out: None = ..., - ddof: int = ..., - keepdims: Literal[False] = ..., -) -> number: ... -@overload -def std( - a: ArrayLike, - axis: Optional[_ShapeLike] = ..., - dtype: DtypeLike = ..., - out: Optional[ndarray] = ..., - ddof: int = ..., - keepdims: bool = ..., -) -> Union[number, ndarray]: ... -@overload -def var( - a: ArrayLike, - axis: None = ..., - dtype: DtypeLike = ..., - out: None = ..., - ddof: int = ..., - keepdims: Literal[False] = ..., -) -> number: ... -@overload -def var( - a: ArrayLike, - axis: Optional[_ShapeLike] = ..., - dtype: DtypeLike = ..., - out: Optional[ndarray] = ..., - ddof: int = ..., - keepdims: bool = ..., -) -> Union[number, ndarray]: ... diff --git a/numpy/compat/py3k.py b/numpy/compat/py3k.py index fd9f8bd4217e..f36aaca170ff 100644 --- a/numpy/compat/py3k.py +++ b/numpy/compat/py3k.py @@ -18,7 +18,7 @@ import sys import os -from pathlib import Path, PurePath +from pathlib import Path import io import abc @@ -78,11 +78,11 @@ def asunicode_nested(x): def is_pathlib_path(obj): """ - Check whether obj is a pathlib.Path object. + Check whether obj is a `pathlib.Path` object. - Prefer using `isinstance(obj, os_PathLike)` instead of this function. + Prefer using ``isinstance(obj, os.PathLike)`` instead of this function. """ - return Path is not None and isinstance(obj, Path) + return isinstance(obj, Path) # from Python 3.7 class contextlib_nullcontext: @@ -132,55 +132,5 @@ def npy_load_module(name, fn, info=None): return SourceFileLoader(name, fn).load_module() -# Backport os.fs_path, os.PathLike, and PurePath.__fspath__ -if sys.version_info[:2] >= (3, 6): - os_fspath = os.fspath - os_PathLike = os.PathLike -else: - def _PurePath__fspath__(self): - return str(self) - - class os_PathLike(abc_ABC): - """Abstract base class for implementing the file system path protocol.""" - - @abc.abstractmethod - def __fspath__(self): - """Return the file system path representation of the object.""" - raise NotImplementedError - - @classmethod - def __subclasshook__(cls, subclass): - if PurePath is not None and issubclass(subclass, PurePath): - return True - return hasattr(subclass, '__fspath__') - - - def os_fspath(path): - """Return the path representation of a path-like object. - If str or bytes is passed in, it is returned unchanged. Otherwise the - os.PathLike interface is used to get the path representation. If the - path representation is not str or bytes, TypeError is raised. If the - provided path is not str, bytes, or os.PathLike, TypeError is raised. - """ - if isinstance(path, (str, bytes)): - return path - - # Work from the object's type to match method resolution of other magic - # methods. - path_type = type(path) - try: - path_repr = path_type.__fspath__(path) - except AttributeError: - if hasattr(path_type, '__fspath__'): - raise - elif PurePath is not None and issubclass(path_type, PurePath): - return _PurePath__fspath__(path) - else: - raise TypeError("expected str, bytes or os.PathLike object, " - "not " + path_type.__name__) - if isinstance(path_repr, (str, bytes)): - return path_repr - else: - raise TypeError("expected {}.__fspath__() to return str or bytes, " - "not {}".format(path_type.__name__, - type(path_repr).__name__)) +os_fspath = os.fspath +os_PathLike = os.PathLike diff --git a/numpy/core/__init__.py b/numpy/core/__init__.py index c77885954ef1..e8d3a381b602 100644 --- a/numpy/core/__init__.py +++ b/numpy/core/__init__.py @@ -96,6 +96,7 @@ # do this after everything else, to minimize the chance of this misleadingly # appearing in an import-time traceback from . import _add_newdocs +from . import _add_newdocs_scalars # add these for module-freeze analysis (like PyInstaller) from . import _dtype_ctypes from . import _internal @@ -113,10 +114,9 @@ __all__ += shape_base.__all__ __all__ += einsumfunc.__all__ -# Make it possible so that ufuncs can be pickled -# Here are the loading and unloading functions -# The name numpy.core._ufunc_reconstruct must be -# available for unpickling to work. +# We used to use `np.core._ufunc_reconstruct` to unpickle. This is unnecessary, +# but old pickles saved before 1.20 will be using it, and there is no reason +# to break loading them. def _ufunc_reconstruct(module, name): # The `fromlist` kwarg is required to ensure that `mod` points to the # inner-most module rather than the parent package when module name is @@ -126,14 +126,17 @@ def _ufunc_reconstruct(module, name): return getattr(mod, name) def _ufunc_reduce(func): - from pickle import whichmodule - name = func.__name__ - return _ufunc_reconstruct, (whichmodule(func, name), name) + # Report the `__name__`. pickle will try to find the module. Note that + # pickle supports for this `__name__` to be a `__qualname__`. It may + # make sense to add a `__qualname__` to ufuncs, to allow this more + # explicitly (Numba has ufuncs as attributes). + # See also: https://github.com/dask/distributed/issues/3450 + return func.__name__ import copyreg -copyreg.pickle(ufunc, _ufunc_reduce, _ufunc_reconstruct) +copyreg.pickle(ufunc, _ufunc_reduce) # Unclutter namespace (must keep _ufunc_reconstruct for unpickling) del copyreg del _ufunc_reduce diff --git a/numpy/core/__init__.pyi b/numpy/core/__init__.pyi new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/numpy/core/_add_newdocs.py b/numpy/core/_add_newdocs.py index 879b3645d74d..c9968f122942 100644 --- a/numpy/core/_add_newdocs.py +++ b/numpy/core/_add_newdocs.py @@ -9,8 +9,6 @@ """ -from numpy.core import numerictypes as _numerictypes -from numpy.core import dtype from numpy.core.function_base import add_newdoc from numpy.core.overrides import array_function_like_doc @@ -607,6 +605,7 @@ -------- broadcast_arrays broadcast_to + broadcast_shapes Examples -------- @@ -2618,7 +2617,7 @@ """ a.argmin(axis=None, out=None) - Return indices of the minimum values along the given axis of `a`. + Return indices of the minimum values along the given axis. Refer to `numpy.argmin` for detailed documentation. @@ -6283,183 +6282,3 @@ def refer_to_array_attribute(attr, method=True): Abstract base class of all character string scalar types. """) - - -############################################################################## -# -# Documentation for concrete scalar classes -# -############################################################################## - -def numeric_type_aliases(aliases): - def type_aliases_gen(): - for alias, doc in aliases: - try: - alias_type = getattr(_numerictypes, alias) - except AttributeError: - # The set of aliases that actually exist varies between platforms - pass - else: - yield (alias_type, alias, doc) - return list(type_aliases_gen()) - - -possible_aliases = numeric_type_aliases([ - ('int8', '8-bit signed integer (-128 to 127)'), - ('int16', '16-bit signed integer (-32768 to 32767)'), - ('int32', '32-bit signed integer (-2147483648 to 2147483647)'), - ('int64', '64-bit signed integer (-9223372036854775808 to 9223372036854775807)'), - ('intp', 'Signed integer large enough to fit pointer, compatible with C ``intptr_t``'), - ('uint8', '8-bit unsigned integer (0 to 255)'), - ('uint16', '16-bit unsigned integer (0 to 65535)'), - ('uint32', '32-bit unsigned integer (0 to 4294967295)'), - ('uint64', '64-bit unsigned integer (0 to 18446744073709551615)'), - ('uintp', 'Unsigned integer large enough to fit pointer, compatible with C ``uintptr_t``'), - ('float16', '16-bit-precision floating-point number type: sign bit, 5 bits exponent, 10 bits mantissa'), - ('float32', '32-bit-precision floating-point number type: sign bit, 8 bits exponent, 23 bits mantissa'), - ('float64', '64-bit precision floating-point number type: sign bit, 11 bits exponent, 52 bits mantissa'), - ('float96', '96-bit extended-precision floating-point number type'), - ('float128', '128-bit extended-precision floating-point number type'), - ('complex64', 'Complex number type composed of 2 32-bit-precision floating-point numbers'), - ('complex128', 'Complex number type composed of 2 64-bit-precision floating-point numbers'), - ('complex192', 'Complex number type composed of 2 96-bit extended-precision floating-point numbers'), - ('complex256', 'Complex number type composed of 2 128-bit extended-precision floating-point numbers'), - ]) - - -def add_newdoc_for_scalar_type(obj, fixed_aliases, doc): - o = getattr(_numerictypes, obj) - - character_code = dtype(o).char - canonical_name_doc = "" if obj == o.__name__ else "Canonical name: ``np.{}``.\n ".format(obj) - alias_doc = ''.join("Alias: ``np.{}``.\n ".format(alias) for alias in fixed_aliases) - alias_doc += ''.join("Alias *on this platform*: ``np.{}``: {}.\n ".format(alias, doc) - for (alias_type, alias, doc) in possible_aliases if alias_type is o) - - docstring = """ - {doc} - Character code: ``'{character_code}'``. - {canonical_name_doc}{alias_doc} - """.format(doc=doc.strip(), character_code=character_code, - canonical_name_doc=canonical_name_doc, alias_doc=alias_doc) - - add_newdoc('numpy.core.numerictypes', obj, docstring) - - -add_newdoc_for_scalar_type('bool_', ['bool8'], - """ - Boolean type (True or False), stored as a byte. - """) - -add_newdoc_for_scalar_type('byte', [], - """ - Signed integer type, compatible with C ``char``. - """) - -add_newdoc_for_scalar_type('short', [], - """ - Signed integer type, compatible with C ``short``. - """) - -add_newdoc_for_scalar_type('intc', [], - """ - Signed integer type, compatible with C ``int``. - """) - -add_newdoc_for_scalar_type('int_', [], - """ - Signed integer type, compatible with Python `int` anc C ``long``. - """) - -add_newdoc_for_scalar_type('longlong', [], - """ - Signed integer type, compatible with C ``long long``. - """) - -add_newdoc_for_scalar_type('ubyte', [], - """ - Unsigned integer type, compatible with C ``unsigned char``. - """) - -add_newdoc_for_scalar_type('ushort', [], - """ - Unsigned integer type, compatible with C ``unsigned short``. - """) - -add_newdoc_for_scalar_type('uintc', [], - """ - Unsigned integer type, compatible with C ``unsigned int``. - """) - -add_newdoc_for_scalar_type('uint', [], - """ - Unsigned integer type, compatible with C ``unsigned long``. - """) - -add_newdoc_for_scalar_type('ulonglong', [], - """ - Signed integer type, compatible with C ``unsigned long long``. - """) - -add_newdoc_for_scalar_type('half', [], - """ - Half-precision floating-point number type. - """) - -add_newdoc_for_scalar_type('single', [], - """ - Single-precision floating-point number type, compatible with C ``float``. - """) - -add_newdoc_for_scalar_type('double', ['float_'], - """ - Double-precision floating-point number type, compatible with Python `float` - and C ``double``. - """) - -add_newdoc_for_scalar_type('longdouble', ['longfloat'], - """ - Extended-precision floating-point number type, compatible with C - ``long double`` but not necessarily with IEEE 754 quadruple-precision. - """) - -add_newdoc_for_scalar_type('csingle', ['singlecomplex'], - """ - Complex number type composed of two single-precision floating-point - numbers. - """) - -add_newdoc_for_scalar_type('cdouble', ['cfloat', 'complex_'], - """ - Complex number type composed of two double-precision floating-point - numbers, compatible with Python `complex`. - """) - -add_newdoc_for_scalar_type('clongdouble', ['clongfloat', 'longcomplex'], - """ - Complex number type composed of two extended-precision floating-point - numbers. - """) - -add_newdoc_for_scalar_type('object_', [], - """ - Any Python object. - """) - -# TODO: work out how to put this on the base class, np.floating -for float_name in ('half', 'single', 'double', 'longdouble'): - add_newdoc('numpy.core.numerictypes', float_name, ('as_integer_ratio', - """ - {ftype}.as_integer_ratio() -> (int, int) - - Return a pair of integers, whose ratio is exactly equal to the original - floating point number, and with a positive denominator. - Raise OverflowError on infinities and a ValueError on NaNs. - - >>> np.{ftype}(10.0).as_integer_ratio() - (10, 1) - >>> np.{ftype}(0.0).as_integer_ratio() - (0, 1) - >>> np.{ftype}(-.25).as_integer_ratio() - (-1, 4) - """.format(ftype=float_name))) diff --git a/numpy/core/_add_newdocs_scalars.py b/numpy/core/_add_newdocs_scalars.py new file mode 100644 index 000000000000..b9b151224e61 --- /dev/null +++ b/numpy/core/_add_newdocs_scalars.py @@ -0,0 +1,251 @@ +""" +This file is separate from ``_add_newdocs.py`` so that it can be mocked out by +our sphinx ``conf.py`` during doc builds, where we want to avoid showing +platform-dependent information. +""" +from numpy.core import dtype +from numpy.core import numerictypes as _numerictypes +from numpy.core.function_base import add_newdoc + +############################################################################## +# +# Documentation for concrete scalar classes +# +############################################################################## + +def numeric_type_aliases(aliases): + def type_aliases_gen(): + for alias, doc in aliases: + try: + alias_type = getattr(_numerictypes, alias) + except AttributeError: + # The set of aliases that actually exist varies between platforms + pass + else: + yield (alias_type, alias, doc) + return list(type_aliases_gen()) + + +possible_aliases = numeric_type_aliases([ + ('int8', '8-bit signed integer (``-128`` to ``127``)'), + ('int16', '16-bit signed integer (``-32_768`` to ``32_767``)'), + ('int32', '32-bit signed integer (``-2_147_483_648`` to ``2_147_483_647``)'), + ('int64', '64-bit signed integer (``-9_223_372_036_854_775_808`` to ``9_223_372_036_854_775_807``)'), + ('intp', 'Signed integer large enough to fit pointer, compatible with C ``intptr_t``'), + ('uint8', '8-bit unsigned integer (``0`` to ``255``)'), + ('uint16', '16-bit unsigned integer (``0`` to ``65_535``)'), + ('uint32', '32-bit unsigned integer (``0`` to ``4_294_967_295``)'), + ('uint64', '64-bit unsigned integer (``0`` to ``18_446_744_073_709_551_615``)'), + ('uintp', 'Unsigned integer large enough to fit pointer, compatible with C ``uintptr_t``'), + ('float16', '16-bit-precision floating-point number type: sign bit, 5 bits exponent, 10 bits mantissa'), + ('float32', '32-bit-precision floating-point number type: sign bit, 8 bits exponent, 23 bits mantissa'), + ('float64', '64-bit precision floating-point number type: sign bit, 11 bits exponent, 52 bits mantissa'), + ('float96', '96-bit extended-precision floating-point number type'), + ('float128', '128-bit extended-precision floating-point number type'), + ('complex64', 'Complex number type composed of 2 32-bit-precision floating-point numbers'), + ('complex128', 'Complex number type composed of 2 64-bit-precision floating-point numbers'), + ('complex192', 'Complex number type composed of 2 96-bit extended-precision floating-point numbers'), + ('complex256', 'Complex number type composed of 2 128-bit extended-precision floating-point numbers'), + ]) + + +def add_newdoc_for_scalar_type(obj, fixed_aliases, doc): + # note: `:field: value` is rST syntax which renders as field lists. + o = getattr(_numerictypes, obj) + + character_code = dtype(o).char + canonical_name_doc = "" if obj == o.__name__ else ":Canonical name: `numpy.{}`\n ".format(obj) + alias_doc = ''.join(":Alias: `numpy.{}`\n ".format(alias) for alias in fixed_aliases) + alias_doc += ''.join(":Alias on this platform: `numpy.{}`: {}.\n ".format(alias, doc) + for (alias_type, alias, doc) in possible_aliases if alias_type is o) + docstring = """ + {doc} + + :Character code: ``'{character_code}'`` + {canonical_name_doc}{alias_doc} + """.format(doc=doc.strip(), character_code=character_code, + canonical_name_doc=canonical_name_doc, alias_doc=alias_doc) + + add_newdoc('numpy.core.numerictypes', obj, docstring) + + +add_newdoc_for_scalar_type('bool_', ['bool8'], + """ + Boolean type (True or False), stored as a byte. + + .. warning:: + + The :class:`bool_` type is not a subclass of the :class:`int_` type + (the :class:`bool_` is not even a number type). This is different + than Python's default implementation of :class:`bool` as a + sub-class of :class:`int`. + """) + +add_newdoc_for_scalar_type('byte', [], + """ + Signed integer type, compatible with C ``char``. + """) + +add_newdoc_for_scalar_type('short', [], + """ + Signed integer type, compatible with C ``short``. + """) + +add_newdoc_for_scalar_type('intc', [], + """ + Signed integer type, compatible with C ``int``. + """) + +add_newdoc_for_scalar_type('int_', [], + """ + Signed integer type, compatible with Python `int` and C ``long``. + """) + +add_newdoc_for_scalar_type('longlong', [], + """ + Signed integer type, compatible with C ``long long``. + """) + +add_newdoc_for_scalar_type('ubyte', [], + """ + Unsigned integer type, compatible with C ``unsigned char``. + """) + +add_newdoc_for_scalar_type('ushort', [], + """ + Unsigned integer type, compatible with C ``unsigned short``. + """) + +add_newdoc_for_scalar_type('uintc', [], + """ + Unsigned integer type, compatible with C ``unsigned int``. + """) + +add_newdoc_for_scalar_type('uint', [], + """ + Unsigned integer type, compatible with C ``unsigned long``. + """) + +add_newdoc_for_scalar_type('ulonglong', [], + """ + Signed integer type, compatible with C ``unsigned long long``. + """) + +add_newdoc_for_scalar_type('half', [], + """ + Half-precision floating-point number type. + """) + +add_newdoc_for_scalar_type('single', [], + """ + Single-precision floating-point number type, compatible with C ``float``. + """) + +add_newdoc_for_scalar_type('double', ['float_'], + """ + Double-precision floating-point number type, compatible with Python `float` + and C ``double``. + """) + +add_newdoc_for_scalar_type('longdouble', ['longfloat'], + """ + Extended-precision floating-point number type, compatible with C + ``long double`` but not necessarily with IEEE 754 quadruple-precision. + """) + +add_newdoc_for_scalar_type('csingle', ['singlecomplex'], + """ + Complex number type composed of two single-precision floating-point + numbers. + """) + +add_newdoc_for_scalar_type('cdouble', ['cfloat', 'complex_'], + """ + Complex number type composed of two double-precision floating-point + numbers, compatible with Python `complex`. + """) + +add_newdoc_for_scalar_type('clongdouble', ['clongfloat', 'longcomplex'], + """ + Complex number type composed of two extended-precision floating-point + numbers. + """) + +add_newdoc_for_scalar_type('object_', [], + """ + Any Python object. + """) + +add_newdoc_for_scalar_type('str_', ['unicode_'], + r""" + A unicode string. + + When used in arrays, this type strips trailing null codepoints. + + Unlike the builtin `str`, this supports the :ref:`python:bufferobjects`, exposing its + contents as UCS4: + + >>> m = memoryview(np.str_("abc")) + >>> m.format + '3w' + >>> m.tobytes() + b'a\x00\x00\x00b\x00\x00\x00c\x00\x00\x00' + """) + +add_newdoc_for_scalar_type('bytes_', ['string_'], + r""" + A byte string. + + When used in arrays, this type strips trailing null bytes. + """) + +add_newdoc_for_scalar_type('void', [], + r""" + Either an opaque sequence of bytes, or a structure. + + >>> np.void(b'abcd') + void(b'\x61\x62\x63\x64') + + Structured `void` scalars can only be constructed via extraction from :ref:`structured_arrays`: + + >>> arr = np.array((1, 2), dtype=[('x', np.int8), ('y', np.int8)]) + >>> arr[()] + (1, 2) # looks like a tuple, but is `np.void` + """) + +add_newdoc_for_scalar_type('datetime64', [], + """ + A datetime stored as a 64-bit integer, counting from ``1970-01-01T00:00:00``. + + >>> np.datetime64(10, 'Y') + numpy.datetime64('1980') + >>> np.datetime64(10, 'D') + numpy.datetime64('1970-01-11') + + See :ref:`arrays.datetime` for more information. + """) + +add_newdoc_for_scalar_type('timedelta64', [], + """ + A timedelta stored as a 64-bit integer. + + See :ref:`arrays.datetime` for more information. + """) + +# TODO: work out how to put this on the base class, np.floating +for float_name in ('half', 'single', 'double', 'longdouble'): + add_newdoc('numpy.core.numerictypes', float_name, ('as_integer_ratio', + """ + {ftype}.as_integer_ratio() -> (int, int) + + Return a pair of integers, whose ratio is exactly equal to the original + floating point number, and with a positive denominator. + Raise `OverflowError` on infinities and a `ValueError` on NaNs. + + >>> np.{ftype}(10.0).as_integer_ratio() + (10, 1) + >>> np.{ftype}(0.0).as_integer_ratio() + (0, 1) + >>> np.{ftype}(-.25).as_integer_ratio() + (-1, 4) + """.format(ftype=float_name))) diff --git a/numpy/core/_asarray.pyi b/numpy/core/_asarray.pyi new file mode 100644 index 000000000000..8c200ba22ae2 --- /dev/null +++ b/numpy/core/_asarray.pyi @@ -0,0 +1,77 @@ +import sys +from typing import TypeVar, Union, Iterable, overload + +from numpy import ndarray, _OrderKACF +from numpy.typing import ArrayLike, DTypeLike + +if sys.version_info >= (3, 8): + from typing import Literal +else: + from typing_extensions import Literal + +_ArrayType = TypeVar("_ArrayType", bound=ndarray) + +def asarray( + a: object, + dtype: DTypeLike = ..., + order: _OrderKACF = ..., + *, + like: ArrayLike = ... +) -> ndarray: ... +@overload +def asanyarray( + a: _ArrayType, + dtype: None = ..., + order: _OrderKACF = ..., + *, + like: ArrayLike = ... +) -> _ArrayType: ... +@overload +def asanyarray( + a: object, + dtype: DTypeLike = ..., + order: _OrderKACF = ..., + *, + like: ArrayLike = ... +) -> ndarray: ... +def ascontiguousarray( + a: object, dtype: DTypeLike = ..., *, like: ArrayLike = ... +) -> ndarray: ... +def asfortranarray( + a: object, dtype: DTypeLike = ..., *, like: ArrayLike = ... +) -> ndarray: ... + +_Requirements = Literal[ + "C", "C_CONTIGUOUS", "CONTIGUOUS", + "F", "F_CONTIGUOUS", "FORTRAN", + "A", "ALIGNED", + "W", "WRITEABLE", + "O", "OWNDATA" +] +_E = Literal["E", "ENSUREARRAY"] +_RequirementsWithE = Union[_Requirements, _E] + +@overload +def require( + a: _ArrayType, + dtype: None = ..., + requirements: Union[None, _Requirements, Iterable[_Requirements]] = ..., + *, + like: ArrayLike = ... +) -> _ArrayType: ... +@overload +def require( + a: object, + dtype: DTypeLike = ..., + requirements: Union[_E, Iterable[_RequirementsWithE]] = ..., + *, + like: ArrayLike = ... +) -> ndarray: ... +@overload +def require( + a: object, + dtype: DTypeLike = ..., + requirements: Union[None, _Requirements, Iterable[_Requirements]] = ..., + *, + like: ArrayLike = ... +) -> ndarray: ... diff --git a/numpy/core/_dtype.py b/numpy/core/_dtype.py index 50aeeb5bc921..4249071ffe98 100644 --- a/numpy/core/_dtype.py +++ b/numpy/core/_dtype.py @@ -176,7 +176,7 @@ def _byte_order_str(dtype): def _datetime_metadata_str(dtype): - # TODO: this duplicates the C append_metastr_to_string + # TODO: this duplicates the C metastr_to_unicode functionality unit, count = np.datetime_data(dtype) if unit == 'generic': return '' diff --git a/numpy/core/_type_aliases.pyi b/numpy/core/_type_aliases.pyi new file mode 100644 index 000000000000..6a1099cd3fad --- /dev/null +++ b/numpy/core/_type_aliases.pyi @@ -0,0 +1,19 @@ +import sys +from typing import Dict, Union, Type, List + +from numpy import generic, signedinteger, unsignedinteger, floating, complexfloating + +if sys.version_info >= (3, 8): + from typing import TypedDict +else: + from typing_extensions import TypedDict + +class _SCTypes(TypedDict): + int: List[Type[signedinteger]] + uint: List[Type[unsignedinteger]] + float: List[Type[floating]] + complex: List[Type[complexfloating]] + others: List[type] + +sctypeDict: Dict[Union[int, str], Type[generic]] +sctypes: _SCTypes diff --git a/numpy/core/code_generators/ufunc_docstrings.py b/numpy/core/code_generators/ufunc_docstrings.py index 82cd6fb273f2..b7edd2834b43 100644 --- a/numpy/core/code_generators/ufunc_docstrings.py +++ b/numpy/core/code_generators/ufunc_docstrings.py @@ -107,6 +107,13 @@ def add_newdoc(place, name, doc): >>> plt.imshow(np.abs(xx), extent=[-10, 10, -10, 10], cmap='gray') >>> plt.show() + The `abs` function can be used as a shorthand for ``np.absolute`` on + ndarrays. + + >>> x = np.array([-1.2, 1.2]) + >>> abs(x) + array([1.2, 1.2]) + """) add_newdoc('numpy.core.umath', 'add', @@ -141,6 +148,14 @@ def add_newdoc(place, name, doc): [ 3., 5., 7.], [ 6., 8., 10.]]) + The ``+`` operator can be used as a shorthand for ``np.add`` on ndarrays. + + >>> x1 = np.arange(9.0).reshape((3, 3)) + >>> x2 = np.arange(3.0) + >>> x1 + x2 + array([[ 0., 2., 4.], + [ 3., 5., 7.], + [ 6., 8., 10.]]) """) add_newdoc('numpy.core.umath', 'arccos', @@ -608,6 +623,14 @@ def add_newdoc(place, name, doc): >>> np.bitwise_and([True, True], [False, True]) array([False, True]) + The ``&`` operator can be used as a shorthand for ``np.bitwise_and`` on + ndarrays. + + >>> x1 = np.array([2, 5, 255]) + >>> x2 = np.array([3, 14, 16]) + >>> x1 & x2 + array([ 2, 4, 16]) + """) add_newdoc('numpy.core.umath', 'bitwise_or', @@ -667,6 +690,14 @@ def add_newdoc(place, name, doc): >>> np.bitwise_or([True, True], [False, True]) array([ True, True]) + The ``|`` operator can be used as a shorthand for ``np.bitwise_or`` on + ndarrays. + + >>> x1 = np.array([2, 5, 255]) + >>> x2 = np.array([4, 4, 4]) + >>> x1 | x2 + array([ 6, 5, 255]) + """) add_newdoc('numpy.core.umath', 'bitwise_xor', @@ -719,6 +750,14 @@ def add_newdoc(place, name, doc): >>> np.bitwise_xor([True, True], [False, True]) array([ True, False]) + The ``^`` operator can be used as a shorthand for ``np.bitwise_xor`` on + ndarrays. + + >>> x1 = np.array([True, True]) + >>> x2 = np.array([False, True]) + >>> x1 ^ x2 + array([ True, False]) + """) add_newdoc('numpy.core.umath', 'ceil', @@ -1088,6 +1127,16 @@ def add_newdoc(place, name, doc): >>> np.divide(1, 0) 0 + The ``/`` operator can be used as a shorthand for ``np.divide`` on + ndarrays. + + >>> x1 = np.arange(9.0).reshape((3, 3)) + >>> x2 = 2 * np.ones(3) + >>> x1 / x2 + array([[0. , 0.5, 1. ], + [1.5, 2. , 2.5], + [3. , 3.5, 4. ]]) + """) add_newdoc('numpy.core.umath', 'equal', @@ -1123,6 +1172,14 @@ def add_newdoc(place, name, doc): >>> np.equal(1, np.ones(1)) array([ True]) + The ``==`` operator can be used as a shorthand for ``np.equal`` on + ndarrays. + + >>> a = np.array([2, 4, 6]) + >>> b = np.array([2, 4, 2]) + >>> a == b + array([ True, True, False]) + """) add_newdoc('numpy.core.umath', 'exp', @@ -1370,6 +1427,13 @@ def add_newdoc(place, name, doc): >>> np.floor_divide([1., 2., 3., 4.], 2.5) array([ 0., 0., 1., 1.]) + The ``//`` operator can be used as a shorthand for ``np.floor_divide`` + on ndarrays. + + >>> x1 = np.array([1., 2., 3., 4.]) + >>> x1 // 2.5 + array([0., 0., 1., 1.]) + """) add_newdoc('numpy.core.umath', 'fmod', @@ -1458,10 +1522,11 @@ def add_newdoc(place, name, doc): >>> np.greater([4,2],[2,2]) array([ True, False]) - If the inputs are ndarrays, then np.greater is equivalent to '>'. + The ``>`` operator can be used as a shorthand for ``np.greater`` on + ndarrays. - >>> a = np.array([4,2]) - >>> b = np.array([2,2]) + >>> a = np.array([4, 2]) + >>> b = np.array([2, 2]) >>> a > b array([ True, False]) @@ -1494,6 +1559,14 @@ def add_newdoc(place, name, doc): >>> np.greater_equal([4, 2, 1], [2, 2, 2]) array([ True, True, False]) + The ``>=`` operator can be used as a shorthand for ``np.greater_equal`` + on ndarrays. + + >>> a = np.array([4, 2, 1]) + >>> b = np.array([2, 2, 2]) + >>> a >= b + array([ True, True, False]) + """) add_newdoc('numpy.core.umath', 'hypot', @@ -1612,6 +1685,13 @@ def add_newdoc(place, name, doc): >>> np.invert(np.array([True, False])) array([False, True]) + The ``~`` operator can be used as a shorthand for ``np.invert`` on + ndarrays. + + >>> x1 = np.array([True, False]) + >>> ~x1 + array([False, True]) + """) add_newdoc('numpy.core.umath', 'isfinite', @@ -1846,6 +1926,14 @@ def add_newdoc(place, name, doc): >>> print(b, type(b)) 254 + The ``<<`` operator can be used as a shorthand for ``np.left_shift`` on + ndarrays. + + >>> x1 = 5 + >>> x2 = np.array([1, 2, 3]) + >>> x1 << x2 + array([10, 20, 40]) + """) add_newdoc('numpy.core.umath', 'less', @@ -1875,11 +1963,18 @@ def add_newdoc(place, name, doc): >>> np.less([1, 2], [2, 2]) array([ True, False]) + The ``<`` operator can be used as a shorthand for ``np.less`` on ndarrays. + + >>> a = np.array([1, 2]) + >>> b = np.array([2, 2]) + >>> a < b + array([ True, False]) + """) add_newdoc('numpy.core.umath', 'less_equal', """ - Return the truth value of (x1 =< x2) element-wise. + Return the truth value of (x1 <= x2) element-wise. Parameters ---------- @@ -1904,6 +1999,14 @@ def add_newdoc(place, name, doc): >>> np.less_equal([4, 2, 1], [2, 2, 2]) array([False, True, True]) + The ``<=`` operator can be used as a shorthand for ``np.less_equal`` on + ndarrays. + + >>> a = np.array([4, 2, 1]) + >>> b = np.array([2, 2, 2]) + >>> a <= b + array([False, True, True]) + """) add_newdoc('numpy.core.umath', 'log', @@ -2231,6 +2334,15 @@ def add_newdoc(place, name, doc): >>> np.logical_and(x>1, x<4) array([False, False, True, True, False]) + + The ``&`` operator can be used as a shorthand for ``np.logical_and`` on + boolean ndarrays. + + >>> a = np.array([True, False]) + >>> b = np.array([False, False]) + >>> a & b + array([False, False]) + """) add_newdoc('numpy.core.umath', 'logical_not', @@ -2301,6 +2413,14 @@ def add_newdoc(place, name, doc): >>> np.logical_or(x < 1, x > 3) array([ True, False, False, False, True]) + The ``|`` operator can be used as a shorthand for ``np.logical_or`` on + boolean ndarrays. + + >>> a = np.array([True, False]) + >>> b = np.array([False, False]) + >>> a | b + array([ True, False]) + """) add_newdoc('numpy.core.umath', 'logical_xor', @@ -2646,8 +2766,8 @@ def add_newdoc(place, name, doc): Raises ------ ValueError - If the last dimension of `a` is not the same size as - the second-to-last dimension of `b`. + If the last dimension of `x1` is not the same size as + the second-to-last dimension of `x2`. If a scalar value is passed in. @@ -2738,6 +2858,14 @@ def add_newdoc(place, name, doc): ... ValueError: matmul: Input operand 1 does not have enough dimensions ... + The ``@`` operator can be used as a shorthand for ``np.matmul`` on + ndarrays. + + >>> x1 = np.array([2j, 3j]) + >>> x2 = np.array([2j, 3j]) + >>> x1 @ x2 + (-13+0j) + .. versionadded:: 1.10.0 """) @@ -2814,6 +2942,16 @@ def add_newdoc(place, name, doc): [ 0., 4., 10.], [ 0., 7., 16.]]) + The ``*`` operator can be used as a shorthand for ``np.multiply`` on + ndarrays. + + >>> x1 = np.arange(9.0).reshape((3, 3)) + >>> x2 = np.arange(3.0) + >>> x1 * x2 + array([[ 0., 1., 4.], + [ 0., 4., 10.], + [ 0., 7., 16.]]) + """) add_newdoc('numpy.core.umath', 'negative', @@ -2837,6 +2975,13 @@ def add_newdoc(place, name, doc): >>> np.negative([1.,-1.]) array([-1., 1.]) + The unary ``-`` operator can be used as a shorthand for ``np.negative`` on + ndarrays. + + >>> x1 = np.array(([1., -1.])) + >>> -x1 + array([-1., 1.]) + """) add_newdoc('numpy.core.umath', 'positive', @@ -2861,6 +3006,20 @@ def add_newdoc(place, name, doc): Equivalent to `x.copy()`, but only defined for types that support arithmetic. + Examples + -------- + + >>> x1 = np.array(([1., -1.])) + >>> np.positive(x1) + array([ 1., -1.]) + + The unary ``+`` operator can be used as a shorthand for ``np.positive`` on + ndarrays. + + >>> x1 = np.array(([1., -1.])) + >>> +x1 + array([ 1., -1.]) + """) add_newdoc('numpy.core.umath', 'not_equal', @@ -2893,6 +3052,15 @@ def add_newdoc(place, name, doc): array([[False, True], [False, True]]) + The ``!=`` operator can be used as a shorthand for ``np.not_equal`` on + ndarrays. + + >>> a = np.array([1., 2.]) + >>> b = np.array([1., 3.]) + >>> a != b + array([False, True]) + + """) add_newdoc('numpy.core.umath', '_ones_like', @@ -2936,9 +3104,9 @@ def add_newdoc(place, name, doc): Examples -------- - Cube each element in a list. + Cube each element in an array. - >>> x1 = range(6) + >>> x1 = np.arange(6) >>> x1 [0, 1, 2, 3, 4, 5] >>> np.power(x1, 3) @@ -2960,6 +3128,14 @@ def add_newdoc(place, name, doc): array([[ 0, 1, 8, 27, 16, 5], [ 0, 1, 8, 27, 16, 5]]) + The ``**`` operator can be used as a shorthand for ``np.power`` on + ndarrays. + + >>> x2 = np.array([1, 2, 3, 3, 2, 1]) + >>> x1 = np.arange(6) + >>> x1 ** x2 + array([ 0, 1, 8, 27, 16, 5]) + """) add_newdoc('numpy.core.umath', 'float_power', @@ -3183,6 +3359,13 @@ def add_newdoc(place, name, doc): >>> np.remainder(np.arange(7), 5) array([0, 1, 2, 3, 4, 0, 1]) + The ``%`` operator can be used as a shorthand for ``np.remainder`` on + ndarrays. + + >>> x1 = np.arange(7) + >>> x1 % 5 + array([0, 1, 2, 3, 4, 0, 1]) + """) add_newdoc('numpy.core.umath', 'divmod', @@ -3225,6 +3408,13 @@ def add_newdoc(place, name, doc): >>> np.divmod(np.arange(5), 3) (array([0, 0, 0, 1, 1]), array([0, 1, 2, 0, 1])) + The `divmod` function can be used as a shorthand for ``np.divmod`` on + ndarrays. + + >>> x = np.arange(5) + >>> divmod(x, 3) + (array([0, 0, 0, 1, 1]), array([0, 1, 2, 0, 1])) + """) add_newdoc('numpy.core.umath', 'right_shift', @@ -3268,6 +3458,14 @@ def add_newdoc(place, name, doc): >>> np.right_shift(10, [1,2,3]) array([5, 2, 1]) + The ``>>`` operator can be used as a shorthand for ``np.right_shift`` on + ndarrays. + + >>> x1 = 10 + >>> x2 = np.array([1,2,3]) + >>> x1 >> x2 + array([5, 2, 1]) + """) add_newdoc('numpy.core.umath', 'rint', @@ -3709,6 +3907,16 @@ def add_newdoc(place, name, doc): [ 3., 3., 3.], [ 6., 6., 6.]]) + The ``-`` operator can be used as a shorthand for ``np.subtract`` on + ndarrays. + + >>> x1 = np.arange(9.0).reshape((3, 3)) + >>> x2 = np.arange(3.0) + >>> x1 - x2 + array([[0., 0., 0.], + [3., 3., 3.], + [6., 6., 6.]]) + """) add_newdoc('numpy.core.umath', 'tan', @@ -3851,6 +4059,14 @@ def add_newdoc(place, name, doc): >>> x//4 array([0, 0, 0, 0, 1]) + + The ``/`` operator can be used as a shorthand for ``np.true_divide`` on + ndarrays. + + >>> x = np.arange(5) + >>> x / 4 + array([0. , 0.25, 0.5 , 0.75, 1. ]) + """) add_newdoc('numpy.core.umath', 'frexp', diff --git a/numpy/core/einsumfunc.py b/numpy/core/einsumfunc.py index f65f4015c928..e0942becaaa5 100644 --- a/numpy/core/einsumfunc.py +++ b/numpy/core/einsumfunc.py @@ -1062,6 +1062,17 @@ def einsum(*operands, out=None, optimize=False, **kwargs): -------- einsum_path, dot, inner, outer, tensordot, linalg.multi_dot + einops: + similar verbose interface is provided by + `einops `_ package to cover + additional operations: transpose, reshape/flatten, repeat/tile, + squeeze/unsqueeze and reductions. + + opt_einsum: + `opt_einsum `_ + optimizes contraction order for einsum-like expressions + in backend-agnostic manner. + Notes ----- .. versionadded:: 1.6.0 diff --git a/numpy/core/fromnumeric.py b/numpy/core/fromnumeric.py index b1524b8915f4..771dfb66e15b 100644 --- a/numpy/core/fromnumeric.py +++ b/numpy/core/fromnumeric.py @@ -1840,11 +1840,11 @@ def nonzero(a): .. note:: When called on a zero-d array or scalar, ``nonzero(a)`` is treated - as ``nonzero(atleast1d(a))``. + as ``nonzero(atleast_1d(a))``. .. deprecated:: 1.17.0 - Use `atleast1d` explicitly if this behavior is deliberate. + Use `atleast_1d` explicitly if this behavior is deliberate. Parameters ---------- diff --git a/numpy/core/fromnumeric.pyi b/numpy/core/fromnumeric.pyi new file mode 100644 index 000000000000..66eb3bfb892f --- /dev/null +++ b/numpy/core/fromnumeric.pyi @@ -0,0 +1,483 @@ +import sys +import datetime as dt +from typing import Optional, Union, Sequence, Tuple, Any, overload, TypeVar + +from numpy import ( + ndarray, + number, + integer, + bool_, + generic, + _OrderKACF, + _OrderACF, + _ArrayLikeBool, + _ArrayLikeIntOrBool, + _ModeKind, + _PartitionKind, + _SortKind, + _SortSide, +) +from numpy.typing import ( + DTypeLike, + ArrayLike, + _ShapeLike, + _Shape, + _IntLike, + _BoolLike, + _NumberLike, +) + +if sys.version_info >= (3, 8): + from typing import Literal +else: + from typing_extensions import Literal + +# Various annotations for scalars + +# While dt.datetime and dt.timedelta are not technically part of NumPy, +# they are one of the rare few builtin scalars which serve as valid return types. +# See https://github.com/numpy/numpy-stubs/pull/67#discussion_r412604113. +_ScalarNumpy = Union[generic, dt.datetime, dt.timedelta] +_ScalarBuiltin = Union[str, bytes, dt.date, dt.timedelta, bool, int, float, complex] +_Scalar = Union[_ScalarBuiltin, _ScalarNumpy] + +# Integers and booleans can generally be used interchangeably +_ScalarIntOrBool = TypeVar("_ScalarIntOrBool", bound=Union[integer, bool_]) +_ScalarGeneric = TypeVar("_ScalarGeneric", bound=generic) +_ScalarGenericDT = TypeVar( + "_ScalarGenericDT", bound=Union[dt.datetime, dt.timedelta, generic] +) + +_Number = TypeVar("_Number", bound=number) + +# The signature of take() follows a common theme with its overloads: +# 1. A generic comes in; the same generic comes out +# 2. A scalar comes in; a generic comes out +# 3. An array-like object comes in; some keyword ensures that a generic comes out +# 4. An array-like object comes in; an ndarray or generic comes out +@overload +def take( + a: _ScalarGenericDT, + indices: int, + axis: Optional[int] = ..., + out: Optional[ndarray] = ..., + mode: _ModeKind = ..., +) -> _ScalarGenericDT: ... +@overload +def take( + a: _Scalar, + indices: int, + axis: Optional[int] = ..., + out: Optional[ndarray] = ..., + mode: _ModeKind = ..., +) -> _ScalarNumpy: ... +@overload +def take( + a: ArrayLike, + indices: int, + axis: Optional[int] = ..., + out: Optional[ndarray] = ..., + mode: _ModeKind = ..., +) -> _ScalarNumpy: ... +@overload +def take( + a: ArrayLike, + indices: _ArrayLikeIntOrBool, + axis: Optional[int] = ..., + out: Optional[ndarray] = ..., + mode: _ModeKind = ..., +) -> Union[_ScalarNumpy, ndarray]: ... +def reshape(a: ArrayLike, newshape: _ShapeLike, order: _OrderACF = ...) -> ndarray: ... +@overload +def choose( + a: _ScalarIntOrBool, + choices: ArrayLike, + out: Optional[ndarray] = ..., + mode: _ModeKind = ..., +) -> _ScalarIntOrBool: ... +@overload +def choose( + a: Union[_IntLike, _BoolLike], choices: ArrayLike, out: Optional[ndarray] = ..., mode: _ModeKind = ... +) -> Union[integer, bool_]: ... +@overload +def choose( + a: _ArrayLikeIntOrBool, + choices: ArrayLike, + out: Optional[ndarray] = ..., + mode: _ModeKind = ..., +) -> ndarray: ... +def repeat( + a: ArrayLike, repeats: _ArrayLikeIntOrBool, axis: Optional[int] = ... +) -> ndarray: ... +def put( + a: ndarray, ind: _ArrayLikeIntOrBool, v: ArrayLike, mode: _ModeKind = ... +) -> None: ... +def swapaxes(a: ArrayLike, axis1: int, axis2: int) -> ndarray: ... +def transpose( + a: ArrayLike, axes: Union[None, Sequence[int], ndarray] = ... +) -> ndarray: ... +def partition( + a: ArrayLike, + kth: _ArrayLikeIntOrBool, + axis: Optional[int] = ..., + kind: _PartitionKind = ..., + order: Union[None, str, Sequence[str]] = ..., +) -> ndarray: ... +@overload +def argpartition( + a: generic, + kth: _ArrayLikeIntOrBool, + axis: Optional[int] = ..., + kind: _PartitionKind = ..., + order: Union[None, str, Sequence[str]] = ..., +) -> integer: ... +@overload +def argpartition( + a: _ScalarBuiltin, + kth: _ArrayLikeIntOrBool, + axis: Optional[int] = ..., + kind: _PartitionKind = ..., + order: Union[None, str, Sequence[str]] = ..., +) -> ndarray: ... +@overload +def argpartition( + a: ArrayLike, + kth: _ArrayLikeIntOrBool, + axis: Optional[int] = ..., + kind: _PartitionKind = ..., + order: Union[None, str, Sequence[str]] = ..., +) -> ndarray: ... +def sort( + a: ArrayLike, + axis: Optional[int] = ..., + kind: Optional[_SortKind] = ..., + order: Union[None, str, Sequence[str]] = ..., +) -> ndarray: ... +def argsort( + a: ArrayLike, + axis: Optional[int] = ..., + kind: Optional[_SortKind] = ..., + order: Union[None, str, Sequence[str]] = ..., +) -> ndarray: ... +@overload +def argmax(a: ArrayLike, axis: None = ..., out: Optional[ndarray] = ...) -> integer: ... +@overload +def argmax( + a: ArrayLike, axis: int = ..., out: Optional[ndarray] = ... +) -> Union[integer, ndarray]: ... +@overload +def argmin(a: ArrayLike, axis: None = ..., out: Optional[ndarray] = ...) -> integer: ... +@overload +def argmin( + a: ArrayLike, axis: int = ..., out: Optional[ndarray] = ... +) -> Union[integer, ndarray]: ... +@overload +def searchsorted( + a: ArrayLike, + v: _Scalar, + side: _SortSide = ..., + sorter: Optional[_ArrayLikeIntOrBool] = ..., # 1D int array +) -> integer: ... +@overload +def searchsorted( + a: ArrayLike, + v: ArrayLike, + side: _SortSide = ..., + sorter: Optional[_ArrayLikeIntOrBool] = ..., # 1D int array +) -> ndarray: ... +def resize(a: ArrayLike, new_shape: _ShapeLike) -> ndarray: ... +@overload +def squeeze(a: _ScalarGeneric, axis: Optional[_ShapeLike] = ...) -> _ScalarGeneric: ... +@overload +def squeeze(a: ArrayLike, axis: Optional[_ShapeLike] = ...) -> ndarray: ... +def diagonal( + a: ArrayLike, offset: int = ..., axis1: int = ..., axis2: int = ... # >= 2D array +) -> ndarray: ... +def trace( + a: ArrayLike, # >= 2D array + offset: int = ..., + axis1: int = ..., + axis2: int = ..., + dtype: DTypeLike = ..., + out: Optional[ndarray] = ..., +) -> Union[number, ndarray]: ... +def ravel(a: ArrayLike, order: _OrderKACF = ...) -> ndarray: ... +def nonzero(a: ArrayLike) -> Tuple[ndarray, ...]: ... +def shape(a: ArrayLike) -> _Shape: ... +def compress( + condition: ArrayLike, # 1D bool array + a: ArrayLike, + axis: Optional[int] = ..., + out: Optional[ndarray] = ..., +) -> ndarray: ... +@overload +def clip( + a: _Number, + a_min: ArrayLike, + a_max: Optional[ArrayLike], + out: Optional[ndarray] = ..., + **kwargs: Any, +) -> _Number: ... +@overload +def clip( + a: _Number, + a_min: None, + a_max: ArrayLike, + out: Optional[ndarray] = ..., + **kwargs: Any, +) -> _Number: ... +@overload +def clip( + a: ArrayLike, + a_min: ArrayLike, + a_max: Optional[ArrayLike], + out: Optional[ndarray] = ..., + **kwargs: Any, +) -> Union[number, ndarray]: ... +@overload +def clip( + a: ArrayLike, + a_min: None, + a_max: ArrayLike, + out: Optional[ndarray] = ..., + **kwargs: Any, +) -> Union[number, ndarray]: ... +@overload +def sum( + a: _Number, + axis: Optional[_ShapeLike] = ..., + dtype: DTypeLike = ..., + out: Optional[ndarray] = ..., + keepdims: bool = ..., + initial: _NumberLike = ..., + where: _ArrayLikeBool = ..., +) -> _Number: ... +@overload +def sum( + a: ArrayLike, + axis: _ShapeLike = ..., + dtype: DTypeLike = ..., + out: Optional[ndarray] = ..., + keepdims: bool = ..., + initial: _NumberLike = ..., + where: _ArrayLikeBool = ..., +) -> Union[number, ndarray]: ... +@overload +def all( + a: ArrayLike, + axis: None = ..., + out: Optional[ndarray] = ..., + keepdims: Literal[False] = ..., +) -> bool_: ... +@overload +def all( + a: ArrayLike, + axis: Optional[_ShapeLike] = ..., + out: Optional[ndarray] = ..., + keepdims: bool = ..., +) -> Union[bool_, ndarray]: ... +@overload +def any( + a: ArrayLike, + axis: None = ..., + out: Optional[ndarray] = ..., + keepdims: Literal[False] = ..., +) -> bool_: ... +@overload +def any( + a: ArrayLike, + axis: Optional[_ShapeLike] = ..., + out: Optional[ndarray] = ..., + keepdims: bool = ..., +) -> Union[bool_, ndarray]: ... +def cumsum( + a: ArrayLike, + axis: Optional[int] = ..., + dtype: DTypeLike = ..., + out: Optional[ndarray] = ..., +) -> ndarray: ... +@overload +def ptp( + a: _Number, + axis: Optional[_ShapeLike] = ..., + out: Optional[ndarray] = ..., + keepdims: bool = ..., +) -> _Number: ... +@overload +def ptp( + a: ArrayLike, + axis: None = ..., + out: Optional[ndarray] = ..., + keepdims: Literal[False] = ..., +) -> number: ... +@overload +def ptp( + a: ArrayLike, + axis: Optional[_ShapeLike] = ..., + out: Optional[ndarray] = ..., + keepdims: bool = ..., +) -> Union[number, ndarray]: ... +@overload +def amax( + a: _Number, + axis: Optional[_ShapeLike] = ..., + out: Optional[ndarray] = ..., + keepdims: bool = ..., + initial: _NumberLike = ..., + where: _ArrayLikeBool = ..., +) -> _Number: ... +@overload +def amax( + a: ArrayLike, + axis: None = ..., + out: Optional[ndarray] = ..., + keepdims: Literal[False] = ..., + initial: _NumberLike = ..., + where: _ArrayLikeBool = ..., +) -> number: ... +@overload +def amax( + a: ArrayLike, + axis: Optional[_ShapeLike] = ..., + out: Optional[ndarray] = ..., + keepdims: bool = ..., + initial: _NumberLike = ..., + where: _ArrayLikeBool = ..., +) -> Union[number, ndarray]: ... +@overload +def amin( + a: _Number, + axis: Optional[_ShapeLike] = ..., + out: Optional[ndarray] = ..., + keepdims: bool = ..., + initial: _NumberLike = ..., + where: _ArrayLikeBool = ..., +) -> _Number: ... +@overload +def amin( + a: ArrayLike, + axis: None = ..., + out: Optional[ndarray] = ..., + keepdims: Literal[False] = ..., + initial: _NumberLike = ..., + where: _ArrayLikeBool = ..., +) -> number: ... +@overload +def amin( + a: ArrayLike, + axis: Optional[_ShapeLike] = ..., + out: Optional[ndarray] = ..., + keepdims: bool = ..., + initial: _NumberLike = ..., + where: _ArrayLikeBool = ..., +) -> Union[number, ndarray]: ... + +# TODO: `np.prod()``: For object arrays `initial` does not necessarily +# have to be a numerical scalar. +# The only requirement is that it is compatible +# with the `.__mul__()` method(s) of the passed array's elements. + +# Note that the same situation holds for all wrappers around +# `np.ufunc.reduce`, e.g. `np.sum()` (`.__add__()`). +@overload +def prod( + a: _Number, + axis: Optional[_ShapeLike] = ..., + dtype: DTypeLike = ..., + out: None = ..., + keepdims: bool = ..., + initial: _NumberLike = ..., + where: _ArrayLikeBool = ..., +) -> _Number: ... +@overload +def prod( + a: ArrayLike, + axis: None = ..., + dtype: DTypeLike = ..., + out: None = ..., + keepdims: Literal[False] = ..., + initial: _NumberLike = ..., + where: _ArrayLikeBool = ..., +) -> number: ... +@overload +def prod( + a: ArrayLike, + axis: Optional[_ShapeLike] = ..., + dtype: DTypeLike = ..., + out: Optional[ndarray] = ..., + keepdims: bool = ..., + initial: _NumberLike = ..., + where: _ArrayLikeBool = ..., +) -> Union[number, ndarray]: ... +def cumprod( + a: ArrayLike, + axis: Optional[int] = ..., + dtype: DTypeLike = ..., + out: Optional[ndarray] = ..., +) -> ndarray: ... +def ndim(a: ArrayLike) -> int: ... +def size(a: ArrayLike, axis: Optional[int] = ...) -> int: ... +@overload +def around( + a: _Number, decimals: int = ..., out: Optional[ndarray] = ... +) -> _Number: ... +@overload +def around( + a: _NumberLike, decimals: int = ..., out: Optional[ndarray] = ... +) -> number: ... +@overload +def around( + a: ArrayLike, decimals: int = ..., out: Optional[ndarray] = ... +) -> ndarray: ... +@overload +def mean( + a: ArrayLike, + axis: None = ..., + dtype: DTypeLike = ..., + out: None = ..., + keepdims: Literal[False] = ..., +) -> number: ... +@overload +def mean( + a: ArrayLike, + axis: Optional[_ShapeLike] = ..., + dtype: DTypeLike = ..., + out: Optional[ndarray] = ..., + keepdims: bool = ..., +) -> Union[number, ndarray]: ... +@overload +def std( + a: ArrayLike, + axis: None = ..., + dtype: DTypeLike = ..., + out: None = ..., + ddof: int = ..., + keepdims: Literal[False] = ..., +) -> number: ... +@overload +def std( + a: ArrayLike, + axis: Optional[_ShapeLike] = ..., + dtype: DTypeLike = ..., + out: Optional[ndarray] = ..., + ddof: int = ..., + keepdims: bool = ..., +) -> Union[number, ndarray]: ... +@overload +def var( + a: ArrayLike, + axis: None = ..., + dtype: DTypeLike = ..., + out: None = ..., + ddof: int = ..., + keepdims: Literal[False] = ..., +) -> number: ... +@overload +def var( + a: ArrayLike, + axis: Optional[_ShapeLike] = ..., + dtype: DTypeLike = ..., + out: Optional[ndarray] = ..., + ddof: int = ..., + keepdims: bool = ..., +) -> Union[number, ndarray]: ... diff --git a/numpy/core/function_base.pyi b/numpy/core/function_base.pyi index c6ebbd5f5db4..1490bed4aff3 100644 --- a/numpy/core/function_base.pyi +++ b/numpy/core/function_base.pyi @@ -1,8 +1,8 @@ import sys from typing import overload, Tuple, Union, Sequence, Any -from numpy import ndarray, inexact, _NumberLike -from numpy.typing import ArrayLike, DtypeLike, _SupportsArray +from numpy import ndarray, inexact +from numpy.typing import ArrayLike, DTypeLike, _SupportsArray, _NumberLike if sys.version_info >= (3, 8): from typing import SupportsIndex, Literal @@ -24,7 +24,7 @@ def linspace( num: SupportsIndex = ..., endpoint: bool = ..., retstep: Literal[False] = ..., - dtype: DtypeLike = ..., + dtype: DTypeLike = ..., axis: SupportsIndex = ..., ) -> ndarray: ... @overload @@ -34,7 +34,7 @@ def linspace( num: SupportsIndex = ..., endpoint: bool = ..., retstep: Literal[True] = ..., - dtype: DtypeLike = ..., + dtype: DTypeLike = ..., axis: SupportsIndex = ..., ) -> Tuple[ndarray, inexact]: ... def logspace( @@ -43,7 +43,7 @@ def logspace( num: SupportsIndex = ..., endpoint: bool = ..., base: _ArrayLikeNumber = ..., - dtype: DtypeLike = ..., + dtype: DTypeLike = ..., axis: SupportsIndex = ..., ) -> ndarray: ... def geomspace( @@ -51,6 +51,6 @@ def geomspace( stop: _ArrayLikeNumber, num: SupportsIndex = ..., endpoint: bool = ..., - dtype: DtypeLike = ..., + dtype: DTypeLike = ..., axis: SupportsIndex = ..., ) -> ndarray: ... diff --git a/numpy/core/include/numpy/ndarraytypes.h b/numpy/core/include/numpy/ndarraytypes.h index 6eca4afdb418..6bf54938f2b2 100644 --- a/numpy/core/include/numpy/ndarraytypes.h +++ b/numpy/core/include/numpy/ndarraytypes.h @@ -1839,6 +1839,10 @@ typedef void (PyDataMem_EventHookFunc)(void *inp, void *outp, size_t size, PyArray_DTypeMeta *cls, PyTypeObject *obj); typedef PyArray_Descr *(default_descr_function)(PyArray_DTypeMeta *cls); + typedef PyArray_DTypeMeta *(common_dtype_function)( + PyArray_DTypeMeta *dtype1, PyArray_DTypeMeta *dtyep2); + typedef PyArray_Descr *(common_instance_function)( + PyArray_Descr *dtype1, PyArray_Descr *dtyep2); /* * While NumPy DTypes would not need to be heap types the plan is to @@ -1894,6 +1898,8 @@ typedef void (PyDataMem_EventHookFunc)(void *inp, void *outp, size_t size, discover_descr_from_pyobject_function *discover_descr_from_pyobject; is_known_scalar_type_function *is_known_scalar_type; default_descr_function *default_descr; + common_dtype_function *common_dtype; + common_instance_function *common_instance; }; #endif /* NPY_INTERNAL_BUILD */ diff --git a/numpy/core/include/numpy/npy_3kcompat.h b/numpy/core/include/numpy/npy_3kcompat.h index 4bc06fc9695e..191cd244f875 100644 --- a/numpy/core/include/numpy/npy_3kcompat.h +++ b/numpy/core/include/numpy/npy_3kcompat.h @@ -28,6 +28,30 @@ extern "C" { * PyInt -> PyLong */ + +/* + * This is a renamed copy of the Python non-limited API function _PyLong_AsInt. It is + * included here because it is missing from the PyPy API. It completes the PyLong_As* + * group of functions and can be useful in replacing PyInt_Check. + */ +static NPY_INLINE int +Npy__PyLong_AsInt(PyObject *obj) +{ + int overflow; + long result = PyLong_AsLongAndOverflow(obj, &overflow); + + /* INT_MAX and INT_MIN are defined in Python.h */ + if (overflow || result > INT_MAX || result < INT_MIN) { + /* XXX: could be cute and give a different + message for overflow == -1 */ + PyErr_SetString(PyExc_OverflowError, + "Python int too large to convert to C int"); + return -1; + } + return (int)result; +} + + #if defined(NPY_PY3K) /* Return True only if the long fits in a C long */ static NPY_INLINE int PyInt_Check(PyObject *op) { @@ -39,6 +63,7 @@ static NPY_INLINE int PyInt_Check(PyObject *op) { return (overflow == 0); } + #define PyInt_FromLong PyLong_FromLong #define PyInt_AsLong PyLong_AsLong #define PyInt_AS_LONG PyLong_AsLong diff --git a/numpy/core/multiarray.py b/numpy/core/multiarray.py index 225c9554cec2..d293700f2ee9 100644 --- a/numpy/core/multiarray.py +++ b/numpy/core/multiarray.py @@ -163,12 +163,12 @@ def concatenate(arrays, axis=None, out=None, *, dtype=None, casting=None): If provided, the destination array will have this dtype. Cannot be provided together with `out`. - ..versionadded:: 1.20.0 + .. versionadded:: 1.20.0 casting : {'no', 'equiv', 'safe', 'same_kind', 'unsafe'}, optional Controls what kind of data casting may occur. Defaults to 'same_kind'. - ..versionadded:: 1.20.0 + .. versionadded:: 1.20.0 Returns ------- @@ -406,7 +406,7 @@ def lexsort(keys, axis=None): for the primary sort order, the second-to-last key for the secondary sort order, and so on. The keys argument must be a sequence of objects that can be converted to arrays of the same shape. If a 2D array is provided - for the keys argument, it's rows are interpreted as the sorting keys and + for the keys argument, its rows are interpreted as the sorting keys and sorting is according to the last row, second last row etc. Parameters @@ -1100,7 +1100,7 @@ def putmask(a, mask, values): Parameters ---------- - a : array_like + a : ndarray Target array. mask : array_like Boolean mask array. It has to be the same shape as `a`. diff --git a/numpy/core/numeric.py b/numpy/core/numeric.py index a023bf0da32e..25235f738bd2 100644 --- a/numpy/core/numeric.py +++ b/numpy/core/numeric.py @@ -409,7 +409,7 @@ def full_like(a, fill_value, dtype=None, order='K', subok=True, shape=None): >>> y = np.arange(6, dtype=np.double) >>> np.full_like(y, 0.1) - array([0.1, 0.1, 0.1, 0.1, 0.1, 0.1]) + array([0.1, 0.1, 0.1, 0.1, 0.1, 0.1]) """ res = empty_like(a, dtype=dtype, order=order, subok=subok, shape=shape) diff --git a/numpy/core/numeric.pyi b/numpy/core/numeric.pyi new file mode 100644 index 000000000000..d91cb31c2031 --- /dev/null +++ b/numpy/core/numeric.pyi @@ -0,0 +1,189 @@ +import sys +from typing import ( + Any, + Optional, + Union, + Sequence, + Tuple, + Callable, + List, + overload, + TypeVar, + Iterable, +) + +from numpy import ndarray, generic, dtype, bool_, signedinteger, _OrderKACF, _OrderCF +from numpy.typing import ArrayLike, DTypeLike, _ShapeLike + +if sys.version_info >= (3, 8): + from typing import Literal +else: + from typing_extensions import Literal + +_T = TypeVar("_T") +_ArrayType = TypeVar("_ArrayType", bound=ndarray) + +_CorrelateMode = Literal["valid", "same", "full"] + +@overload +def zeros_like( + a: _ArrayType, + dtype: None = ..., + order: _OrderKACF = ..., + subok: Literal[True] = ..., + shape: None = ..., +) -> _ArrayType: ... +@overload +def zeros_like( + a: ArrayLike, + dtype: DTypeLike = ..., + order: _OrderKACF = ..., + subok: bool = ..., + shape: Optional[_ShapeLike] = ..., +) -> ndarray: ... +def ones( + shape: _ShapeLike, + dtype: DTypeLike = ..., + order: _OrderCF = ..., + *, + like: ArrayLike = ..., +) -> ndarray: ... +@overload +def ones_like( + a: _ArrayType, + dtype: None = ..., + order: _OrderKACF = ..., + subok: Literal[True] = ..., + shape: None = ..., +) -> _ArrayType: ... +@overload +def ones_like( + a: ArrayLike, + dtype: DTypeLike = ..., + order: _OrderKACF = ..., + subok: bool = ..., + shape: Optional[_ShapeLike] = ..., +) -> ndarray: ... +@overload +def empty_like( + a: _ArrayType, + dtype: None = ..., + order: _OrderKACF = ..., + subok: Literal[True] = ..., + shape: None = ..., +) -> _ArrayType: ... +@overload +def empty_like( + a: ArrayLike, + dtype: DTypeLike = ..., + order: _OrderKACF = ..., + subok: bool = ..., + shape: Optional[_ShapeLike] = ..., +) -> ndarray: ... +def full( + shape: _ShapeLike, + fill_value: Any, + dtype: DTypeLike = ..., + order: _OrderCF = ..., + *, + like: ArrayLike = ..., +) -> ndarray: ... +@overload +def full_like( + a: _ArrayType, + fill_value: Any, + dtype: None = ..., + order: _OrderKACF = ..., + subok: Literal[True] = ..., + shape: None = ..., +) -> _ArrayType: ... +@overload +def full_like( + a: ArrayLike, + fill_value: Any, + dtype: DTypeLike = ..., + order: _OrderKACF = ..., + subok: bool = ..., + shape: Optional[_ShapeLike] = ..., +) -> ndarray: ... +@overload +def count_nonzero( + a: ArrayLike, axis: None = ..., *, keepdims: Literal[False] = ... +) -> int: ... +@overload +def count_nonzero( + a: ArrayLike, axis: _ShapeLike = ..., *, keepdims: bool = ... +) -> Union[signedinteger[Any], ndarray]: ... # TODO: np.intp +def isfortran(a: Union[ndarray, generic]) -> bool: ... +def argwhere(a: ArrayLike) -> ndarray: ... +def flatnonzero(a: ArrayLike) -> ndarray: ... +def correlate(a: ArrayLike, v: ArrayLike, mode: _CorrelateMode = ...) -> ndarray: ... +def convolve(a: ArrayLike, v: ArrayLike, mode: _CorrelateMode = ...) -> ndarray: ... +@overload +def outer(a: ArrayLike, b: ArrayLike, out: None = ...) -> ndarray: ... +@overload +def outer(a: ArrayLike, b: ArrayLike, out: _ArrayType = ...) -> _ArrayType: ... +def tensordot( + a: ArrayLike, + b: ArrayLike, + axes: Union[int, Tuple[_ShapeLike, _ShapeLike]] = ..., +) -> ndarray: ... +def roll( + a: ArrayLike, + shift: _ShapeLike, + axis: Optional[_ShapeLike] = ..., +) -> ndarray: ... +def rollaxis(a: ndarray, axis: int, start: int = ...) -> ndarray: ... +def moveaxis( + a: ndarray, + source: _ShapeLike, + destination: _ShapeLike, +) -> ndarray: ... +def cross( + a: ArrayLike, + b: ArrayLike, + axisa: int = ..., + axisb: int = ..., + axisc: int = ..., + axis: Optional[int] = ..., +) -> ndarray: ... +@overload +def indices( + dimensions: Sequence[int], + dtype: DTypeLike = ..., + sparse: Literal[False] = ..., +) -> ndarray: ... +@overload +def indices( + dimensions: Sequence[int], + dtype: DTypeLike = ..., + sparse: Literal[True] = ..., +) -> Tuple[ndarray, ...]: ... +def fromfunction( + function: Callable[..., _T], + shape: Sequence[int], + *, + dtype: DTypeLike = ..., + like: ArrayLike = ..., + **kwargs: Any, +) -> _T: ... +def isscalar(element: Any) -> bool: ... +def binary_repr(num: int, width: Optional[int] = ...) -> str: ... +def base_repr(number: int, base: int = ..., padding: int = ...) -> str: ... +def identity(n: int, dtype: DTypeLike = ..., *, like: ArrayLike = ...) -> ndarray: ... +def allclose( + a: ArrayLike, + b: ArrayLike, + rtol: float = ..., + atol: float = ..., + equal_nan: bool = ..., +) -> bool: ... +def isclose( + a: ArrayLike, + b: ArrayLike, + rtol: float = ..., + atol: float = ..., + equal_nan: bool = ..., +) -> Union[bool_, ndarray]: ... +def array_equal(a1: ArrayLike, a2: ArrayLike) -> bool: ... +def array_equiv(a1: ArrayLike, a2: ArrayLike) -> bool: ... diff --git a/numpy/core/numerictypes.py b/numpy/core/numerictypes.py index 2a015f48fc98..e705dd3ea855 100644 --- a/numpy/core/numerictypes.py +++ b/numpy/core/numerictypes.py @@ -358,13 +358,15 @@ def issubsctype(arg1, arg2): @set_module('numpy') def issubdtype(arg1, arg2): - """ + r""" Returns True if first argument is a typecode lower/equal in type hierarchy. + This is like the builtin :func:`issubclass`, but for `dtype`\ s. + Parameters ---------- arg1, arg2 : dtype_like - dtype or string representing a typecode. + `dtype` or object coercible to one Returns ------- @@ -372,15 +374,45 @@ def issubdtype(arg1, arg2): See Also -------- + :ref:`arrays.scalars` : Overview of the numpy type hierarchy. issubsctype, issubclass_ - numpy.core.numerictypes : Overview of numpy type hierarchy. Examples -------- - >>> np.issubdtype('S1', np.string_) + `issubdtype` can be used to check the type of arrays: + + >>> ints = np.array([1, 2, 3], dtype=np.int32) + >>> np.issubdtype(ints.dtype, np.integer) + True + >>> np.issubdtype(ints.dtype, np.floating) + False + + >>> floats = np.array([1, 2, 3], dtype=np.float32) + >>> np.issubdtype(floats.dtype, np.integer) + False + >>> np.issubdtype(floats.dtype, np.floating) True + + Similar types of different sizes are not subdtypes of each other: + >>> np.issubdtype(np.float64, np.float32) False + >>> np.issubdtype(np.float32, np.float64) + False + + but both are subtypes of `floating`: + + >>> np.issubdtype(np.float64, np.floating) + True + >>> np.issubdtype(np.float32, np.floating) + True + + For convenience, dtype-like objects are allowed too: + + >>> np.issubdtype('S1', np.string_) + True + >>> np.issubdtype('i4', np.signedinteger) + True """ if not issubclass_(arg1, generic): diff --git a/numpy/core/numerictypes.pyi b/numpy/core/numerictypes.pyi new file mode 100644 index 000000000000..192015ff13b4 --- /dev/null +++ b/numpy/core/numerictypes.pyi @@ -0,0 +1,29 @@ +from typing import TypeVar, Optional, Type, Union, Tuple, Sequence, overload, Any + +from numpy import generic, ndarray, dtype +from numpy.typing import DTypeLike + +_DefaultType = TypeVar("_DefaultType") + +def maximum_sctype(t: DTypeLike) -> dtype: ... +def issctype(rep: object) -> bool: ... +@overload +def obj2sctype(rep: object) -> Optional[generic]: ... +@overload +def obj2sctype(rep: object, default: None) -> Optional[generic]: ... +@overload +def obj2sctype( + rep: object, default: Type[_DefaultType] +) -> Union[generic, Type[_DefaultType]]: ... +def issubclass_(arg1: object, arg2: Union[object, Tuple[object, ...]]) -> bool: ... +def issubsctype( + arg1: Union[ndarray, DTypeLike], arg2: Union[ndarray, DTypeLike] +) -> bool: ... +def issubdtype(arg1: DTypeLike, arg2: DTypeLike) -> bool: ... +def sctype2char(sctype: object) -> str: ... +def find_common_type( + array_types: Sequence[DTypeLike], scalar_types: Sequence[DTypeLike] +) -> dtype: ... + +# TODO: Add annotations for the following objects: +# typeDict, nbytes, cast, ScalarType & typecodes diff --git a/numpy/core/setup.py b/numpy/core/setup.py index 92dcacede412..775c60d5ed2d 100644 --- a/numpy/core/setup.py +++ b/numpy/core/setup.py @@ -102,7 +102,7 @@ def win32_checks(deflist): if a == "Intel" or a == "AMD64": deflist.append('FORCE_NO_LONG_DOUBLE_FORMATTING') -def check_math_capabilities(config, moredefs, mathlibs): +def check_math_capabilities(config, ext, moredefs, mathlibs): def check_func(func_name): return config.check_func(func_name, libraries=mathlibs, decl=True, call=True) @@ -167,6 +167,14 @@ def check_funcs(funcs_name): for dec, fn in OPTIONAL_FUNCTION_ATTRIBUTES: if config.check_gcc_function_attribute(dec, fn): moredefs.append((fname2def(fn), 1)) + if fn == 'attribute_target_avx512f': + # GH-14787: Work around GCC<8.4 bug when compiling with AVX512 + # support on Windows-based platforms + if (sys.platform in ('win32', 'cygwin') and + config.check_compiler_gcc() and + not config.check_gcc_version_at_least(8, 4)): + ext.extra_compile_args.extend( + ['-ffixed-xmm%s' % n for n in range(16, 32)]) for dec, fn, code, header in OPTIONAL_FUNCTION_ATTRIBUTES_WITH_INTRINSICS: if config.check_gcc_function_attribute_with_intrinsics(dec, fn, code, @@ -434,7 +442,7 @@ def generate_config_h(ext, build_dir): mathlibs = check_mathlib(config_cmd) moredefs.append(('MATHLIB', ','.join(mathlibs))) - check_math_capabilities(config_cmd, moredefs, mathlibs) + check_math_capabilities(config_cmd, ext, moredefs, mathlibs) moredefs.extend(cocache.check_ieee_macros(config_cmd)[0]) moredefs.extend(cocache.check_complex(config_cmd, mathlibs)[0]) @@ -618,6 +626,7 @@ def generate_api(ext, build_dir): config.add_include_dirs(join('src', 'multiarray')) config.add_include_dirs(join('src', 'umath')) config.add_include_dirs(join('src', 'npysort')) + config.add_include_dirs(join('src', '_simd')) config.add_define_macros([("NPY_INTERNAL_BUILD", "1")]) # this macro indicates that Numpy build is in process config.add_define_macros([("HAVE_NPY_CONFIG_H", "1")]) @@ -679,7 +688,7 @@ def get_mathlib_info(*args): install_dir='lib', build_info={ 'include_dirs' : [], # empty list required for creating npy_math_internal.h - 'extra_compiler_args' : (['/GL-'] if is_msvc else []), + 'extra_compiler_args' : (['/GL-','/MT'] if is_msvc else []), }) config.add_npy_pkg_config("npymath.ini.in", "lib/npy-pkg-config", subst_dict) @@ -931,8 +940,11 @@ def generate_umath_c(ext, build_dir): ], depends=deps + multiarray_deps + umath_deps + common_deps, - libraries=['npymath'], - extra_info=extra_info) + libraries=['npymath','libucrt'], + extra_info=extra_info, + extra_compile_args=["/MT"], + #extra_link_args=["/MT","libucrt.lib"] + ) ####################################################################### # umath_tests module # @@ -966,6 +978,28 @@ def generate_umath_c(ext, build_dir): config.add_extension('_operand_flag_tests', sources=[join('src', 'umath', '_operand_flag_tests.c.src')]) + ####################################################################### + # SIMD module # + ####################################################################### + + config.add_extension('_simd', sources=[ + join('src', 'common', 'npy_cpu_features.c.src'), + join('src', '_simd', '_simd.c'), + join('src', '_simd', '_simd_inc.h.src'), + join('src', '_simd', '_simd_data.inc.src'), + join('src', '_simd', '_simd.dispatch.c.src'), + ], depends=[ + join('src', 'common', 'npy_cpu_dispatch.h'), + join('src', 'common', 'simd', 'simd.h'), + join('src', '_simd', '_simd.h'), + join('src', '_simd', '_simd_inc.h.src'), + join('src', '_simd', '_simd_data.inc.src'), + join('src', '_simd', '_simd_arg.inc'), + join('src', '_simd', '_simd_convert.inc'), + join('src', '_simd', '_simd_easyintrin.inc'), + join('src', '_simd', '_simd_vector.inc'), + ]) + config.add_subpackage('tests') config.add_data_dir('tests/data') config.add_data_dir('tests/examples') diff --git a/numpy/core/shape_base.pyi b/numpy/core/shape_base.pyi new file mode 100644 index 000000000000..b20598b1ab5b --- /dev/null +++ b/numpy/core/shape_base.pyi @@ -0,0 +1,41 @@ +import sys +from typing import TypeVar, overload, List, Sequence + +from numpy import ndarray +from numpy.typing import ArrayLike + +if sys.version_info >= (3, 8): + from typing import SupportsIndex +else: + from typing_extensions import Protocol + class SupportsIndex(Protocol): + def __index__(self) -> int: ... + +_ArrayType = TypeVar("_ArrayType", bound=ndarray) + +@overload +def atleast_1d(__arys: ArrayLike) -> ndarray: ... +@overload +def atleast_1d(*arys: ArrayLike) -> List[ndarray]: ... + +@overload +def atleast_2d(__arys: ArrayLike) -> ndarray: ... +@overload +def atleast_2d(*arys: ArrayLike) -> List[ndarray]: ... + +@overload +def atleast_3d(__arys: ArrayLike) -> ndarray: ... +@overload +def atleast_3d(*arys: ArrayLike) -> List[ndarray]: ... + +def vstack(tup: Sequence[ArrayLike]) -> ndarray: ... +def hstack(tup: Sequence[ArrayLike]) -> ndarray: ... +@overload +def stack( + arrays: Sequence[ArrayLike], axis: SupportsIndex = ..., out: None = ... +) -> ndarray: ... +@overload +def stack( + arrays: Sequence[ArrayLike], axis: SupportsIndex = ..., out: _ArrayType = ... +) -> _ArrayType: ... +def block(arrays: ArrayLike) -> ndarray: ... diff --git a/numpy/core/src/_simd/_simd.c b/numpy/core/src/_simd/_simd.c new file mode 100644 index 000000000000..b1fdd4478d9d --- /dev/null +++ b/numpy/core/src/_simd/_simd.c @@ -0,0 +1,73 @@ +#include "_simd.h" + +PyMODINIT_FUNC PyInit__simd(void) +{ + static struct PyModuleDef defs = { + .m_base = PyModuleDef_HEAD_INIT, + .m_name = "numpy.core._simd", + .m_size = -1 + }; + if (npy_cpu_init() < 0) { + return NULL; + } + PyObject *m = PyModule_Create(&defs); + if (m == NULL) { + return NULL; + } + PyObject *targets = PyDict_New(); + if (targets == NULL) { + goto err; + } + if (PyModule_AddObject(m, "targets", targets) < 0) { + Py_DECREF(targets); + goto err; + } + // add keys for non-supported optimizations with None value + #define ATTACH_MODULE(TESTED_FEATURES, TARGET_NAME, MAKE_MSVC_HAPPY) \ + { \ + PyObject *simd_mod; \ + if (!TESTED_FEATURES) { \ + Py_INCREF(Py_None); \ + simd_mod = Py_None; \ + } else { \ + simd_mod = NPY_CAT(simd_create_module_, TARGET_NAME)(); \ + if (simd_mod == NULL) { \ + goto err; \ + } \ + } \ + const char *target_name = NPY_TOSTRING(TARGET_NAME); \ + if (PyDict_SetItemString(targets, target_name, simd_mod) < 0) { \ + Py_DECREF(simd_mod); \ + goto err; \ + } \ + Py_INCREF(simd_mod); \ + if (PyModule_AddObject(m, target_name, simd_mod) < 0) { \ + Py_DECREF(simd_mod); \ + goto err; \ + } \ + } + + #define ATTACH_BASELINE_MODULE(MAKE_MSVC_HAPPY) \ + { \ + PyObject *simd_mod = simd_create_module(); \ + if (simd_mod == NULL) { \ + goto err; \ + } \ + if (PyDict_SetItemString(targets, "baseline", simd_mod) < 0) { \ + Py_DECREF(simd_mod); \ + goto err; \ + } \ + Py_INCREF(simd_mod); \ + if (PyModule_AddObject(m, "baseline", simd_mod) < 0) { \ + Py_DECREF(simd_mod); \ + goto err; \ + } \ + } + + NPY__CPU_DISPATCH_CALL(NPY_CPU_HAVE, ATTACH_MODULE, MAKE_MSVC_HAPPY) + NPY__CPU_DISPATCH_BASELINE_CALL(ATTACH_BASELINE_MODULE, MAKE_MSVC_HAPPY) + return m; +err: + Py_DECREF(m); + return NULL; +} diff --git a/numpy/core/src/_simd/_simd.dispatch.c.src b/numpy/core/src/_simd/_simd.dispatch.c.src new file mode 100644 index 000000000000..3d7af23331b7 --- /dev/null +++ b/numpy/core/src/_simd/_simd.dispatch.c.src @@ -0,0 +1,559 @@ +/*@targets $werror #simd_test*/ +#include "_simd.h" +#include "_simd_inc.h" + +#if NPY_SIMD +#include "_simd_data.inc" +#include "_simd_convert.inc" +#include "_simd_vector.inc" +#include "_simd_arg.inc" +#include "_simd_easyintrin.inc" + +/************************************************************************* + * Defining NPYV intrinsics as module functions + *************************************************************************/ +/**begin repeat + * #sfx = u8, s8, u16, s16, u32, s32, u64, s64, f32, f64# + * #bsfx = b8, b8, b16, b16, b32, b32, b64, b64, b32, b64# + * #simd_sup = 1, 1, 1, 1, 1, 1, 1, 1, 1, NPY_SIMD_F64# + * #sat_sup = 1, 1, 1, 1, 0, 0, 0, 0, 0, 0# + * #mul_sup = 1, 1, 1, 1, 1, 1, 0, 0, 1, 1# + * #div_sup = 0, 0, 0, 0, 0, 0, 0, 0, 1, 1# + * #fused_sup = 0, 0, 0, 0, 0, 0, 0, 0, 1, 1# + * #sum_sup = 0, 0, 0, 0, 0, 0, 0, 0, 1, 1# + * #ncont_sup = 0, 0, 0, 0, 1, 1, 1, 1, 1, 1# + * #shl_imm = 0, 0, 15, 15, 31, 31, 63, 63, 0, 0# + * #shr_imm = 0, 0, 16, 16, 32, 32, 64, 64, 0, 0# + */ +#if @simd_sup@ +/*************************** + * Memory + ***************************/ +/**begin repeat1 + * # intrin = load, loada, loads, loadl# + */ +SIMD_IMPL_INTRIN_1(@intrin@_@sfx@, v@sfx@, q@sfx@) +/**end repeat1**/ +/**begin repeat1 + * # intrin = store, storea, stores, storel, storeh# + */ +// special definition due to the nature of @intrin@ +static PyObject * +simd__intrin_@intrin@_@sfx@(PyObject* NPY_UNUSED(self), PyObject *args) +{ + simd_arg seq_arg = {.dtype = simd_data_q@sfx@}; + simd_arg vec_arg = {.dtype = simd_data_v@sfx@}; + if (!PyArg_ParseTuple( + args, "O&O&:@intrin@_@sfx@", + simd_arg_converter, &seq_arg, + simd_arg_converter, &vec_arg + )) { + return NULL; + } + npyv_@intrin@_@sfx@(seq_arg.data.q@sfx@, vec_arg.data.v@sfx@); + // write-back + if (simd_sequence_fill_iterable(seq_arg.obj, seq_arg.data.q@sfx@, simd_data_q@sfx@)) { + simd_arg_free(&seq_arg); + return NULL; + } + simd_arg_free(&seq_arg); + Py_RETURN_NONE; +} +/**end repeat1**/ + +/**************************************** + * Non-contiguous/Partial Memory access + ****************************************/ +#if @ncont_sup@ +// Partial Load +SIMD_IMPL_INTRIN_3(load_till_@sfx@, v@sfx@, q@sfx@, u32, @sfx@) +SIMD_IMPL_INTRIN_2(load_tillz_@sfx@, v@sfx@, q@sfx@, u32) + +// Partial Store +static PyObject * +simd__intrin_store_till_@sfx@(PyObject* NPY_UNUSED(self), PyObject *args) +{ + simd_arg seq_arg = {.dtype = simd_data_q@sfx@}; + simd_arg nlane_arg = {.dtype = simd_data_u32}; + simd_arg vec_arg = {.dtype = simd_data_v@sfx@}; + if (!PyArg_ParseTuple( + args, "O&O&O&:store_till_@sfx@", + simd_arg_converter, &seq_arg, + simd_arg_converter, &nlane_arg, + simd_arg_converter, &vec_arg + )) { + return NULL; + } + npyv_store_till_@sfx@( + seq_arg.data.q@sfx@, nlane_arg.data.u32, vec_arg.data.v@sfx@ + ); + // write-back + if (simd_sequence_fill_iterable(seq_arg.obj, seq_arg.data.q@sfx@, simd_data_q@sfx@)) { + simd_arg_free(&seq_arg); + return NULL; + } + simd_arg_free(&seq_arg); + Py_RETURN_NONE; +} + +// Non-contiguous Load +/**begin repeat1 + * #intrin = loadn, loadn_till, loadn_tillz# + * #till = 0, 1, 1# + * #fill = 0, 1, 0# + * #format = , O&O&, O&# + */ +static PyObject * +simd__intrin_@intrin@_@sfx@(PyObject* NPY_UNUSED(self), PyObject *args) +{ + simd_arg seq_arg = {.dtype = simd_data_q@sfx@}; + simd_arg stride_arg = {.dtype = simd_data_s64}; +#if @till@ + simd_arg nlane_arg = {.dtype = simd_data_u32}; +#endif // till +#if @fill@ + simd_arg fill_arg = {.dtype = simd_data_@sfx@}; +#endif + if (!PyArg_ParseTuple( + args, "@format@O&O&:@intrin@_@sfx@", + simd_arg_converter, &seq_arg, + simd_arg_converter, &stride_arg +#if @till@ + ,simd_arg_converter, &nlane_arg +#endif +#if @fill@ + ,simd_arg_converter, &fill_arg +#endif + )) { + return NULL; + } + npyv_lanetype_@sfx@ *seq_ptr = seq_arg.data.q@sfx@; + npy_intp stride = (npy_intp)stride_arg.data.s64; + Py_ssize_t cur_seq_len = simd_sequence_len(seq_ptr); + Py_ssize_t min_seq_len = stride * npyv_nlanes_@sfx@; + if (stride < 0) { + seq_ptr += cur_seq_len -1; + min_seq_len = -min_seq_len; + } + if (cur_seq_len < min_seq_len) { + PyErr_Format(PyExc_ValueError, + "@intrin@_@sfx@(), according to provided stride %d, the " + "minimum acceptable size of the required sequence is %d, given(%d)", + stride, min_seq_len, cur_seq_len + ); + goto err; + } + npyv_@sfx@ rvec = npyv_@intrin@_@sfx@( + seq_ptr, stride + #if @till@ + , nlane_arg.data.u32 + #endif + #if @fill@ + , fill_arg.data.@sfx@ + #endif + ); + simd_arg ret = { + .dtype = simd_data_v@sfx@, .data = {.v@sfx@=rvec} + }; + simd_arg_free(&seq_arg); + return simd_arg_to_obj(&ret); +err: + simd_arg_free(&seq_arg); + return NULL; +} +/**end repeat1**/ + +// Non-contiguous Store +/**begin repeat1 + * #intrin = storen, storen_till# + * #till = 0, 1# + * #format = , O&# + */ +static PyObject * +simd__intrin_@intrin@_@sfx@(PyObject* NPY_UNUSED(self), PyObject *args) +{ + simd_arg seq_arg = {.dtype = simd_data_q@sfx@}; + simd_arg stride_arg = {.dtype = simd_data_s64}; + simd_arg vec_arg = {.dtype = simd_data_v@sfx@}; +#if @till@ + simd_arg nlane_arg = {.dtype = simd_data_u32}; +#endif + if (!PyArg_ParseTuple( + args, "@format@O&O&O&:storen_@sfx@", + simd_arg_converter, &seq_arg, + simd_arg_converter, &stride_arg +#if @till@ + ,simd_arg_converter, &nlane_arg +#endif + ,simd_arg_converter, &vec_arg + )) { + return NULL; + } + npyv_lanetype_@sfx@ *seq_ptr = seq_arg.data.q@sfx@; + npy_intp stride = (npy_intp)stride_arg.data.s64; + Py_ssize_t cur_seq_len = simd_sequence_len(seq_ptr); + Py_ssize_t min_seq_len = stride * npyv_nlanes_@sfx@; + if (stride < 0) { + seq_ptr += cur_seq_len -1; + min_seq_len = -min_seq_len; + } + // overflow guard + if (cur_seq_len < min_seq_len) { + PyErr_Format(PyExc_ValueError, + "@intrin@_@sfx@(), according to provided stride %d, the" + "minimum acceptable size of the required sequence is %d, given(%d)", + stride, min_seq_len, cur_seq_len + ); + goto err; + } + npyv_@intrin@_@sfx@( + seq_ptr, stride + #if @till@ + ,nlane_arg.data.u32 + #endif + ,vec_arg.data.v@sfx@ + ); + // write-back + if (simd_sequence_fill_iterable(seq_arg.obj, seq_arg.data.q@sfx@, simd_data_q@sfx@)) { + goto err; + } + simd_arg_free(&seq_arg); + Py_RETURN_NONE; +err: + simd_arg_free(&seq_arg); + return NULL; +} +/**end repeat1**/ +#endif // @ncont_sup@ + + +/*************************** + * Misc + ***************************/ +SIMD_IMPL_INTRIN_0(zero_@sfx@, v@sfx@) +SIMD_IMPL_INTRIN_1(setall_@sfx@, v@sfx@, @sfx@) +SIMD_IMPL_INTRIN_3(select_@sfx@, v@sfx@, v@bsfx@, v@sfx@, v@sfx@) + +/**begin repeat1 + * #sfx_to = u8, s8, u16, s16, u32, s32, u64, s64, f32, f64# + * #simd_sup2 = 1, 1, 1, 1, 1, 1, 1, 1, 1, NPY_SIMD_F64# + */ +#if @simd_sup2@ +SIMD_IMPL_INTRIN_1(reinterpret_@sfx_to@_@sfx@, v@sfx_to@, v@sfx@) +#endif // simd_sup2 +/**end repeat1**/ + +/** + * special definition due to the nature of intrinsics + * npyv_setf_@sfx@ and npy_set_@sfx@. +*/ +/**begin repeat1 + * #intrin = setf, set# + */ +static PyObject * +simd__intrin_@intrin@_@sfx@(PyObject* NPY_UNUSED(self), PyObject *args) +{ + npyv_lanetype_@sfx@ *data = simd_sequence_from_iterable(args, simd_data_q@sfx@, npyv_nlanes_@sfx@); + if (data == NULL) { + return NULL; + } + simd_data r = {.v@sfx@ = npyv_@intrin@_@sfx@( + data[0], data[1], data[2], data[3], data[4], data[5], data[6], data[7], + data[8], data[9], data[10], data[11], data[12], data[13], data[14], data[15], + data[16], data[17], data[18], data[19], data[20], data[21], data[22], data[23], + data[24], data[25], data[26], data[27], data[28], data[29], data[30], data[31], + data[32], data[33], data[34], data[35], data[36], data[37], data[38], data[39], + data[40], data[41], data[42], data[43], data[44], data[45], data[46], data[47], + data[48], data[49], data[50], data[51], data[52], data[53], data[54], data[55], + data[56], data[57], data[58], data[59], data[60], data[61], data[62], data[63], + data[64] // for setf + )}; + simd_sequence_free(data); + return (PyObject*)PySIMDVector_FromData(r, simd_data_v@sfx@); +} +/**end repeat1**/ + +/*************************** + * Reorder + ***************************/ +/**begin repeat1 + * # intrin = combinel, combineh# + */ +SIMD_IMPL_INTRIN_2(@intrin@_@sfx@, v@sfx@, v@sfx@, v@sfx@) +/**end repeat1**/ + +/**begin repeat1 + * # intrin = combine, zip# + */ +SIMD_IMPL_INTRIN_2(@intrin@_@sfx@, v@sfx@x2, v@sfx@, v@sfx@) +/**end repeat1**/ + +/*************************** + * Operators + ***************************/ +#if @shl_imm@ > 0 +SIMD_IMPL_INTRIN_2(shl_@sfx@, v@sfx@, v@sfx@, u8) +SIMD_IMPL_INTRIN_2(shr_@sfx@, v@sfx@, v@sfx@, u8) +// immediate constant +SIMD_IMPL_INTRIN_2IMM(shli_@sfx@, v@sfx@, v@sfx@, @shl_imm@) +SIMD_IMPL_INTRIN_2IMM(shri_@sfx@, v@sfx@, v@sfx@, @shr_imm@) +#endif // shl_imm + +/**begin repeat1 + * #intrin = and, or, xor# + */ +SIMD_IMPL_INTRIN_2(@intrin@_@sfx@, v@sfx@, v@sfx@, v@sfx@) +/**end repeat1**/ + +SIMD_IMPL_INTRIN_1(not_@sfx@, v@sfx@, v@sfx@) + +/**begin repeat1 + * #intrin = cmpeq, cmpneq, cmpgt, cmpge, cmplt, cmple# + */ +SIMD_IMPL_INTRIN_2(@intrin@_@sfx@, v@bsfx@, v@sfx@, v@sfx@) +/**end repeat1**/ + +/*************************** + * Conversion + ***************************/ +SIMD_IMPL_INTRIN_1(cvt_@sfx@_@bsfx@, v@sfx@, v@bsfx@) +SIMD_IMPL_INTRIN_1(cvt_@bsfx@_@sfx@, v@bsfx@, v@sfx@) + +/*************************** + * Arithmetic + ***************************/ +/**begin repeat1 + * #intrin = add, sub# + */ +SIMD_IMPL_INTRIN_2(@intrin@_@sfx@, v@sfx@, v@sfx@, v@sfx@) +/**end repeat1**/ + +#if @sat_sup@ +/**begin repeat1 + * #intrin = adds, subs# + */ +SIMD_IMPL_INTRIN_2(@intrin@_@sfx@, v@sfx@, v@sfx@, v@sfx@) +/**end repeat1**/ +#endif // sat_sup + +#if @mul_sup@ +SIMD_IMPL_INTRIN_2(mul_@sfx@, v@sfx@, v@sfx@, v@sfx@) +#endif // mul_sup + +#if @div_sup@ +SIMD_IMPL_INTRIN_2(div_@sfx@, v@sfx@, v@sfx@, v@sfx@) +#endif // div_sup + +#if @fused_sup@ +/**begin repeat1 + * #intrin = muladd, mulsub, nmuladd, nmulsub# + */ +SIMD_IMPL_INTRIN_3(@intrin@_@sfx@, v@sfx@, v@sfx@, v@sfx@, v@sfx@) +/**end repeat1**/ +#endif // fused_sup + +#if @sum_sup@ +SIMD_IMPL_INTRIN_1(sum_@sfx@, @sfx@, v@sfx@) +#endif // sum_sup + +#endif // simd_sup +/**end repeat**/ +/*************************** + * Variant + ***************************/ +SIMD_IMPL_INTRIN_0N(cleanup) + +/************************************************************************* + * Attach module functions + *************************************************************************/ +static PyMethodDef simd__intrinsics_methods[] = { +/**begin repeat + * #sfx = u8, s8, u16, s16, u32, s32, u64, s64, f32, f64# + * #bsfx = b8, b8, b16, b16, b32, b32, b64, b64, b32, b64# + * #simd_sup = 1, 1, 1, 1, 1, 1, 1, 1, 1, NPY_SIMD_F64# + * #sat_sup = 1, 1, 1, 1, 0, 0, 0, 0, 0, 0# + * #mul_sup = 1, 1, 1, 1, 1, 1, 0, 0, 1, 1# + * #div_sup = 0, 0, 0, 0, 0, 0, 0, 0, 1, 1# + * #fused_sup = 0, 0, 0, 0, 0, 0, 0, 0, 1, 1# + * #sum_sup = 0, 0, 0, 0, 0, 0, 0, 0, 1, 1# + * #ncont_sup = 0, 0, 0, 0, 1, 1, 1, 1, 1, 1# + * #shl_imm = 0, 0, 15, 15, 31, 31, 63, 63, 0, 0# + * #shr_imm = 0, 0, 16, 16, 32, 32, 64, 64, 0, 0# + */ +#if @simd_sup@ + +/*************************** + * Memory + ***************************/ +/**begin repeat1 + * # intrin = load, loada, loads, loadl, store, storea, stores, storel, storeh# + */ +SIMD_INTRIN_DEF(@intrin@_@sfx@) +/**end repeat1**/ + +/**************************************** + * Non-contiguous/Partial Memory access + ****************************************/ +#if @ncont_sup@ +/**begin repeat1 + * #intrin = load_till, load_tillz, loadn, loadn_till, loadn_tillz, + * store_till, storen, storen_till# + */ +SIMD_INTRIN_DEF(@intrin@_@sfx@) +/**end repeat1**/ +#endif // ncont_sup + + +/*************************** + * Misc + ***************************/ +/**begin repeat1 + * #sfx_to = u8, s8, u16, s16, u32, s32, u64, s64, f32, f64# + * #simd_sup2 = 1, 1, 1, 1, 1, 1, 1, 1, 1, NPY_SIMD_F64# + */ +#if @simd_sup2@ +SIMD_INTRIN_DEF(reinterpret_@sfx_to@_@sfx@) +#endif // simd_sup2 +/**end repeat1**/ + +/**begin repeat1 + * # intrin = set, setf, setall, zero, select# + */ +SIMD_INTRIN_DEF(@intrin@_@sfx@) +/**end repeat1**/ + +/*************************** + * Reorder + ***************************/ +/**begin repeat1 + * # intrin = combinel, combineh, combine, zip# + */ +SIMD_INTRIN_DEF(@intrin@_@sfx@) +/**end repeat1**/ + +SIMD_INTRIN_DEF(cvt_@sfx@_@bsfx@) +SIMD_INTRIN_DEF(cvt_@bsfx@_@sfx@) + +/*************************** + * Operators + ***************************/ +#if @shl_imm@ > 0 +/**begin repeat1 + * # intrin = shl, shr, shli, shri# + */ +SIMD_INTRIN_DEF(@intrin@_@sfx@) +/**end repeat1**/ +#endif // shl_imm + +/**begin repeat1 + * #intrin = and, or, xor, not, cmpeq, cmpneq, cmpgt, cmpge, cmplt, cmple# + */ +SIMD_INTRIN_DEF(@intrin@_@sfx@) +/**end repeat1**/ + +/*************************** + * Conversion + ***************************/ +SIMD_INTRIN_DEF(cvt_@sfx@_@bsfx@) +SIMD_INTRIN_DEF(cvt_@bsfx@_@sfx@) + +/*************************** + * Arithmetic + ***************************/ +/**begin repeat1 + * #intrin = add, sub# + */ +SIMD_INTRIN_DEF(@intrin@_@sfx@) +/**end repeat1**/ + +#if @sat_sup@ +/**begin repeat1 + * #intrin = adds, subs# + */ +SIMD_INTRIN_DEF(@intrin@_@sfx@) +/**end repeat1**/ +#endif // sat_sup + +#if @mul_sup@ +SIMD_INTRIN_DEF(mul_@sfx@) +#endif // mul_sup + +#if @div_sup@ +SIMD_INTRIN_DEF(div_@sfx@) +#endif // div_sup + +#if @fused_sup@ +/**begin repeat1 + * #intrin = muladd, mulsub, nmuladd, nmulsub# + */ +SIMD_INTRIN_DEF(@intrin@_@sfx@) +/**end repeat1**/ +#endif // fused_sup + +#if @sum_sup@ +SIMD_INTRIN_DEF(sum_@sfx@) +#endif // sum_sup + +#endif // simd_sup +/**end repeat**/ + +/*************************** + * Variant + ***************************/ +SIMD_INTRIN_DEF(cleanup) +/***************************/ +{NULL, NULL, 0, NULL} +}; // PyMethodDef + +#endif // NPY_SIMD + +/************************************************************************* + * Defining a separate module for each target + *************************************************************************/ +NPY_VISIBILITY_HIDDEN PyObject * +NPY_CPU_DISPATCH_CURFX(simd_create_module)(void) +{ + static struct PyModuleDef defs = { + .m_base = PyModuleDef_HEAD_INIT, + .m_size = -1, + #ifdef NPY__CPU_TARGET_CURRENT + .m_name = "numpy.core._simd." NPY_TOSTRING(NPY__CPU_TARGET_CURRENT), + #else + .m_name = "numpy.core._simd.baseline", + #endif + #if NPY_SIMD + .m_methods = simd__intrinsics_methods + #else + .m_methods = NULL + #endif + }; + PyObject *m = PyModule_Create(&defs); + if (m == NULL) { + return NULL; + } + if (PyModule_AddIntConstant(m, "simd", NPY_SIMD)) { + goto err; + } + if (PyModule_AddIntConstant(m, "simd_f64", NPY_SIMD_F64)) { + goto err; + } + if (PyModule_AddIntConstant(m, "simd_width", NPY_SIMD_WIDTH)) { + goto err; + } +#if NPY_SIMD + if (PySIMDVectorType_Init(m)) { + goto err; + } + /**begin repeat + * #sfx = u8, s8, u16, s16, u32, s32, u64, s64, f32, f64# + */ + if (PyModule_AddIntConstant(m, "nlanes_@sfx@", npyv_nlanes_@sfx@)) { + goto err; + } + /**end repeat**/ +#endif // NPY_SIMD + return m; +err: + Py_DECREF(m); + return NULL; +} diff --git a/numpy/core/src/_simd/_simd.h b/numpy/core/src/_simd/_simd.h new file mode 100644 index 000000000000..d9905c80127c --- /dev/null +++ b/numpy/core/src/_simd/_simd.h @@ -0,0 +1,30 @@ +/** + * A module to expose the NumPy C SIMD vectorization interface "NPYV" for testing purposes. + * + * Please keep this module independent from other c-extension modules, + * since NPYV intrinsics may be involved in their functionality, + * which increases the degree of complexity in tracking and detecting errors. + * + * TODO: Add an independent sphinx doc. + * + * Please add any new NPYV intrinsics in '_simd.dispatch.c.src'. + */ +#ifndef _SIMD_SIMD_H_ +#define _SIMD_SIMD_H_ + +#include +#include "numpy/npy_common.h" + +#ifndef NPY_DISABLE_OPTIMIZATION +// autogenerated, required for CPU dispatch macros +#include "_simd.dispatch.h" +#endif +/** + * Create a new module for each required optimization which contains all NPYV intrinsics, + * + * If required optimization is not supported by NPYV, the module will still provides + * access to NPYV constants NPY_SIMD, NPY_SIMD_F64, and NPY_SIMD_WIDTH but without + * any intrinsics. + */ +NPY_CPU_DISPATCH_DECLARE(NPY_VISIBILITY_HIDDEN PyObject *simd_create_module, (void)) +#endif // _SIMD_SIMD_H_ diff --git a/numpy/core/src/_simd/_simd_arg.inc b/numpy/core/src/_simd/_simd_arg.inc new file mode 100644 index 000000000000..f5bcf5487c65 --- /dev/null +++ b/numpy/core/src/_simd/_simd_arg.inc @@ -0,0 +1,85 @@ +/** + * This file is included by `_simd.dispatch.c.src`. Its contents are affected by the simd configuration, and + * therefore must be built multiple times. Making it a standalone `.c` file with `NPY_VISIBILITY_HIDDEN` + * symbols would require judicious use of `NPY_CPU_DISPATCH_DECLARE` and `NPY_CPU_DISPATCH_CURFX`, which was + * deemed too harmful to readability. + */ +/************************************ + ** Protected Definitions + ************************************/ +static int +simd_arg_from_obj(PyObject *obj, simd_arg *arg) +{ + assert(arg->dtype != 0); + const simd_data_info *info = simd_data_getinfo(arg->dtype); + if (info->is_scalar) { + arg->data = simd_scalar_from_number(obj, arg->dtype); + } + else if (info->is_sequence) { + unsigned min_seq_size = simd_data_getinfo(info->to_vector)->nlanes; + arg->data.qu8 = simd_sequence_from_iterable(obj, arg->dtype, min_seq_size); + } + else if (info->is_vectorx) { + arg->data = simd_vectorx_from_tuple(obj, arg->dtype); + } + else if (info->is_vector) { + arg->data = PySIMDVector_AsData((PySIMDVectorObject*)obj, arg->dtype); + } else { + arg->data.u64 = 0; + PyErr_Format(PyExc_RuntimeError, + "unhandled arg from obj type id:%d, name:%s", arg->dtype, info->pyname + ); + return -1; + } + if (PyErr_Occurred()) { + return -1; + } + return 0; +} + +static PyObject * +simd_arg_to_obj(const simd_arg *arg) +{ + assert(arg->dtype != 0); + const simd_data_info *info = simd_data_getinfo(arg->dtype); + if (info->is_scalar) { + return simd_scalar_to_number(arg->data, arg->dtype); + } + if (info->is_sequence) { + return simd_sequence_to_list(arg->data.qu8, arg->dtype); + } + if (info->is_vectorx) { + return simd_vectorx_to_tuple(arg->data, arg->dtype); + } + if (info->is_vector) { + return (PyObject*)PySIMDVector_FromData(arg->data, arg->dtype); + } + PyErr_Format(PyExc_RuntimeError, + "unhandled arg to object type id:%d, name:%s", arg->dtype, info->pyname + ); + return NULL; +} + +static void +simd_arg_free(simd_arg *arg) +{ + const simd_data_info *info = simd_data_getinfo(arg->dtype); + if (info->is_sequence) { + simd_sequence_free(arg->data.qu8); + } +} + +static int +simd_arg_converter(PyObject *obj, simd_arg *arg) +{ + if (obj != NULL) { + if (simd_arg_from_obj(obj, arg) < 0) { + return 0; + } + arg->obj = obj; + return Py_CLEANUP_SUPPORTED; + } else { + simd_arg_free(arg); + } + return 1; +} diff --git a/numpy/core/src/_simd/_simd_convert.inc b/numpy/core/src/_simd/_simd_convert.inc new file mode 100644 index 000000000000..f5bfc3f50409 --- /dev/null +++ b/numpy/core/src/_simd/_simd_convert.inc @@ -0,0 +1,209 @@ +/** + * This file is included by `_simd.dispatch.c.src`. Its contents are affected by the simd configuration, and + * therefore must be built multiple times. Making it a standalone `.c` file with `NPY_VISIBILITY_HIDDEN` + * symbols would require judicious use of `NPY_CPU_DISPATCH_DECLARE` and `NPY_CPU_DISPATCH_CURFX`, which was + * deemed too harmful to readability. + */ +/************************************ + ** Protected Definitions + ************************************/ +static simd_data +simd_scalar_from_number(PyObject *obj, simd_data_type dtype) +{ + const simd_data_info *info = simd_data_getinfo(dtype); + assert(info->is_scalar && info->lane_size > 0); + simd_data data; + if (info->is_float) { + data.f64 = PyFloat_AsDouble(obj); + if (dtype == simd_data_f32){ + data.f32 = (float)data.f64; + } + } else { + data.u64 = PyLong_AsUnsignedLongLongMask(obj); + } + return data; +} + +static PyObject * +simd_scalar_to_number(simd_data data, simd_data_type dtype) +{ + const simd_data_info *info = simd_data_getinfo(dtype); + assert(info->is_scalar && info->lane_size > 0); + if (info->is_float) { + if (dtype == simd_data_f32) { + return PyFloat_FromDouble(data.f32); + } + return PyFloat_FromDouble(data.f64); + } + int leftb = (sizeof(npyv_lanetype_u64) - info->lane_size) * 8; + data.u64 <<= leftb; + if (info->is_signed) { + return PyLong_FromLongLong(data.s64 >> leftb); + } + return PyLong_FromUnsignedLongLong(data.u64 >> leftb); +} + +typedef struct { + Py_ssize_t len; + void *ptr; +} simd__alloc_data; + +static void * +simd_sequence_new(Py_ssize_t len, simd_data_type dtype) +{ + const simd_data_info *info = simd_data_getinfo(dtype); + assert(len > 0 && info->is_sequence && info->lane_size > 0); + size_t size = sizeof(simd__alloc_data) + len * info->lane_size + NPY_SIMD_WIDTH; + void *ptr = malloc(size); + if (ptr == NULL) { + return PyErr_NoMemory(); + } + // align the pointer + simd__alloc_data *a_ptr = (simd__alloc_data *)( + ((uintptr_t)ptr + sizeof(simd__alloc_data) + NPY_SIMD_WIDTH) & ~(uintptr_t)(NPY_SIMD_WIDTH-1) + ); + a_ptr[-1].len = len; + a_ptr[-1].ptr = ptr; + return a_ptr; +} + +static Py_ssize_t +simd_sequence_len(void const *ptr) +{ + return ((simd__alloc_data const*)ptr)[-1].len; +} + +static void +simd_sequence_free(void *ptr) +{ + free(((simd__alloc_data *)ptr)[-1].ptr); +} + +static void * +simd_sequence_from_iterable(PyObject *obj, simd_data_type dtype, Py_ssize_t min_size) +{ + const simd_data_info *info = simd_data_getinfo(dtype); + assert(info->is_sequence && info->lane_size > 0); + PyObject *seq_obj = PySequence_Fast(obj, "expected a sequence"); + if (seq_obj == NULL) { + return NULL; + } + Py_ssize_t seq_size = PySequence_Fast_GET_SIZE(seq_obj); + if (seq_size < min_size) { + PyErr_Format(PyExc_ValueError, + "minimum acceptable size of the required sequence is %d, given(%d)", + min_size, seq_size + ); + return NULL; + } + npyv_lanetype_u8 *dst = simd_sequence_new(seq_size, dtype); + if (dst == NULL) { + return NULL; + } + PyObject **seq_items = PySequence_Fast_ITEMS(seq_obj); + for (Py_ssize_t i = 0; i < seq_size; ++i) { + simd_data data = simd_scalar_from_number(seq_items[i], info->to_scalar); + npyv_lanetype_u8 *sdst = dst + i * info->lane_size; + memcpy(sdst, &data.u64, info->lane_size); + } + Py_DECREF(seq_obj); + + if (PyErr_Occurred()) { + simd_sequence_free(dst); + return NULL; + } + return dst; +} + +static int +simd_sequence_fill_iterable(PyObject *obj, const void *ptr, simd_data_type dtype) +{ + const simd_data_info *info = simd_data_getinfo(dtype); + if (!PySequence_Check(obj)) { + PyErr_Format(PyExc_TypeError, + "a sequence object is required to fill %s", info->pyname + ); + return -1; + } + const npyv_lanetype_u8 *src = ptr; + Py_ssize_t seq_len = simd_sequence_len(ptr); + for (Py_ssize_t i = 0; i < seq_len; ++i) { + const npyv_lanetype_u8 *ssrc = src + i * info->lane_size; + simd_data data; + memcpy(&data.u64, ssrc, info->lane_size); + PyObject *item = simd_scalar_to_number(data, info->to_scalar); + if (item == NULL) { + return -1; + } + if (PySequence_SetItem(obj, i, item) < 0) { + Py_DECREF(item); + return -1; + } + } + return 0; +} + +static PyObject * +simd_sequence_to_list(const void *ptr, simd_data_type dtype) +{ + PyObject *list = PyList_New(simd_sequence_len(ptr)); + if (list == NULL) { + return NULL; + } + if (simd_sequence_fill_iterable(list, ptr, dtype) < 0) { + Py_DECREF(list); + return NULL; + } + return list; +} + +static simd_data +simd_vectorx_from_tuple(PyObject *obj, simd_data_type dtype) +{ + const simd_data_info *info = simd_data_getinfo(dtype); + // NPYV currently only supports x2 and x3 + assert(info->is_vectorx > 1 && info->is_vectorx < 4); + + simd_data data = {.u64 = 0}; + if (!PyTuple_Check(obj) || PyTuple_GET_SIZE(obj) != info->is_vectorx) { + PyErr_Format(PyExc_TypeError, + "a tuple of %d vector type %s is required", + info->is_vectorx, simd_data_getinfo(info->to_vector)->pyname + ); + return data; + } + for (int i = 0; i < info->is_vectorx; ++i) { + PyObject *item = PyTuple_GET_ITEM(obj, i); + // get the max multi-vec and let the compiler do the rest + data.vu64x3.val[i] = PySIMDVector_AsData((PySIMDVectorObject*)item, info->to_vector).vu64; + if (PyErr_Occurred()) { + return data; + } + } + return data; +} + +static PyObject * +simd_vectorx_to_tuple(simd_data data, simd_data_type dtype) +{ + const simd_data_info *info = simd_data_getinfo(dtype); + // NPYV currently only supports x2 and x3 + assert(info->is_vectorx > 1 && info->is_vectorx < 4); + + PyObject *tuple = PyTuple_New(info->is_vectorx); + if (tuple == NULL) { + return NULL; + } + for (int i = 0; i < info->is_vectorx; ++i) { + // get the max multi-vector and let the compiler handle the rest + simd_data vdata = {.vu64 = data.vu64x3.val[i]}; + PyObject *item = (PyObject*)PySIMDVector_FromData(vdata, info->to_vector); + if (item == NULL) { + // TODO: improve log add item number + Py_DECREF(tuple); + return NULL; + } + PyTuple_SET_ITEM(tuple, i, item); + } + return tuple; +} diff --git a/numpy/core/src/_simd/_simd_data.inc.src b/numpy/core/src/_simd/_simd_data.inc.src new file mode 100644 index 000000000000..5c796487c923 --- /dev/null +++ b/numpy/core/src/_simd/_simd_data.inc.src @@ -0,0 +1,93 @@ +/** + * This file is included by `_simd.dispatch.c.src`. Its contents are affected by the simd configuration, and + * therefore must be built multiple times. Making it a standalone `.c` file with `NPY_VISIBILITY_HIDDEN` + * symbols would require judicious use of `NPY_CPU_DISPATCH_DECLARE` and `NPY_CPU_DISPATCH_CURFX`, which was + * deemed too harmful to readability. + */ +/************************************ + ** Private Definitions + ************************************/ +static simd_data_info simd__data_registry[simd_data_end] = +{ + [simd_data_none] = {.pyname="none"}, + /**begin repeat + * #sfx = u8, u16, u32, u64, s8, s16, s32, s64, f32, f64# + * #sig = 0*4, 1*4, 0*2# + * #fp = 0*4, 0*4, 1*2# + * #name = int*8, float, float# + */ + [simd_data_@sfx@] = { + .pyname="@name@", .is_unsigned=!@sig@&&!@fp@, .is_signed=@sig@, .is_float=@fp@, + .is_scalar=1, .to_scalar = simd_data_@sfx@, .to_vector = simd_data_v@sfx@, + .lane_size = sizeof(npyv_lanetype_@sfx@) + }, + /**end repeat**/ + // sequences + /**begin repeat + * #sfx = u8, u16, u32, u64, s8, s16, s32, s64, f32, f64# + * #sig = 0*4, 1*4, 0*2# + * #fp = 0*4, 0*4, 1*2# + * #name = int*8, float, float# + */ + [simd_data_q@sfx@] = { + .pyname="[@name@]", .is_unsigned=!@sig@&&!@fp@, .is_signed=@sig@, .is_float=@fp@, + .is_sequence=1, .to_scalar = simd_data_@sfx@, .to_vector = simd_data_v@sfx@, + .nlanes=npyv_nlanes_@sfx@, .lane_size = sizeof(npyv_lanetype_@sfx@) + }, + /**end repeat**/ + // vectors + /**begin repeat + * #sfx = u8, u16, u32, u64, s8, s16, s32, s64, f32, f64# + * #sig = 0*4, 1*4, 0*2# + * #fp = 0*4, 0*4, 1*2# + */ + [simd_data_v@sfx@] = { + .pyname="npyv_@sfx@", .is_unsigned=!@sig@&&!@fp@, .is_signed=@sig@, .is_float=@fp@, + .is_vector=1, .to_scalar = simd_data_@sfx@, .to_vector = simd_data_v@sfx@, + .nlanes=npyv_nlanes_@sfx@, .lane_size = sizeof(npyv_lanetype_@sfx@) + }, + /**end repeat**/ + // boolean vectors, treated as unsigned and converted internally + // to add compatibility among all SIMD extensions + /**begin repeat + * #sfx = u8, u16, u32, u64# + * #bsfx = b8, b16, b32, b64# + */ + [simd_data_v@bsfx@] = { + .pyname="npyv_@bsfx@", .is_bool=1, .is_vector=1, + .to_scalar = simd_data_@sfx@, .to_vector = simd_data_v@sfx@, + .nlanes=npyv_nlanes_@sfx@, .lane_size = sizeof(npyv_lanetype_@sfx@) + }, + /**end repeat**/ + // multi-vectors x2 + /**begin repeat + * #sfx = u8, u16, u32, u64, s8, s16, s32, s64, f32, f64# + * #sig = 0*4, 1*4, 0*2# + * #fp = 0*4, 0*4, 1*2# + */ + [simd_data_v@sfx@x2] = { + .pyname="npyv_@sfx@x2", .is_unsigned=!@sig@&&!@fp@, .is_signed=@sig@, .is_float=@fp@, + .is_vectorx=2, .to_scalar = simd_data_@sfx@, .to_vector = simd_data_v@sfx@, + .nlanes=2, .lane_size = sizeof(npyv_lanetype_@sfx@) + }, + /**end repeat**/ + // multi-vectors x3 + /**begin repeat + * #sfx = u8, u16, u32, u64, s8, s16, s32, s64, f32, f64# + * #sig = 0*4, 1*4, 0*2# + * #fp = 0*4, 0*4, 1*2# + */ + [simd_data_v@sfx@x3] = { + .pyname="npyv_@sfx@x3", .is_unsigned=!@sig@&&!@fp@, .is_signed=@sig@, .is_float=@fp@, + .is_vectorx=3, .to_scalar = simd_data_@sfx@, .to_vector = simd_data_v@sfx@, + .nlanes=3, .lane_size = sizeof(npyv_lanetype_@sfx@) + }, + /**end repeat**/ +}; + +/************************************ + ** Protected Definitions + ************************************/ +static const simd_data_info * +simd_data_getinfo(simd_data_type dtype) +{ return &simd__data_registry[dtype]; } diff --git a/numpy/core/src/_simd/_simd_easyintrin.inc b/numpy/core/src/_simd/_simd_easyintrin.inc new file mode 100644 index 000000000000..54e7ccf01f50 --- /dev/null +++ b/numpy/core/src/_simd/_simd_easyintrin.inc @@ -0,0 +1,214 @@ +/** + * This file is included by `_simd.dispatch.c.src`. Its contents are affected by the simd configuration, and + * therefore must be built multiple times. Making it a standalone `.c` file with `NPY_VISIBILITY_HIDDEN` + * symbols would require judicious use of `NPY_CPU_DISPATCH_DECLARE` and `NPY_CPU_DISPATCH_CURFX`, which was + * deemed too harmful to readability. + */ +#define SIMD_INTRIN_DEF(NAME) \ + { NPY_TOSTRING(NAME), simd__intrin_##NAME, METH_VARARGS, NULL } , // comma + +#define SIMD_IMPL_INTRIN_0(NAME, RET) \ + static PyObject *simd__intrin_##NAME \ + (PyObject* NPY_UNUSED(self), PyObject *args) \ + { \ + if (!PyArg_ParseTuple( \ + args, ":" NPY_TOSTRING(NAME)) \ + ) return NULL; \ + simd_arg a = { \ + .dtype = simd_data_##RET, \ + .data = {.RET = npyv_##NAME()}, \ + }; \ + return simd_arg_to_obj(&a); \ + } + +#define SIMD_IMPL_INTRIN_0N(NAME) \ + static PyObject *simd__intrin_##NAME \ + (PyObject* NPY_UNUSED(self), PyObject *args) \ + { \ + if (!PyArg_ParseTuple( \ + args, ":" NPY_TOSTRING(NAME)) \ + ) return NULL; \ + npyv_##NAME(); \ + Py_RETURN_NONE; \ + } + +#define SIMD_IMPL_INTRIN_1(NAME, RET, IN0) \ + static PyObject *simd__intrin_##NAME \ + (PyObject* NPY_UNUSED(self), PyObject *args) \ + { \ + simd_arg arg = {.dtype = simd_data_##IN0}; \ + if (!PyArg_ParseTuple( \ + args, "O&:"NPY_TOSTRING(NAME), \ + simd_arg_converter, &arg \ + )) return NULL; \ + simd_data data = {.RET = npyv_##NAME( \ + arg.data.IN0 \ + )}; \ + simd_arg_free(&arg); \ + simd_arg ret = { \ + .data = data, .dtype = simd_data_##RET \ + }; \ + return simd_arg_to_obj(&ret); \ + } + +#define SIMD_IMPL_INTRIN_2(NAME, RET, IN0, IN1) \ + static PyObject *simd__intrin_##NAME \ + (PyObject* NPY_UNUSED(self), PyObject *args) \ + { \ + simd_arg arg1 = {.dtype = simd_data_##IN0}; \ + simd_arg arg2 = {.dtype = simd_data_##IN1}; \ + if (!PyArg_ParseTuple( \ + args, "O&O&:"NPY_TOSTRING(NAME), \ + simd_arg_converter, &arg1, \ + simd_arg_converter, &arg2 \ + )) return NULL; \ + simd_data data = {.RET = npyv_##NAME( \ + arg1.data.IN0, arg2.data.IN1 \ + )}; \ + simd_arg_free(&arg1); \ + simd_arg_free(&arg2); \ + simd_arg ret = { \ + .data = data, .dtype = simd_data_##RET \ + }; \ + return simd_arg_to_obj(&ret); \ + } + +#define SIMD__REPEAT_2IMM(C, NAME, IN0) \ + C == arg2.data.u8 ? NPY_CAT(npyv_, NAME)(arg1.data.IN0, C) : + +#define SIMD_IMPL_INTRIN_2IMM(NAME, RET, IN0, CONST_RNG) \ + static PyObject *simd__intrin_##NAME \ + (PyObject* NPY_UNUSED(self), PyObject *args) \ + { \ + simd_arg arg1 = {.dtype = simd_data_##IN0}; \ + simd_arg arg2 = {.dtype = simd_data_u8}; \ + if (!PyArg_ParseTuple( \ + args, "O&O&:"NPY_TOSTRING(NAME), \ + simd_arg_converter, &arg1, \ + simd_arg_converter, &arg2 \ + )) return NULL; \ + simd_data data; \ + data.RET = NPY_CAT(SIMD__IMPL_COUNT_, CONST_RNG)( \ + SIMD__REPEAT_2IMM, NAME, IN0 \ + ) npyv_##NAME(arg1.data.IN0, 0); \ + simd_arg_free(&arg1); \ + simd_arg ret = { \ + .data = data, .dtype = simd_data_##RET \ + }; \ + return simd_arg_to_obj(&ret); \ + } + +#define SIMD_IMPL_INTRIN_3(NAME, RET, IN0, IN1, IN2) \ + static PyObject *simd__intrin_##NAME \ + (PyObject* NPY_UNUSED(self), PyObject *args) \ + { \ + simd_arg arg1 = {.dtype = simd_data_##IN0}; \ + simd_arg arg2 = {.dtype = simd_data_##IN1}; \ + simd_arg arg3 = {.dtype = simd_data_##IN2}; \ + if (!PyArg_ParseTuple( \ + args, "O&O&O&:"NPY_TOSTRING(NAME), \ + simd_arg_converter, &arg1, \ + simd_arg_converter, &arg2, \ + simd_arg_converter, &arg3 \ + )) return NULL; \ + simd_data data = {.RET = npyv_##NAME( \ + arg1.data.IN0, arg2.data.IN1, \ + arg3.data.IN2 \ + )}; \ + simd_arg_free(&arg1); \ + simd_arg_free(&arg2); \ + simd_arg_free(&arg3); \ + simd_arg ret = { \ + .data = data, .dtype = simd_data_##RET \ + }; \ + return simd_arg_to_obj(&ret); \ + } +/** + * Helper macros for repeating and expand a certain macro. + * Mainly used for converting a scalar to an immediate constant. + */ +#define SIMD__IMPL_COUNT_7(FN, ...) \ + NPY_EXPAND(FN(0, __VA_ARGS__)) \ + SIMD__IMPL_COUNT_7_(FN, __VA_ARGS__) + +#define SIMD__IMPL_COUNT_8(FN, ...) \ + SIMD__IMPL_COUNT_7_(FN, __VA_ARGS__) \ + NPY_EXPAND(FN(8, __VA_ARGS__)) + +#define SIMD__IMPL_COUNT_15(FN, ...) \ + NPY_EXPAND(FN(0, __VA_ARGS__)) \ + SIMD__IMPL_COUNT_15_(FN, __VA_ARGS__) + +#define SIMD__IMPL_COUNT_16(FN, ...) \ + SIMD__IMPL_COUNT_15_(FN, __VA_ARGS__) \ + NPY_EXPAND(FN(16, __VA_ARGS__)) + +#define SIMD__IMPL_COUNT_31(FN, ...) \ + NPY_EXPAND(FN(0, __VA_ARGS__)) \ + SIMD__IMPL_COUNT_31_(FN, __VA_ARGS__) + +#define SIMD__IMPL_COUNT_32(FN, ...) \ + SIMD__IMPL_COUNT_31_(FN, __VA_ARGS__) \ + NPY_EXPAND(FN(32, __VA_ARGS__)) + +#define SIMD__IMPL_COUNT_47(FN, ...) \ + NPY_EXPAND(FN(0, __VA_ARGS__)) \ + SIMD__IMPL_COUNT_47_(FN, __VA_ARGS__) + +#define SIMD__IMPL_COUNT_48(FN, ...) \ + SIMD__IMPL_COUNT_47_(FN, __VA_ARGS__) \ + NPY_EXPAND(FN(48, __VA_ARGS__)) + +#define SIMD__IMPL_COUNT_63(FN, ...) \ + NPY_EXPAND(FN(0, __VA_ARGS__)) \ + SIMD__IMPL_COUNT_63_(FN, __VA_ARGS__) + +#define SIMD__IMPL_COUNT_64(FN, ...) \ + SIMD__IMPL_COUNT_63_(FN, __VA_ARGS__) \ + NPY_EXPAND(FN(64, __VA_ARGS__)) + +#define SIMD__IMPL_COUNT_7_(FN, ...) \ + NPY_EXPAND(FN(1, __VA_ARGS__)) \ + NPY_EXPAND(FN(2, __VA_ARGS__)) NPY_EXPAND(FN(3, __VA_ARGS__)) \ + NPY_EXPAND(FN(4, __VA_ARGS__)) NPY_EXPAND(FN(5, __VA_ARGS__)) \ + NPY_EXPAND(FN(6, __VA_ARGS__)) NPY_EXPAND(FN(7, __VA_ARGS__)) + +#define SIMD__IMPL_COUNT_15_(FN, ...) \ + SIMD__IMPL_COUNT_7_(FN, __VA_ARGS__) \ + NPY_EXPAND(FN(8, __VA_ARGS__)) NPY_EXPAND(FN(9, __VA_ARGS__)) \ + NPY_EXPAND(FN(10, __VA_ARGS__)) NPY_EXPAND(FN(11, __VA_ARGS__)) \ + NPY_EXPAND(FN(12, __VA_ARGS__)) NPY_EXPAND(FN(13, __VA_ARGS__)) \ + NPY_EXPAND(FN(14, __VA_ARGS__)) NPY_EXPAND(FN(15, __VA_ARGS__)) + +#define SIMD__IMPL_COUNT_31_(FN, ...) \ + SIMD__IMPL_COUNT_15_(FN, __VA_ARGS__) \ + NPY_EXPAND(FN(16, __VA_ARGS__)) NPY_EXPAND(FN(17, __VA_ARGS__)) \ + NPY_EXPAND(FN(18, __VA_ARGS__)) NPY_EXPAND(FN(19, __VA_ARGS__)) \ + NPY_EXPAND(FN(20, __VA_ARGS__)) NPY_EXPAND(FN(21, __VA_ARGS__)) \ + NPY_EXPAND(FN(22, __VA_ARGS__)) NPY_EXPAND(FN(23, __VA_ARGS__)) \ + NPY_EXPAND(FN(24, __VA_ARGS__)) NPY_EXPAND(FN(25, __VA_ARGS__)) \ + NPY_EXPAND(FN(26, __VA_ARGS__)) NPY_EXPAND(FN(27, __VA_ARGS__)) \ + NPY_EXPAND(FN(28, __VA_ARGS__)) NPY_EXPAND(FN(29, __VA_ARGS__)) \ + NPY_EXPAND(FN(30, __VA_ARGS__)) NPY_EXPAND(FN(31, __VA_ARGS__)) + +#define SIMD__IMPL_COUNT_47_(FN, ...) \ + SIMD__IMPL_COUNT_31_(FN, __VA_ARGS__) \ + NPY_EXPAND(FN(32, __VA_ARGS__)) NPY_EXPAND(FN(33, __VA_ARGS__)) \ + NPY_EXPAND(FN(34, __VA_ARGS__)) NPY_EXPAND(FN(35, __VA_ARGS__)) \ + NPY_EXPAND(FN(36, __VA_ARGS__)) NPY_EXPAND(FN(37, __VA_ARGS__)) \ + NPY_EXPAND(FN(38, __VA_ARGS__)) NPY_EXPAND(FN(39, __VA_ARGS__)) \ + NPY_EXPAND(FN(40, __VA_ARGS__)) NPY_EXPAND(FN(41, __VA_ARGS__)) \ + NPY_EXPAND(FN(42, __VA_ARGS__)) NPY_EXPAND(FN(43, __VA_ARGS__)) \ + NPY_EXPAND(FN(44, __VA_ARGS__)) NPY_EXPAND(FN(45, __VA_ARGS__)) \ + NPY_EXPAND(FN(46, __VA_ARGS__)) NPY_EXPAND(FN(47, __VA_ARGS__)) + +#define SIMD__IMPL_COUNT_63_(FN, ...) \ + SIMD__IMPL_COUNT_47_(FN, __VA_ARGS__) \ + NPY_EXPAND(FN(48, __VA_ARGS__)) NPY_EXPAND(FN(49, __VA_ARGS__)) \ + NPY_EXPAND(FN(50, __VA_ARGS__)) NPY_EXPAND(FN(51, __VA_ARGS__)) \ + NPY_EXPAND(FN(52, __VA_ARGS__)) NPY_EXPAND(FN(53, __VA_ARGS__)) \ + NPY_EXPAND(FN(54, __VA_ARGS__)) NPY_EXPAND(FN(55, __VA_ARGS__)) \ + NPY_EXPAND(FN(56, __VA_ARGS__)) NPY_EXPAND(FN(57, __VA_ARGS__)) \ + NPY_EXPAND(FN(58, __VA_ARGS__)) NPY_EXPAND(FN(59, __VA_ARGS__)) \ + NPY_EXPAND(FN(60, __VA_ARGS__)) NPY_EXPAND(FN(61, __VA_ARGS__)) \ + NPY_EXPAND(FN(62, __VA_ARGS__)) NPY_EXPAND(FN(63, __VA_ARGS__)) diff --git a/numpy/core/src/_simd/_simd_inc.h.src b/numpy/core/src/_simd/_simd_inc.h.src new file mode 100644 index 000000000000..9858fc0dc624 --- /dev/null +++ b/numpy/core/src/_simd/_simd_inc.h.src @@ -0,0 +1,421 @@ +#ifndef _SIMD_SIMD_INC_H_ +#define _SIMD_SIMD_INC_H_ + +#include +#include "simd/simd.h" + +#if NPY_SIMD +/************************************ + ** Types + ************************************/ +/** + * Gather all data types supported by the module. +*/ +typedef union +{ + // scalars + /**begin repeat + * #sfx = u8, u16, u32, u64, s8, s16, s32, s64, f32, f64# + */ + npyv_lanetype_@sfx@ @sfx@; + /**end repeat**/ + // sequence + /**begin repeat + * #sfx = u8, u16, u32, u64, s8, s16, s32, s64, f32, f64# + */ + npyv_lanetype_@sfx@ *q@sfx@; + /**end repeat**/ + // vectors + /**begin repeat + * #sfx = u8, u16, u32, u64, s8, s16, s32, s64, f32, b8, b16, b32, b64# + */ + npyv_@sfx@ v@sfx@; + /**end repeat**/ + // multi-vectors x2 + /**begin repeat + * #sfx = u8, u16, u32, u64, s8, s16, s32, s64, f32# + */ + npyv_@sfx@x2 v@sfx@x2; + /**end repeat**/ + // multi-vectors x3 + /**begin repeat + * #sfx = u8, u16, u32, u64, s8, s16, s32, s64, f32# + */ + npyv_@sfx@x3 v@sfx@x3; + /**end repeat**/ +#if NPY_SIMD_F64 + npyv_f64 vf64; + npyv_f64x2 vf64x2; + npyv_f64x3 vf64x3; +#endif +} simd_data; + +/** + * Data types IDs and suffixes. Must be same data types as the ones + * in union 'simd_data' to fit the macros in '_simd_inc_easyintrin.h'. +*/ +typedef enum +{ + simd_data_none = 0, + // scalars + /**begin repeat + * #sfx = u8, u16, u32, u64, s8, s16, s32, s64, f32, f64# + */ + simd_data_@sfx@, + /**end repeat**/ + // sequences + /**begin repeat + * #sfx = u8, u16, u32, u64, s8, s16, s32, s64, f32, f64# + */ + simd_data_q@sfx@, + /**end repeat**/ + // vectors + /**begin repeat + * #sfx = u8, u16, u32, u64, s8, s16, s32, s64, f32, f64, b8, b16, b32, b64# + */ + simd_data_v@sfx@, + /**end repeat**/ + // multi-vectors x2 + /**begin repeat + * #sfx = u8, u16, u32, u64, s8, s16, s32, s64, f32, f64# + */ + simd_data_v@sfx@x2, + /**end repeat**/ + // multi-vectors x3 + /**begin repeat + * #sfx = u8, u16, u32, u64, s8, s16, s32, s64, f32, f64# + */ + simd_data_v@sfx@x3, + /**end repeat**/ + simd_data_end, +} simd_data_type; +/************************************ + ** Declarations (inc_data) + ************************************/ +/** + * simd_data_type information + */ +typedef struct +{ + // type name compatible with python style + const char *pyname; + // returns '1' if the type represent a unsigned integer + int is_unsigned:1; + // returns '1' if the type represent a signed integer + int is_signed:1; + // returns '1' if the type represent a single or double precision + int is_float:1; + // returns '1' if the type represent a boolean + int is_bool:1; + // returns '1' if the type represent a sequence + int is_sequence:1; + // returns '1' if the type represent a scalar + int is_scalar:1; + // returns '1' if the type represent a vector + int is_vector:1; + // returns the len of multi-vector if the type reprsent x2 or x3 vector + // otherwise returns 0, e.g. returns 2 if data type is simd_data_vu8x2 + int is_vectorx; + // returns the equivalent scalar data type e.g. simd_data_vu8 -> simd_data_u8 + simd_data_type to_scalar; + // returns the equivalent scalar data type e.g. simd_data_s8 -> simd_data_vs8 + // NOTE: returns the will equivalent "unsigned" vector type in case of "boolean" vector + // e.g. simd_data_vb8 -> simd_data_vu8 + simd_data_type to_vector; + // number of vector lanes + int nlanes; + // sizeof lane type + int lane_size; +} simd_data_info; + +/** + * Returns data info of certain dtype. + * + * Example: + ** const simd_data_info *info = simd_data_getinfo(simd_data_vu8); + ** if (info->is_vector && info->is_unsigned) { + ** ... + ** } + */ +static const simd_data_info * +simd_data_getinfo(simd_data_type dtype); + +/************************************ + ** Declarations (inc_vector) + ************************************/ +typedef struct +{ + PyObject_HEAD + // vector type id + simd_data_type dtype; + // vector data, aligned for safe casting + npyv_lanetype_u8 NPY_DECL_ALIGNED(NPY_SIMD_WIDTH) data[NPY_SIMD_WIDTH]; +} PySIMDVectorObject; +/** + * Create a Python obj(PySIMDVectorObject) from a NPYV vector based on the contents + * of `data`(simd_data) and according to the vector data type `dtype` + * on range(simd_data_[vu8:vf64]). + * Return NULL and a Python exception on failure, otherwise new reference. + * + * Example: + ** simd_data data = {.vu8 = npyv_setall_u8(0xff)}; + ** PySIMDVectorObject *obj = PySIMDVector_FromData(data, simd_data_vu8); + ** if (obj != NULL) { + ** printf("I have a valid vector obj and first element is \n", obj->data[0]); + ** Py_DECREF(obj); + ** } + */ +static PySIMDVectorObject * +PySIMDVector_FromData(simd_data data, simd_data_type dtype); +/** + * Return a NPYV vector(simd_data) representation of `obj`(PySIMDVectorObject) and + * according to the vector data type `dtype` on range (simd_data_[vu8:vf64]). + * Raise a Python exception on failure. + * + * Example: + ** simd_data data = PySIMDVector_AsData(vec_obj, simd_data_vf32); + ** if (!PyErr_Occurred()) { + ** npyv_f32 add_1 = npyv_add_f32(data.vf32, npyv_setall_f32(1)); + ** ... + ** } + */ +static simd_data +PySIMDVector_AsData(PySIMDVectorObject *obj, simd_data_type dtype); +/** + * initialize and register PySIMDVectorType to certain PyModule, + * PySIMDVectorType can be reached through attribute 'vector_type'. + * return -1 on error, 0 on success. + */ +static int +PySIMDVectorType_Init(PyObject *module); + +/************************************ + ** Declarations (inc_convert) + ************************************/ +/** + * Return a C scalar(simd_data) representation of `obj` and + * according to the scalar data type `dtype` on range (simd_data_[u8:f64]). + * Raise a Python exception on failure. + * + * Example: + ** simd_data data = simd_scalar_from_number(obj, simd_data_f32); + ** if (!PyErr_Occurred()) { + ** printf("I have a valid float %d\n", data.f32); + ** } + */ +static simd_data +simd_scalar_from_number(PyObject *obj, simd_data_type dtype); +/** + * Create a Python scalar from a C scalar based on the contents + * of `data`(simd_data) and according to the scalar data type `dtype` + * on range(simd_data_[u8:f64]). + * Return NULL and a Python exception on failure, otherwise new reference. + * + * Example: + ** simd_data data = {.u32 = 0x7fffffff}; + ** PyObject *obj = simd_scalar_to_number(data, simd_data_s32); + ** if (obj != NULL) { + ** printf("I have a valid Python integer %d\n", PyLong_AsLong(obj)); + ** Py_DECREF(obj); + ** } + */ +static PyObject * +simd_scalar_to_number(simd_data data, simd_data_type dtype); +/** + * Allocate a C array in memory according to number of elements `len` + * and sequence data type `dtype` on range(simd_data_[qu8:qf64]). + * + * Return aligned pointer based on `NPY_SIMD_WIDTH` or NULL + * with a Python exception on failure. + * + * Example: + ** npyv_lanetype_f64 *aligned_ptr = simd_sequence_new(npyv_nlanes_f64, simd_data_f64); + ** if (aligned_ptr != NULL) { + ** // aligned store + ** npyv_storea_f64(aligned_ptr, npyv_setall_f64(1.0)); + ** printf("The first element of my array %f\n", aligned_ptr[0]); + ** simd_sequence_free(aligned_ptr); + ** } + */ +static void * +simd_sequence_new(Py_ssize_t len, simd_data_type dtype); +/** + * Return the number of elements of the allocated C array `ptr` + * by `simd_sequence_new()` or `simd_sequence_from_iterable()`. + */ +static Py_ssize_t +simd_sequence_len(const void *ptr); +/** + * Free the allocated C array by `simd_sequence_new()` or + * `simd_sequence_from_iterable()`. + */ +static void +simd_sequence_free(void *ptr); +/** + * Return a C array representation of a PyObject sequence `obj` and + * according to the sequence data type `dtype` on range (simd_data_[qu8:qf64]). + * + * Note: parameter `min_size` takes the number of minimum acceptable elements. + * + * Return aligned pointer based on `NPY_SIMD_WIDTH` or NULL + * with a Python exception on failure. + * + * Example: + ** npyv_lanetype_u32 *ptr = simd_sequence_from_iterable(seq_obj, simd_data_qu32, npyv_nlanes_u32); + ** if (ptr != NULL) { + ** npyv_u32 a = npyv_load_u32(ptr); + ** ... + ** simd_sequence_free(ptr); + ** } + ** + */ +static void * +simd_sequence_from_iterable(PyObject *obj, simd_data_type dtype, Py_ssize_t min_size); +/** + * Fill a Python sequence object `obj` with a C array `ptr` allocated by + * `simd_sequence_new()` or `simd_sequence_from_iterable()` according to + * to the sequence data type `dtype` on range (simd_data_[qu8:qf64]). + * + * Return 0 on success or -1 with a Python exception on failure. + */ +static int +simd_sequence_fill_iterable(PyObject *obj, const void *ptr, simd_data_type dtype); +/** + * Create a Python list from a C array `ptr` allocated by + * `simd_sequence_new()` or `simd_sequence_from_iterable()` according to + * to the sequence data type `dtype` on range (simd_data_[qu8:qf64]). + * + * Return NULL and a Python exception on failure, otherwise new reference. + */ +static PyObject * +simd_sequence_to_list(const void *ptr, simd_data_type dtype); +/** + * Return a SIMD multi-vector(simd_data) representation of Python tuple of + * (simd_vector*,) `obj` according to the scalar data type `dtype` + * on range (simd_data_[vu8x2:vf64x2])-(simd_data_[vu8x3:vf64x3]). + * + * Raise a Python exception on failure. + * + * Example: + ** simd_data data = simd_vectorx_from_tuple(tuple_obj, simd_data_vf32x2); + ** if (!PyErr_Occurred()) { + ** npyv_f32 sum = npyv_add_f32(data.vf32x2.val[0], data.vf32x2.val[1]); + ** ... + ** } + ** + */ +static simd_data +simd_vectorx_from_tuple(PyObject *obj, simd_data_type dtype); +/** + * Create a Python tuple of 'simd_vector' from a SIMD multi-vector + * based on the contents of `data`(simd_data) and according to + * the multi-vector data type `dtype` on range + * (simd_data_[vu8x2:vf64x2])-(simd_data_[vu8x3:vf64x3]). + * + * Return NULL and a Python exception on failure, otherwise new reference. + */ +static PyObject * +simd_vectorx_to_tuple(simd_data data, simd_data_type dtype); + +/************************************ + ** Declarations (inc_arg) + ************************************/ +typedef struct +{ + simd_data_type dtype; + simd_data data; + // set by simd_arg_converter() + PyObject *obj; +} simd_arg; +/** + * The following functions gather all conversions between all data types + * and they can used instead of all above functions. + */ +/** + * Convert a Python object `obj` into simd_data `arg->data` according to the + * required data type `arg->dtype`. + * + * Return -1 and raise Python exception on failure, otherwise return 0. + * + * Notes: + * - requires `simd_arg_free()` or `simd_sequence_free()` + * to free allocated C array, in case of sequence data types. + * - the number of minimum acceptable elements for sequence data + * types is the number of lanes of the equivalent vector data type. + * + * Example #1: + ** simd_arg arg = {.dtype = simd_data_qu8}; + ** if (simd_arg_from_obj(seq_obj, &arg) < 0) { + ** // fails to convert a python sequence object to C array of uint8 + ** return; + ** } + ** npyv_u8 v_u8 = npyv_load_u8(arg->data.qu8); + ** ... + ** simd_arg_free(&arg); + * + * Example #2: + ** simd_arg arg = {.dtype = simd_data_vf32}; + ** if (simd_arg_from_obj(vector_obj, &arg) < 0) { + ** // fails to convert a python simd_vector to NPYV vector + ** return; + ** } + ** npyv_f32 add_one = npyv_add_f32(arg->data.vu8, npyv_setall_f32(1)); + ** ... + */ +static int +simd_arg_from_obj(PyObject *obj, simd_arg *arg); +/** + * Convert a simd_data `arg->data` to into a Python object according to the + * required data type `arg->dtype`. + * + * Return NULL and raise Python exception on failure, otherwise return + * new reference. + * + * Example: + ** simd_arg arg = {.dtype = simd_data_u32, .data = {.u32 = 0xffffffff}}; + ** PyObject *obj = simd_arg_to_obj(&arg); + ** if (obj == NULL) { + ** // fails convert C uint32 to Python integer. + ** return; + ** } + ** + */ +static PyObject * +simd_arg_to_obj(const simd_arg *arg); +/** + * Converter function used similar to simd_arg_from_obj() but + * used with PyArg_Parse*(). + * + * Notes: + * - requires `simd_arg_free()` or `simd_sequence_free()` + * to free allocated C array, in case of sequence data types. + * - the number of minimum acceptable elements for sequence data + * types is the number of lanes of the equivalent vector data type. + * - use 'arg->obj' to retrieve the parameter obj. + * + * Example: + ** simd_arg seq_f32 = {.dtype = simd_data_qf32}; + ** simd_arg vec_f32 = {.dtype = simd_data_vf32}; + ** if (!PyArg_ParseTuple( + ** args, "O&O&:add_sum_f32", + ** simd_arg_converter, &seq_f32, + ** simd_arg_converter, &vec_f32 + ** )) { + ** // fail + ** return; + ** } + ** npyv_f32 load_a = npyv_load_f32(seq_f32.data.qf32); + ** npyv_f32 sum = npyv_add_f32(load_a, vec_f32.data.vf32); + ** ... + ** simd_arg_free(&seq_f32); + */ +static int +simd_arg_converter(PyObject *obj, simd_arg *arg); +/** + * Free the allocated C array, if the arg hold sequence data type. + */ +static void +simd_arg_free(simd_arg *arg); + +#endif // NPY_SIMD +#endif // _SIMD_SIMD_INC_H_ diff --git a/numpy/core/src/_simd/_simd_vector.inc b/numpy/core/src/_simd/_simd_vector.inc new file mode 100644 index 000000000000..2a1378f22100 --- /dev/null +++ b/numpy/core/src/_simd/_simd_vector.inc @@ -0,0 +1,178 @@ +/** + * This file is included by `_simd.dispatch.c.src`. Its contents are affected by the simd configuration, and + * therefore must be built multiple times. Making it a standalone `.c` file with `NPY_VISIBILITY_HIDDEN` + * symbols would require judicious use of `NPY_CPU_DISPATCH_DECLARE` and `NPY_CPU_DISPATCH_CURFX`, which was + * deemed too harmful to readability. + */ +/************************************ + ** Private Definitions + ************************************/ +static Py_ssize_t +simd__vector_length(PySIMDVectorObject *self) +{ + return simd_data_getinfo(self->dtype)->nlanes; +} +static PyObject * +simd__vector_item(PySIMDVectorObject *self, Py_ssize_t i) +{ + const simd_data_info *info = simd_data_getinfo(self->dtype); + int nlanes = info->nlanes; + if (i >= nlanes) { + PyErr_SetString(PyExc_IndexError, "vector index out of range"); + return NULL; + } + npyv_lanetype_u8 *src = self->data + i * info->lane_size; + simd_data data; + memcpy(&data.u64, src, info->lane_size); + return simd_scalar_to_number(data, info->to_scalar); +} + +static PySequenceMethods simd__vector_as_sequence = { + .sq_length = (lenfunc) simd__vector_length, + .sq_item = (ssizeargfunc) simd__vector_item +}; + +static PyObject * +simd__vector_name(PySIMDVectorObject *self) +{ + return PyUnicode_FromString(simd_data_getinfo(self->dtype)->pyname); +} +static PyGetSetDef simd__vector_getset[] = { + { "__name__", (getter)simd__vector_name, NULL, NULL, NULL }, + { NULL, NULL, NULL, NULL, NULL } +}; + +static PyObject * +simd__vector_repr(PySIMDVectorObject *self) +{ + PyObject *obj = PySequence_List((PyObject*)self); + if (obj != NULL) { + const char *type_name = simd_data_getinfo(self->dtype)->pyname; + PyObject *repr = PyUnicode_FromFormat("<%s of %R>", type_name, obj); + Py_DECREF(obj); + return repr; + } + return obj; +} +static PyObject * +simd__vector_compare(PyObject *self, PyObject *other, int cmp_op) +{ + PyObject *obj; + if (PyTuple_Check(other)) { + obj = PySequence_Tuple(self); + } else if (PyList_Check(other)) { + obj = PySequence_List(self); + } else { + obj = PySequence_Fast(self, "invalid argument, expected a vector"); + } + if (obj != NULL) { + PyObject *rich = PyObject_RichCompare(obj, other, cmp_op); + Py_DECREF(obj); + return rich; + } + return obj; +} +static PyTypeObject PySIMDVectorType = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = NPY_TOSTRING(NPY_CPU_DISPATCH_CURFX(VECTOR)), + .tp_basicsize = sizeof(PySIMDVectorObject), + .tp_repr = (reprfunc)simd__vector_repr, + .tp_as_sequence = &simd__vector_as_sequence, + .tp_flags = Py_TPFLAGS_DEFAULT, + .tp_richcompare = simd__vector_compare, + .tp_getset = simd__vector_getset +}; + +/************************************ + ** Protected Definitions + ************************************/ +static PySIMDVectorObject * +PySIMDVector_FromData(simd_data data, simd_data_type dtype) +{ + const simd_data_info *info = simd_data_getinfo(dtype); + assert(info->is_vector && info->nlanes > 0); + + PySIMDVectorObject *vec = PyObject_New(PySIMDVectorObject, &PySIMDVectorType); + if (vec == NULL) { + return (PySIMDVectorObject*)PyErr_NoMemory(); + } + vec->dtype = dtype; + if (info->is_bool) { + // boolean vectors are internally treated as unsigned + // vectors to add compatibility among all SIMD extensions + switch(dtype) { + case simd_data_vb8: + data.vu8 = npyv_cvt_u8_b8(data.vb8); + break; + case simd_data_vb16: + data.vu16 = npyv_cvt_u16_b16(data.vb16); + break; + case simd_data_vb32: + data.vu32 = npyv_cvt_u32_b32(data.vb32); + break; + default: + data.vu64 = npyv_cvt_u64_b64(data.vb64); + } + } + npyv_store_u8(vec->data, data.vu8); + return vec; +} + +static simd_data +PySIMDVector_AsData(PySIMDVectorObject *vec, simd_data_type dtype) +{ + const simd_data_info *info = simd_data_getinfo(dtype); + assert(info->is_vector && info->nlanes > 0); + + simd_data data = {.u64 = 0}; + if (!PyObject_IsInstance( + (PyObject *)vec, (PyObject *)&PySIMDVectorType + )) { + PyErr_Format(PyExc_TypeError, + "a vector type %s is required", info->pyname + ); + return data; + } + if (vec->dtype != dtype) { + PyErr_Format(PyExc_TypeError, + "a vector type %s is required, got(%s)", + info->pyname, simd_data_getinfo(vec->dtype)->pyname + ); + return data; + } + + data.vu8 = npyv_load_u8(vec->data); + if (info->is_bool) { + // boolean vectors are internally treated as unsigned + // vectors to add compatibility among all SIMD extensions + switch(dtype) { + case simd_data_vb8: + data.vb8 = npyv_cvt_b8_u8(data.vu8); + break; + case simd_data_vb16: + data.vb16 = npyv_cvt_b16_u16(data.vu16); + break; + case simd_data_vb32: + data.vb32 = npyv_cvt_b32_u32(data.vu32); + break; + default: + data.vb64 = npyv_cvt_b64_u64(data.vu64); + } + } + return data; +} + +static int +PySIMDVectorType_Init(PyObject *module) +{ + Py_INCREF(&PySIMDVectorType); + if (PyType_Ready(&PySIMDVectorType)) { + return -1; + } + if (PyModule_AddObject( + module, "vector_type",(PyObject *)&PySIMDVectorType + )) { + return -1; + } + return 0; +} diff --git a/numpy/core/src/common/array_assign.c b/numpy/core/src/common/array_assign.c index 67abcae24268..c55f6bdb4624 100644 --- a/numpy/core/src/common/array_assign.c +++ b/numpy/core/src/common/array_assign.c @@ -64,19 +64,22 @@ broadcast_strides(int ndim, npy_intp const *shape, return 0; broadcast_error: { - PyObject *errmsg; - - errmsg = PyUnicode_FromFormat("could not broadcast %s from shape ", - strides_name); - PyUString_ConcatAndDel(&errmsg, - build_shape_string(strides_ndim, strides_shape)); - PyUString_ConcatAndDel(&errmsg, - PyUnicode_FromString(" into shape ")); - PyUString_ConcatAndDel(&errmsg, - build_shape_string(ndim, shape)); - PyErr_SetObject(PyExc_ValueError, errmsg); - Py_DECREF(errmsg); + PyObject *shape1 = convert_shape_to_string(strides_ndim, + strides_shape, ""); + if (shape1 == NULL) { + return -1; + } + PyObject *shape2 = convert_shape_to_string(ndim, shape, ""); + if (shape2 == NULL) { + Py_DECREF(shape1); + return -1; + } + PyErr_Format(PyExc_ValueError, + "could not broadcast %s from shape %S into shape %S", + strides_name, shape1, shape2); + Py_DECREF(shape1); + Py_DECREF(shape2); return -1; } } diff --git a/numpy/core/src/common/npy_cpu_dispatch.h b/numpy/core/src/common/npy_cpu_dispatch.h index 274520852569..a0f82fa3da05 100644 --- a/numpy/core/src/common/npy_cpu_dispatch.h +++ b/numpy/core/src/common/npy_cpu_dispatch.h @@ -17,7 +17,7 @@ * NumPy module's attributes `__cpu_baseline__` and `__cpu_dispaٍtch__`. */ /** - * Note: Always gaurd the genreated headers within 'NPY_DISABLE_OPTIMIZATION', + * Note: Always guard the generated headers within 'NPY_DISABLE_OPTIMIZATION', * due the nature of command argument '--disable-optimization', * which is explicitly disabling the module ccompiler_opt. */ @@ -29,7 +29,7 @@ * It's better anyway to take them off and use built-in types(__vector, __pixel, __bool) instead, * since c99 supports bool variables which may lead to ambiguous errors. */ - // backup 'bool' before including '_cpu_dispatch.h', since it may not defiend as a compiler token. + // backup 'bool' before including '_cpu_dispatch.h', since it may not defined as a compiler token. #define NPY__DISPATCH_DEFBOOL typedef bool npy__dispatch_bkbool; #endif @@ -134,10 +134,10 @@ * NPY_CPU_DISPATCH_DECLARE(void dispatch_me, (const int*, int*)) * NPY_CPU_DISPATCH_DECLARE(extern cb_type callback_tab, [TAB_SIZE]) * - * By assuming the provided config header drived from a dispatch-able source, + * By assuming the provided config header derived from a dispatch-able source, * that configured with "@targets baseline sse41 vsx3 asimdhp", * they supported by the compiler and enabled via '--cpu-dspatch', - * then the prototype declrations at the above example will equlivent to the follows: + * then the prototype declrations at the above example will equivalent to the follows: * * - x86: * void dispatch_me(const int*, int*); // baseline @@ -179,7 +179,7 @@ /** * Macro NPY_CPU_DISPATCH_DECLARE_XB(LEFT, ...) * - * Same as `NPY_CPU_DISPATCH_DECLARE` but exclude the baseline declration even + * Same as `NPY_CPU_DISPATCH_DECLARE` but exclude the baseline declaration even * if it was provided within the configration statments. */ #define NPY_CPU_DISPATCH_DECLARE_XB(...) \ @@ -206,7 +206,7 @@ * In order to call or to assign the pointer of it from outside the dispatch-able source, * you have to use this Macro as follows: * - * // bring the genreated config header of the dispatch-abel source + * // bring the generated config header of the dispatch-able source * #ifndef NPY_DISABLE_OPTIMIZATION * #include "dispatchable_source_name.dispatch.h" * #endif diff --git a/numpy/core/src/common/simd/avx2/arithmetic.h b/numpy/core/src/common/simd/avx2/arithmetic.h index 4af9e4d1748a..3a6dc953534e 100644 --- a/numpy/core/src/common/simd/avx2/arithmetic.h +++ b/numpy/core/src/common/simd/avx2/arithmetic.h @@ -116,4 +116,27 @@ return npyv_sub_f64(npyv_mul_f64(neg_a, b), c); } #endif // !NPY_HAVE_FMA3 + +// Horizontal add: Calculates the sum of all vector elements. +NPY_FINLINE float npyv_sum_f32(__m256 a) +{ + __m256 sum_halves = _mm256_hadd_ps(a, a); + sum_halves = _mm256_hadd_ps(sum_halves, sum_halves); + __m128 lo = _mm256_castps256_ps128(sum_halves); + __m128 hi = _mm256_extractf128_ps(sum_halves, 1); + __m128 sum = _mm_add_ps(lo, hi); + return _mm_cvtss_f32(sum); +} + +NPY_FINLINE double npyv_sum_f64(__m256d a) +{ + __m256d sum_halves = _mm256_hadd_pd(a, a); + __m128d lo = _mm256_castpd256_pd128(sum_halves); + __m128d hi = _mm256_extractf128_pd(sum_halves, 1); + __m128d sum = _mm_add_pd(lo, hi); + return _mm_cvtsd_f64(sum); +} + #endif // _NPY_SIMD_AVX2_ARITHMETIC_H + + diff --git a/numpy/core/src/common/simd/avx2/avx2.h b/numpy/core/src/common/simd/avx2/avx2.h index c99d628ee408..0641f2314ec6 100644 --- a/numpy/core/src/common/simd/avx2/avx2.h +++ b/numpy/core/src/common/simd/avx2/avx2.h @@ -5,6 +5,8 @@ #define NPY_SIMD 256 #define NPY_SIMD_WIDTH 32 #define NPY_SIMD_F64 1 +// Enough limit to allow us to use _mm256_i32gather_* +#define NPY_SIMD_MAXLOAD_STRIDE32 (0x7fffffff / 8) typedef __m256i npyv_u8; typedef __m256i npyv_s8; diff --git a/numpy/core/src/common/simd/avx2/memory.h b/numpy/core/src/common/simd/avx2/memory.h index 5ea7414fdf5c..e27bf15fec2e 100644 --- a/numpy/core/src/common/simd/avx2/memory.h +++ b/numpy/core/src/common/simd/avx2/memory.h @@ -2,6 +2,8 @@ #error "Not a standalone header" #endif +#include "misc.h" + #ifndef _NPY_SIMD_AVX2_MEMORY_H #define _NPY_SIMD_AVX2_MEMORY_H @@ -66,5 +68,289 @@ NPYV_IMPL_AVX2_MEM_INT(npy_int64, s64) // store higher part #define npyv_storeh_f32(PTR, VEC) _mm_storeu_ps(PTR, _mm256_extractf128_ps(VEC, 1)) #define npyv_storeh_f64(PTR, VEC) _mm_storeu_pd(PTR, _mm256_extractf128_pd(VEC, 1)) +/*************************** + * Non-contiguous Load + ***************************/ +//// 32 +NPY_FINLINE npyv_u32 npyv_loadn_u32(const npy_uint32 *ptr, npy_intp stride) +{ + assert(llabs(stride) <= NPY_SIMD_MAXLOAD_STRIDE32); + const __m256i steps = _mm256_setr_epi32(0, 1, 2, 3, 4, 5, 6, 7); + const __m256i idx = _mm256_mullo_epi32(_mm256_set1_epi32((int)stride), steps); + return _mm256_i32gather_epi32((const int*)ptr, idx, 4); +} +NPY_FINLINE npyv_s32 npyv_loadn_s32(const npy_int32 *ptr, npy_intp stride) +{ return npyv_loadn_u32((const npy_uint32*)ptr, stride); } +NPY_FINLINE npyv_f32 npyv_loadn_f32(const float *ptr, npy_intp stride) +{ return _mm256_castsi256_ps(npyv_loadn_u32((const npy_uint32*)ptr, stride)); } +//// 64 +#if 0 // slower +NPY_FINLINE npyv_u64 npyv_loadn_u64(const npy_uint64 *ptr, npy_intp stride) +{ + const __m256i idx = _mm256_setr_epi64x(0, 1*stride, 2*stride, 3*stride); + return _mm256_i64gather_epi64((const void*)ptr, idx, 8); +} +NPY_FINLINE npyv_s64 npyv_loadn_s64(const npy_int64 *ptr, npy_intp stride) +{ return npyv_loadn_u64((const npy_uint64*)ptr, stride); } +NPY_FINLINE npyv_f64 npyv_loadn_f64(const double *ptr, npy_intp stride) +{ return _mm256_castsi256_pd(npyv_loadn_u64((const npy_uint64*)ptr, stride)); } +#endif +NPY_FINLINE npyv_f64 npyv_loadn_f64(const double *ptr, npy_intp stride) +{ + __m128d a0 = _mm_castsi128_pd(_mm_loadl_epi64((const __m128i*)ptr)); + __m128d a2 = _mm_castsi128_pd(_mm_loadl_epi64((const __m128i*)(ptr + stride*2))); + __m128d a01 = _mm_loadh_pd(a0, ptr + stride); + __m128d a23 = _mm_loadh_pd(a2, ptr + stride*3); + return _mm256_insertf128_pd(_mm256_castpd128_pd256(a01), a23, 1); +} +NPY_FINLINE npyv_u64 npyv_loadn_u64(const npy_uint64 *ptr, npy_intp stride) +{ return _mm256_castpd_si256(npyv_loadn_f64((const double*)ptr, stride)); } +NPY_FINLINE npyv_s64 npyv_loadn_s64(const npy_int64 *ptr, npy_intp stride) +{ return _mm256_castpd_si256(npyv_loadn_f64((const double*)ptr, stride)); } +/*************************** + * Non-contiguous Store + ***************************/ +//// 32 +NPY_FINLINE void npyv_storen_s32(npy_int32 *ptr, npy_intp stride, npyv_s32 a) +{ + __m128i a0 = _mm256_castsi256_si128(a); + __m128i a1 = _mm256_extracti128_si256(a, 1); + ptr[stride * 0] = _mm_cvtsi128_si32(a0); + ptr[stride * 1] = _mm_extract_epi32(a0, 1); + ptr[stride * 2] = _mm_extract_epi32(a0, 2); + ptr[stride * 3] = _mm_extract_epi32(a0, 3); + ptr[stride * 4] = _mm_cvtsi128_si32(a1); + ptr[stride * 5] = _mm_extract_epi32(a1, 1); + ptr[stride * 6] = _mm_extract_epi32(a1, 2); + ptr[stride * 7] = _mm_extract_epi32(a1, 3); +} +NPY_FINLINE void npyv_storen_u32(npy_uint32 *ptr, npy_intp stride, npyv_u32 a) +{ npyv_storen_s32((npy_int32*)ptr, stride, a); } +NPY_FINLINE void npyv_storen_f32(float *ptr, npy_intp stride, npyv_f32 a) +{ npyv_storen_s32((npy_int32*)ptr, stride, _mm256_castps_si256(a)); } +//// 64 +NPY_FINLINE void npyv_storen_f64(double *ptr, npy_intp stride, npyv_f64 a) +{ + __m128d a0 = _mm256_castpd256_pd128(a); + __m128d a1 = _mm256_extractf128_pd(a, 1); + _mm_storel_pd(ptr + stride * 0, a0); + _mm_storeh_pd(ptr + stride * 1, a0); + _mm_storel_pd(ptr + stride * 2, a1); + _mm_storeh_pd(ptr + stride * 3, a1); +} +NPY_FINLINE void npyv_storen_u64(npy_uint64 *ptr, npy_intp stride, npyv_u64 a) +{ npyv_storen_f64((double*)ptr, stride, _mm256_castsi256_pd(a)); } +NPY_FINLINE void npyv_storen_s64(npy_int64 *ptr, npy_intp stride, npyv_s64 a) +{ npyv_storen_f64((double*)ptr, stride, _mm256_castsi256_pd(a)); } + +/********************************* + * Partial Load + *********************************/ +//// 32 +NPY_FINLINE npyv_s32 npyv_load_till_s32(const npy_int32 *ptr, npy_uintp nlane, npy_int32 fill) +{ + assert(nlane > 0); + const __m256i vfill = _mm256_set1_epi32(fill); + const __m256i steps = _mm256_setr_epi32(0, 1, 2, 3, 4, 5, 6, 7); + __m256i vnlane = _mm256_set1_epi32(nlane > 8 ? 8 : (int)nlane); + __m256i mask = _mm256_cmpgt_epi32(vnlane, steps); + __m256i payload = _mm256_maskload_epi32((const int*)ptr, mask); + return _mm256_blendv_epi8(vfill, payload, mask); +} +// fill zero to rest lanes +NPY_FINLINE npyv_s32 npyv_load_tillz_s32(const npy_int32 *ptr, npy_uintp nlane) +{ + assert(nlane > 0); + const __m256i steps = _mm256_setr_epi32(0, 1, 2, 3, 4, 5, 6, 7); + __m256i vnlane = _mm256_set1_epi32(nlane > 8 ? 8 : (int)nlane); + __m256i mask = _mm256_cmpgt_epi32(vnlane, steps); + return _mm256_maskload_epi32((const int*)ptr, mask); +} +//// 64 +NPY_FINLINE npyv_s64 npyv_load_till_s64(const npy_int64 *ptr, npy_uintp nlane, npy_int64 fill) +{ + assert(nlane > 0); + const __m256i vfill = _mm256_set1_epi64x(fill); + const __m256i steps = _mm256_setr_epi64x(0, 1, 2, 3); + __m256i vnlane = _mm256_set1_epi64x(nlane > 4 ? 4 : (int)nlane); + __m256i mask = _mm256_cmpgt_epi64(vnlane, steps); + __m256i payload = _mm256_maskload_epi64((const void*)ptr, mask); + return _mm256_blendv_epi8(vfill, payload, mask); +} +// fill zero to rest lanes +NPY_FINLINE npyv_s64 npyv_load_tillz_s64(const npy_int64 *ptr, npy_uintp nlane) +{ + assert(nlane > 0); + const __m256i steps = _mm256_setr_epi64x(0, 1, 2, 3); + __m256i vnlane = _mm256_set1_epi64x(nlane > 4 ? 4 : (int)nlane); + __m256i mask = _mm256_cmpgt_epi64(vnlane, steps); + return _mm256_maskload_epi64((const void*)ptr, mask); +} +/********************************* + * Non-contiguous partial load + *********************************/ +//// 32 +NPY_FINLINE npyv_s32 +npyv_loadn_till_s32(const npy_int32 *ptr, npy_intp stride, npy_uintp nlane, npy_int32 fill) +{ + assert(nlane > 0); + assert(llabs(stride) <= NPY_SIMD_MAXLOAD_STRIDE32); + const __m256i vfill = _mm256_set1_epi32(fill); + const __m256i steps = _mm256_setr_epi32(0, 1, 2, 3, 4, 5, 6, 7); + const __m256i idx = _mm256_mullo_epi32(_mm256_set1_epi32((int)stride), steps); + __m256i vnlane = _mm256_set1_epi32(nlane > 8 ? 8 : (int)nlane); + __m256i mask = _mm256_cmpgt_epi32(vnlane, steps); + return _mm256_mask_i32gather_epi32(vfill, (const int*)ptr, idx, mask, 4); +} +// fill zero to rest lanes +NPY_FINLINE npyv_s32 +npyv_loadn_tillz_s32(const npy_int32 *ptr, npy_intp stride, npy_uintp nlane) +{ return npyv_loadn_till_s32(ptr, stride, nlane, 0); } +//// 64 +NPY_FINLINE npyv_s64 +npyv_loadn_till_s64(const npy_int64 *ptr, npy_intp stride, npy_uintp nlane, npy_int64 fill) +{ + assert(nlane > 0); + const __m256i vfill = _mm256_set1_epi64x(fill); + const __m256i idx = _mm256_setr_epi64x(0, 1*stride, 2*stride, 3*stride); + const __m256i steps = _mm256_setr_epi64x(0, 1, 2, 3); + __m256i vnlane = _mm256_set1_epi64x(nlane > 4 ? 4 : (int)nlane); + __m256i mask = _mm256_cmpgt_epi64(vnlane, steps); + return _mm256_mask_i64gather_epi64(vfill, (const void*)ptr, idx, mask, 8); +} +// fill zero to rest lanes +NPY_FINLINE npyv_s64 +npyv_loadn_tillz_s64(const npy_int64 *ptr, npy_intp stride, npy_uintp nlane) +{ return npyv_loadn_till_s64(ptr, stride, nlane, 0); } +/********************************* + * Partial store + *********************************/ +//// 32 +NPY_FINLINE void npyv_store_till_s32(npy_int32 *ptr, npy_uintp nlane, npyv_s32 a) +{ + assert(nlane > 0); + const __m256i steps = _mm256_setr_epi32(0, 1, 2, 3, 4, 5, 6, 7); + __m256i vnlane = _mm256_set1_epi32(nlane > 8 ? 8 : (int)nlane); + __m256i mask = _mm256_cmpgt_epi32(vnlane, steps); + _mm256_maskstore_epi32((int*)ptr, mask, a); +} +//// 64 +NPY_FINLINE void npyv_store_till_s64(npy_int64 *ptr, npy_uintp nlane, npyv_s64 a) +{ + assert(nlane > 0); + const __m256i steps = _mm256_setr_epi64x(0, 1, 2, 3); + __m256i vnlane = _mm256_set1_epi64x(nlane > 8 ? 8 : (int)nlane); + __m256i mask = _mm256_cmpgt_epi64(vnlane, steps); + _mm256_maskstore_epi64((void*)ptr, mask, a); +} +/********************************* + * Non-contiguous partial store + *********************************/ +//// 32 +NPY_FINLINE void npyv_storen_till_s32(npy_int32 *ptr, npy_intp stride, npy_uintp nlane, npyv_s32 a) +{ + assert(nlane > 0); + __m128i a0 = _mm256_castsi256_si128(a); + __m128i a1 = _mm256_extracti128_si256(a, 1); + switch(nlane) { + default: + ptr[stride*7] = _mm_extract_epi32(a1, 3); + case 7: + ptr[stride*6] = _mm_extract_epi32(a1, 2); + case 6: + ptr[stride*5] = _mm_extract_epi32(a1, 1); + case 5: + ptr[stride*4] = _mm_extract_epi32(a1, 0); + case 4: + ptr[stride*3] = _mm_extract_epi32(a0, 3); + case 3: + ptr[stride*2] = _mm_extract_epi32(a0, 2); + case 2: + ptr[stride*1] = _mm_extract_epi32(a0, 1); + case 1: + ptr[stride*0] = _mm_extract_epi32(a0, 0); + } +} +//// 64 +NPY_FINLINE void npyv_storen_till_s64(npy_int64 *ptr, npy_intp stride, npy_uintp nlane, npyv_s64 a) +{ + assert(nlane > 0); + __m128d a0 = _mm256_castpd256_pd128(_mm256_castsi256_pd(a)); + __m128d a1 = _mm256_extractf128_pd(_mm256_castsi256_pd(a), 1); + double *dptr = (double*)ptr; + switch(nlane) { + default: + _mm_storeh_pd(dptr + stride * 3, a1); + case 3: + _mm_storel_pd(dptr + stride * 2, a1); + case 2: + _mm_storeh_pd(dptr + stride * 1, a0); + case 1: + _mm_storel_pd(dptr + stride * 0, a0); + } +} + +/***************************************************************************** + * Implement partial load/store for u32/f32/u64/f64... via reinterpret cast + *****************************************************************************/ +#define NPYV_IMPL_AVX2_REST_PARTIAL_TYPES(F_SFX, T_SFX) \ + NPY_FINLINE npyv_##F_SFX npyv_load_till_##F_SFX \ + (const npyv_lanetype_##F_SFX *ptr, npy_uintp nlane, npyv_lanetype_##F_SFX fill) \ + { \ + union { \ + npyv_lanetype_##F_SFX from_##F_SFX; \ + npyv_lanetype_##T_SFX to_##T_SFX; \ + } pun = {.from_##F_SFX = fill}; \ + return npyv_reinterpret_##F_SFX##_##T_SFX(npyv_load_till_##T_SFX( \ + (const npyv_lanetype_##T_SFX *)ptr, nlane, pun.to_##T_SFX \ + )); \ + } \ + NPY_FINLINE npyv_##F_SFX npyv_loadn_till_##F_SFX \ + (const npyv_lanetype_##F_SFX *ptr, npy_intp stride, npy_uintp nlane, \ + npyv_lanetype_##F_SFX fill) \ + { \ + union { \ + npyv_lanetype_##F_SFX from_##F_SFX; \ + npyv_lanetype_##T_SFX to_##T_SFX; \ + } pun = {.from_##F_SFX = fill}; \ + return npyv_reinterpret_##F_SFX##_##T_SFX(npyv_loadn_till_##T_SFX( \ + (const npyv_lanetype_##T_SFX *)ptr, stride, nlane, pun.to_##T_SFX \ + )); \ + } \ + NPY_FINLINE npyv_##F_SFX npyv_load_tillz_##F_SFX \ + (const npyv_lanetype_##F_SFX *ptr, npy_uintp nlane) \ + { \ + return npyv_reinterpret_##F_SFX##_##T_SFX(npyv_load_tillz_##T_SFX( \ + (const npyv_lanetype_##T_SFX *)ptr, nlane \ + )); \ + } \ + NPY_FINLINE npyv_##F_SFX npyv_loadn_tillz_##F_SFX \ + (const npyv_lanetype_##F_SFX *ptr, npy_intp stride, npy_uintp nlane) \ + { \ + return npyv_reinterpret_##F_SFX##_##T_SFX(npyv_loadn_tillz_##T_SFX( \ + (const npyv_lanetype_##T_SFX *)ptr, stride, nlane \ + )); \ + } \ + NPY_FINLINE void npyv_store_till_##F_SFX \ + (npyv_lanetype_##F_SFX *ptr, npy_uintp nlane, npyv_##F_SFX a) \ + { \ + npyv_store_till_##T_SFX( \ + (npyv_lanetype_##T_SFX *)ptr, nlane, \ + npyv_reinterpret_##T_SFX##_##F_SFX(a) \ + ); \ + } \ + NPY_FINLINE void npyv_storen_till_##F_SFX \ + (npyv_lanetype_##F_SFX *ptr, npy_intp stride, npy_uintp nlane, npyv_##F_SFX a) \ + { \ + npyv_storen_till_##T_SFX( \ + (npyv_lanetype_##T_SFX *)ptr, stride, nlane, \ + npyv_reinterpret_##T_SFX##_##F_SFX(a) \ + ); \ + } + +NPYV_IMPL_AVX2_REST_PARTIAL_TYPES(u32, s32) +NPYV_IMPL_AVX2_REST_PARTIAL_TYPES(f32, s32) +NPYV_IMPL_AVX2_REST_PARTIAL_TYPES(u64, s64) +NPYV_IMPL_AVX2_REST_PARTIAL_TYPES(f64, s64) #endif // _NPY_SIMD_AVX2_MEMORY_H diff --git a/numpy/core/src/common/simd/avx512/arithmetic.h b/numpy/core/src/common/simd/avx512/arithmetic.h index 824ae818ee3a..7372ca29e40d 100644 --- a/numpy/core/src/common/simd/avx512/arithmetic.h +++ b/numpy/core/src/common/simd/avx512/arithmetic.h @@ -129,4 +129,47 @@ NPY_FINLINE __m512i npyv_mul_u8(__m512i a, __m512i b) #define npyv_nmulsub_f32 _mm512_fnmsub_ps #define npyv_nmulsub_f64 _mm512_fnmsub_pd +/*************************** + * Reduce Sum + * there are three ways to implement reduce sum for AVX512: + * 1- split(256) /add /split(128) /add /hadd /hadd /extract + * 2- shuff(cross) /add /shuff(cross) /add /shuff /add /shuff /add /extract + * 3- _mm512_reduce_add_ps/pd + * The first one is been widely used by many projects + * + * the second one is used by Intel Compiler, maybe because the + * latency of hadd increased by (2-3) starting from Skylake-X which makes two + * extra shuffles(non-cross) cheaper. check https://godbolt.org/z/s3G9Er for more info. + * + * The third one is almost the same as the second one but only works for + * intel compiler/GCC 7.1/Clang 4, we still need to support older GCC. + ***************************/ +#ifdef NPY_HAVE_AVX512F_REDUCE + #define npyv_sum_f32 _mm512_reduce_add_ps + #define npyv_sum_f64 _mm512_reduce_add_pd +#else + NPY_FINLINE float npyv_sum_f32(npyv_f32 a) + { + __m512 h64 = _mm512_shuffle_f32x4(a, a, _MM_SHUFFLE(3, 2, 3, 2)); + __m512 sum32 = _mm512_add_ps(a, h64); + __m512 h32 = _mm512_shuffle_f32x4(sum32, sum32, _MM_SHUFFLE(1, 0, 3, 2)); + __m512 sum16 = _mm512_add_ps(sum32, h32); + __m512 h16 = _mm512_permute_ps(sum16, _MM_SHUFFLE(1, 0, 3, 2)); + __m512 sum8 = _mm512_add_ps(sum16, h16); + __m512 h4 = _mm512_permute_ps(sum8, _MM_SHUFFLE(2, 3, 0, 1)); + __m512 sum4 = _mm512_add_ps(sum8, h4); + return _mm_cvtss_f32(_mm512_castps512_ps128(sum4)); + } + NPY_FINLINE double npyv_sum_f64(npyv_f64 a) + { + __m512d h64 = _mm512_shuffle_f64x2(a, a, _MM_SHUFFLE(3, 2, 3, 2)); + __m512d sum32 = _mm512_add_pd(a, h64); + __m512d h32 = _mm512_permutex_pd(sum32, _MM_SHUFFLE(1, 0, 3, 2)); + __m512d sum16 = _mm512_add_pd(sum32, h32); + __m512d h16 = _mm512_permute_pd(sum16, _MM_SHUFFLE(2, 3, 0, 1)); + __m512d sum8 = _mm512_add_pd(sum16, h16); + return _mm_cvtsd_f64(_mm512_castpd512_pd128(sum8)); + } +#endif + #endif // _NPY_SIMD_AVX512_ARITHMETIC_H diff --git a/numpy/core/src/common/simd/avx512/avx512.h b/numpy/core/src/common/simd/avx512/avx512.h index 96fdf72b91f1..b09d772f2fec 100644 --- a/numpy/core/src/common/simd/avx512/avx512.h +++ b/numpy/core/src/common/simd/avx512/avx512.h @@ -4,6 +4,9 @@ #define NPY_SIMD 512 #define NPY_SIMD_WIDTH 64 #define NPY_SIMD_F64 1 +// Enough limit to allow us to use _mm512_i32gather_* and _mm512_i32scatter_* +#define NPY_SIMD_MAXLOAD_STRIDE32 (0x7fffffff / 16) +#define NPY_SIMD_MAXSTORE_STRIDE32 (0x7fffffff / 16) typedef __m512i npyv_u8; typedef __m512i npyv_s8; diff --git a/numpy/core/src/common/simd/avx512/memory.h b/numpy/core/src/common/simd/avx512/memory.h index e212c4555270..bffd6e907246 100644 --- a/numpy/core/src/common/simd/avx512/memory.h +++ b/numpy/core/src/common/simd/avx512/memory.h @@ -90,5 +90,243 @@ NPYV_IMPL_AVX512_MEM_INT(npy_int64, s64) // store higher part #define npyv_storeh_f32(PTR, VEC) _mm256_storeu_ps(PTR, npyv512_higher_ps256(VEC)) #define npyv_storeh_f64(PTR, VEC) _mm256_storeu_pd(PTR, npyv512_higher_pd256(VEC)) +/*************************** + * Non-contiguous Load + ***************************/ +//// 32 +NPY_FINLINE npyv_u32 npyv_loadn_u32(const npy_uint32 *ptr, npy_intp stride) +{ + assert(llabs(stride) <= NPY_SIMD_MAXLOAD_STRIDE32); + const __m512i steps = npyv_set_s32( + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 + ); + const __m512i idx = _mm512_mullo_epi32(steps, _mm512_set1_epi32((int)stride)); + return _mm512_i32gather_epi32(idx, (const __m512i*)ptr, 4); +} +NPY_FINLINE npyv_s32 npyv_loadn_s32(const npy_int32 *ptr, npy_intp stride) +{ return npyv_loadn_u32((const npy_uint32*)ptr, stride); } +NPY_FINLINE npyv_f32 npyv_loadn_f32(const float *ptr, npy_intp stride) +{ return _mm512_castsi512_ps(npyv_loadn_u32((const npy_uint32*)ptr, stride)); } +//// 64 +NPY_FINLINE npyv_u64 npyv_loadn_u64(const npy_uint64 *ptr, npy_intp stride) +{ + const __m512i idx = _mm512_setr_epi64( + 0*stride, 1*stride, 2*stride, 3*stride, + 4*stride, 5*stride, 6*stride, 7*stride + ); + return _mm512_i64gather_epi64(idx, (const __m512i*)ptr, 8); +} +NPY_FINLINE npyv_s64 npyv_loadn_s64(const npy_int64 *ptr, npy_intp stride) +{ return npyv_loadn_u64((const npy_uint64*)ptr, stride); } +NPY_FINLINE npyv_f64 npyv_loadn_f64(const double *ptr, npy_intp stride) +{ return _mm512_castsi512_pd(npyv_loadn_u64((const npy_uint64*)ptr, stride)); } +/*************************** + * Non-contiguous Store + ***************************/ +//// 32 +NPY_FINLINE void npyv_storen_u32(npy_uint32 *ptr, npy_intp stride, npyv_u32 a) +{ + assert(llabs(stride) <= NPY_SIMD_MAXSTORE_STRIDE32); + const __m512i steps = _mm512_setr_epi32( + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 + ); + const __m512i idx = _mm512_mullo_epi32(steps, _mm512_set1_epi32((int)stride)); + _mm512_i32scatter_epi32((__m512i*)ptr, idx, a, 4); +} +NPY_FINLINE void npyv_storen_s32(npy_int32 *ptr, npy_intp stride, npyv_s32 a) +{ npyv_storen_u32((npy_uint32*)ptr, stride, a); } +NPY_FINLINE void npyv_storen_f32(float *ptr, npy_intp stride, npyv_f32 a) +{ npyv_storen_u32((npy_uint32*)ptr, stride, _mm512_castps_si512(a)); } +//// 64 +NPY_FINLINE void npyv_storen_u64(npy_uint64 *ptr, npy_intp stride, npyv_u64 a) +{ + const __m512i idx = _mm512_setr_epi64( + 0*stride, 1*stride, 2*stride, 3*stride, + 4*stride, 5*stride, 6*stride, 7*stride + ); + _mm512_i64scatter_epi64((__m512i*)ptr, idx, a, 8); +} +NPY_FINLINE void npyv_storen_s64(npy_int64 *ptr, npy_intp stride, npyv_s64 a) +{ npyv_storen_u64((npy_uint64*)ptr, stride, a); } +NPY_FINLINE void npyv_storen_f64(double *ptr, npy_intp stride, npyv_f64 a) +{ npyv_storen_u64((npy_uint64*)ptr, stride, _mm512_castpd_si512(a)); } + +/********************************* + * Partial Load + *********************************/ +//// 32 +NPY_FINLINE npyv_s32 npyv_load_till_s32(const npy_int32 *ptr, npy_uintp nlane, npy_int32 fill) +{ + assert(nlane > 0); + const __m512i vfill = _mm512_set1_epi32(fill); + const __mmask16 mask = nlane > 31 ? -1 : (1 << nlane) - 1; + return _mm512_mask_loadu_epi32(vfill, mask, (const __m512i*)ptr); +} +// fill zero to rest lanes +NPY_FINLINE npyv_s32 npyv_load_tillz_s32(const npy_int32 *ptr, npy_uintp nlane) +{ + assert(nlane > 0); + const __mmask16 mask = nlane > 31 ? -1 : (1 << nlane) - 1; + return _mm512_maskz_loadu_epi32(mask, (const __m512i*)ptr); +} +//// 64 +NPY_FINLINE npyv_s64 npyv_load_till_s64(const npy_int64 *ptr, npy_uintp nlane, npy_int64 fill) +{ + assert(nlane > 0); + const __m512i vfill = _mm512_set1_epi64(fill); + const __mmask8 mask = nlane > 31 ? -1 : (1 << nlane) - 1; + return _mm512_mask_loadu_epi64(vfill, mask, (const __m512i*)ptr); +} +// fill zero to rest lanes +NPY_FINLINE npyv_s64 npyv_load_tillz_s64(const npy_int64 *ptr, npy_uintp nlane) +{ + assert(nlane > 0); + const __mmask8 mask = nlane > 15 ? -1 : (1 << nlane) - 1; + return _mm512_maskz_loadu_epi64(mask, (const __m512i*)ptr); +} +/********************************* + * Non-contiguous partial load + *********************************/ +//// 32 +NPY_FINLINE npyv_s32 +npyv_loadn_till_s32(const npy_int32 *ptr, npy_intp stride, npy_uintp nlane, npy_int32 fill) +{ + assert(nlane > 0); + assert(llabs(stride) <= NPY_SIMD_MAXLOAD_STRIDE32); + const __m512i steps = npyv_set_s32( + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 + ); + const __m512i idx = _mm512_mullo_epi32(steps, _mm512_set1_epi32((int)stride)); + const __m512i vfill = _mm512_set1_epi32(fill); + const __mmask16 mask = nlane > 31 ? -1 : (1 << nlane) - 1; + return _mm512_mask_i32gather_epi32(vfill, mask, idx, (const __m512i*)ptr, 4); +} +// fill zero to rest lanes +NPY_FINLINE npyv_s32 +npyv_loadn_tillz_s32(const npy_int32 *ptr, npy_intp stride, npy_uintp nlane) +{ return npyv_loadn_till_s32(ptr, stride, nlane, 0); } +//// 64 +NPY_FINLINE npyv_s64 +npyv_loadn_till_s64(const npy_int64 *ptr, npy_intp stride, npy_uintp nlane, npy_int64 fill) +{ + assert(nlane > 0); + const __m512i idx = _mm512_setr_epi64( + 0*stride, 1*stride, 2*stride, 3*stride, + 4*stride, 5*stride, 6*stride, 7*stride + ); + const __m512i vfill = _mm512_set1_epi64(fill); + const __mmask8 mask = nlane > 31 ? -1 : (1 << nlane) - 1; + return _mm512_mask_i64gather_epi64(vfill, mask, idx, (const __m512i*)ptr, 8); +} +// fill zero to rest lanes +NPY_FINLINE npyv_s64 +npyv_loadn_tillz_s64(const npy_int64 *ptr, npy_intp stride, npy_uintp nlane) +{ return npyv_loadn_till_s64(ptr, stride, nlane, 0); } +/********************************* + * Partial store + *********************************/ +//// 32 +NPY_FINLINE void npyv_store_till_s32(npy_int32 *ptr, npy_uintp nlane, npyv_s32 a) +{ + assert(nlane > 0); + const __mmask16 mask = nlane > 31 ? -1 : (1 << nlane) - 1; + _mm512_mask_storeu_epi32((__m512i*)ptr, mask, a); +} +//// 64 +NPY_FINLINE void npyv_store_till_s64(npy_int64 *ptr, npy_uintp nlane, npyv_s64 a) +{ + assert(nlane > 0); + const __mmask8 mask = nlane > 15 ? -1 : (1 << nlane) - 1; + _mm512_mask_storeu_epi64((__m512i*)ptr, mask, a); +} +/********************************* + * Non-contiguous partial store + *********************************/ +//// 32 +NPY_FINLINE void npyv_storen_till_s32(npy_int32 *ptr, npy_intp stride, npy_uintp nlane, npyv_s32 a) +{ + assert(nlane > 0); + assert(llabs(stride) <= NPY_SIMD_MAXSTORE_STRIDE32); + const __m512i steps = _mm512_setr_epi32( + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 + ); + const __m512i idx = _mm512_mullo_epi32(steps, _mm512_set1_epi32((int)stride)); + const __mmask16 mask = nlane > 31 ? -1 : (1 << nlane) - 1; + _mm512_mask_i32scatter_epi32((__m512i*)ptr, mask, idx, a, 4); +} +//// 64 +NPY_FINLINE void npyv_storen_till_s64(npy_int64 *ptr, npy_intp stride, npy_uintp nlane, npyv_s64 a) +{ + assert(nlane > 0); + const __m512i idx = _mm512_setr_epi64( + 0*stride, 1*stride, 2*stride, 3*stride, + 4*stride, 5*stride, 6*stride, 7*stride + ); + const __mmask8 mask = nlane > 15 ? -1 : (1 << nlane) - 1; + _mm512_mask_i64scatter_epi64((__m512i*)ptr, mask, idx, a, 8); +} + +/***************************************************************************** + * Implement partial load/store for u32/f32/u64/f64... via reinterpret cast + *****************************************************************************/ +#define NPYV_IMPL_AVX512_REST_PARTIAL_TYPES(F_SFX, T_SFX) \ + NPY_FINLINE npyv_##F_SFX npyv_load_till_##F_SFX \ + (const npyv_lanetype_##F_SFX *ptr, npy_uintp nlane, npyv_lanetype_##F_SFX fill) \ + { \ + union { \ + npyv_lanetype_##F_SFX from_##F_SFX; \ + npyv_lanetype_##T_SFX to_##T_SFX; \ + } pun = {.from_##F_SFX = fill}; \ + return npyv_reinterpret_##F_SFX##_##T_SFX(npyv_load_till_##T_SFX( \ + (const npyv_lanetype_##T_SFX *)ptr, nlane, pun.to_##T_SFX \ + )); \ + } \ + NPY_FINLINE npyv_##F_SFX npyv_loadn_till_##F_SFX \ + (const npyv_lanetype_##F_SFX *ptr, npy_intp stride, npy_uintp nlane, \ + npyv_lanetype_##F_SFX fill) \ + { \ + union { \ + npyv_lanetype_##F_SFX from_##F_SFX; \ + npyv_lanetype_##T_SFX to_##T_SFX; \ + } pun = {.from_##F_SFX = fill}; \ + return npyv_reinterpret_##F_SFX##_##T_SFX(npyv_loadn_till_##T_SFX( \ + (const npyv_lanetype_##T_SFX *)ptr, stride, nlane, pun.to_##T_SFX \ + )); \ + } \ + NPY_FINLINE npyv_##F_SFX npyv_load_tillz_##F_SFX \ + (const npyv_lanetype_##F_SFX *ptr, npy_uintp nlane) \ + { \ + return npyv_reinterpret_##F_SFX##_##T_SFX(npyv_load_tillz_##T_SFX( \ + (const npyv_lanetype_##T_SFX *)ptr, nlane \ + )); \ + } \ + NPY_FINLINE npyv_##F_SFX npyv_loadn_tillz_##F_SFX \ + (const npyv_lanetype_##F_SFX *ptr, npy_intp stride, npy_uintp nlane) \ + { \ + return npyv_reinterpret_##F_SFX##_##T_SFX(npyv_loadn_tillz_##T_SFX( \ + (const npyv_lanetype_##T_SFX *)ptr, stride, nlane \ + )); \ + } \ + NPY_FINLINE void npyv_store_till_##F_SFX \ + (npyv_lanetype_##F_SFX *ptr, npy_uintp nlane, npyv_##F_SFX a) \ + { \ + npyv_store_till_##T_SFX( \ + (npyv_lanetype_##T_SFX *)ptr, nlane, \ + npyv_reinterpret_##T_SFX##_##F_SFX(a) \ + ); \ + } \ + NPY_FINLINE void npyv_storen_till_##F_SFX \ + (npyv_lanetype_##F_SFX *ptr, npy_intp stride, npy_uintp nlane, npyv_##F_SFX a) \ + { \ + npyv_storen_till_##T_SFX( \ + (npyv_lanetype_##T_SFX *)ptr, stride, nlane, \ + npyv_reinterpret_##T_SFX##_##F_SFX(a) \ + ); \ + } + +NPYV_IMPL_AVX512_REST_PARTIAL_TYPES(u32, s32) +NPYV_IMPL_AVX512_REST_PARTIAL_TYPES(f32, s32) +NPYV_IMPL_AVX512_REST_PARTIAL_TYPES(u64, s64) +NPYV_IMPL_AVX512_REST_PARTIAL_TYPES(f64, s64) #endif // _NPY_SIMD_AVX512_MEMORY_H diff --git a/numpy/core/src/common/simd/neon/arithmetic.h b/numpy/core/src/common/simd/neon/arithmetic.h index 5eeee1bb6d02..bc14ffb75574 100644 --- a/numpy/core/src/common/simd/neon/arithmetic.h +++ b/numpy/core/src/common/simd/neon/arithmetic.h @@ -118,4 +118,17 @@ NPY_FINLINE npyv_f64 npyv_nmulsub_f64(npyv_f64 a, npyv_f64 b, npyv_f64 c) { return vfmsq_f64(vnegq_f64(c), a, b); } #endif // NPY_SIMD_F64 + +// Horizontal add: Calculates the sum of all vector elements. +#if NPY_SIMD_F64 + #define npyv_sum_f32 vaddvq_f32 + #define npyv_sum_f64 vaddvq_f64 +#else + NPY_FINLINE float npyv_sum_f32(npyv_f32 a) + { + float32x2_t r = vadd_f32(vget_high_f32(a), vget_low_f32(a)); + return vget_lane_f32(vpadd_f32(r, r), 0); + } +#endif + #endif // _NPY_SIMD_NEON_ARITHMETIC_H diff --git a/numpy/core/src/common/simd/neon/memory.h b/numpy/core/src/common/simd/neon/memory.h index afa703584dac..1e258f1bcbef 100644 --- a/numpy/core/src/common/simd/neon/memory.h +++ b/numpy/core/src/common/simd/neon/memory.h @@ -5,6 +5,8 @@ #ifndef _NPY_SIMD_NEON_MEMORY_H #define _NPY_SIMD_NEON_MEMORY_H +#include "misc.h" + /*************************** * load/store ***************************/ @@ -45,5 +47,290 @@ NPYV_IMPL_NEON_MEM(f32, float) #if NPY_SIMD_F64 NPYV_IMPL_NEON_MEM(f64, double) #endif +/*************************** + * Non-contiguous Load + ***************************/ +NPY_FINLINE npyv_s32 npyv_loadn_s32(const npy_int32 *ptr, npy_intp stride) +{ + switch (stride) { + case 2: + return vld2q_s32((const int32_t*)ptr).val[0]; + case 3: + return vld3q_s32((const int32_t*)ptr).val[0]; + case 4: + return vld4q_s32((const int32_t*)ptr).val[0]; + default:; + int32x2_t ax = vcreate_s32(*ptr); + int32x4_t a = vcombine_s32(ax, ax); + a = vld1q_lane_s32((const int32_t*)ptr + stride, a, 1); + a = vld1q_lane_s32((const int32_t*)ptr + stride*2, a, 2); + a = vld1q_lane_s32((const int32_t*)ptr + stride*3, a, 3); + return a; + } +} + +NPY_FINLINE npyv_u32 npyv_loadn_u32(const npy_uint32 *ptr, npy_intp stride) +{ + return npyv_reinterpret_u32_s32( + npyv_loadn_s32((const npy_int32*)ptr, stride) + ); +} +NPY_FINLINE npyv_f32 npyv_loadn_f32(const float *ptr, npy_intp stride) +{ + return npyv_reinterpret_f32_s32( + npyv_loadn_s32((const npy_int32*)ptr, stride) + ); +} +//// 64 +NPY_FINLINE npyv_s64 npyv_loadn_s64(const npy_int64 *ptr, npy_intp stride) +{ + return vcombine_s64( + vld1_s64((const int64_t*)ptr), vld1_s64((const int64_t*)ptr + stride) + ); +} +NPY_FINLINE npyv_u64 npyv_loadn_u64(const npy_uint64 *ptr, npy_intp stride) +{ + return npyv_reinterpret_u64_s64( + npyv_loadn_s64((const npy_int64*)ptr, stride) + ); +} +#if NPY_SIMD_F64 +NPY_FINLINE npyv_f64 npyv_loadn_f64(const double *ptr, npy_intp stride) +{ + return npyv_reinterpret_f64_s64( + npyv_loadn_s64((const npy_int64*)ptr, stride) + ); +} +#endif +/*************************** + * Non-contiguous Store + ***************************/ +//// 32 +NPY_FINLINE void npyv_storen_s32(npy_int32 *ptr, npy_intp stride, npyv_s32 a) +{ + vst1q_lane_s32((int32_t*)ptr, a, 0); + vst1q_lane_s32((int32_t*)ptr + stride, a, 1); + vst1q_lane_s32((int32_t*)ptr + stride*2, a, 2); + vst1q_lane_s32((int32_t*)ptr + stride*3, a, 3); +} +NPY_FINLINE void npyv_storen_u32(npy_uint32 *ptr, npy_intp stride, npyv_u32 a) +{ npyv_storen_s32((npy_int32*)ptr, stride, npyv_reinterpret_s32_u32(a)); } +NPY_FINLINE void npyv_storen_f32(float *ptr, npy_intp stride, npyv_f32 a) +{ npyv_storen_s32((npy_int32*)ptr, stride, npyv_reinterpret_s32_f32(a)); } +//// 64 +NPY_FINLINE void npyv_storen_s64(npy_int64 *ptr, npy_intp stride, npyv_s64 a) +{ + vst1q_lane_s64((int64_t*)ptr, a, 0); + vst1q_lane_s64((int64_t*)ptr + stride, a, 1); +} +NPY_FINLINE void npyv_storen_u64(npy_uint64 *ptr, npy_intp stride, npyv_u64 a) +{ npyv_storen_s64((npy_int64*)ptr, stride, npyv_reinterpret_s64_u64(a)); } + +#if NPY_SIMD_F64 +NPY_FINLINE void npyv_storen_f64(double *ptr, npy_intp stride, npyv_f64 a) +{ npyv_storen_s64((npy_int64*)ptr, stride, npyv_reinterpret_s64_f64(a)); } +#endif + +/********************************* + * Partial Load + *********************************/ +//// 32 +NPY_FINLINE npyv_s32 npyv_load_till_s32(const npy_int32 *ptr, npy_uintp nlane, npy_int32 fill) +{ + assert(nlane > 0); + switch(nlane) { + case 1: + return vld1q_lane_s32((const int32_t*)ptr, vdupq_n_s32(fill), 0); + case 2: + return vcombine_s32(vld1_s32((const int32_t*)ptr), vdup_n_s32(fill)); + case 3: + return vcombine_s32( + vld1_s32((const int32_t*)ptr), + vld1_lane_s32((const int32_t*)ptr + 2, vdup_n_s32(fill), 0) + ); + default: + return npyv_load_s32(ptr); + } +} +// fill zero to rest lanes +NPY_FINLINE npyv_s32 npyv_load_tillz_s32(const npy_int32 *ptr, npy_uintp nlane) +{ return npyv_load_till_s32(ptr, nlane, 0); } +//// 64 +NPY_FINLINE npyv_s64 npyv_load_till_s64(const npy_int64 *ptr, npy_uintp nlane, npy_int64 fill) +{ + assert(nlane > 0); + if (nlane == 1) { + return vcombine_s64(vld1_s64((const int64_t*)ptr), vdup_n_s64(fill)); + } + return npyv_load_s64(ptr); +} +// fill zero to rest lanes +NPY_FINLINE npyv_s64 npyv_load_tillz_s64(const npy_int64 *ptr, npy_uintp nlane) +{ return npyv_load_till_s64(ptr, nlane, 0); } + +/********************************* + * Non-contiguous partial load + *********************************/ +//// 32 +NPY_FINLINE npyv_s32 +npyv_loadn_till_s32(const npy_int32 *ptr, npy_intp stride, npy_uintp nlane, npy_int32 fill) +{ + assert(nlane > 0); + int32x4_t vfill = vdupq_n_s32(fill); + switch(nlane) { + case 3: + vfill = vld1q_lane_s32((const int32_t*)ptr + stride*2, vfill, 2); + case 2: + vfill = vld1q_lane_s32((const int32_t*)ptr + stride, vfill, 1); + case 1: + vfill = vld1q_lane_s32((const int32_t*)ptr, vfill, 0); + return vfill; + default: + return npyv_loadn_s32(ptr, stride); + } +} +NPY_FINLINE npyv_s32 +npyv_loadn_tillz_s32(const npy_int32 *ptr, npy_intp stride, npy_uintp nlane) +{ return npyv_loadn_till_s32(ptr, stride, nlane, 0); } + +NPY_FINLINE npyv_s64 +npyv_loadn_till_s64(const npy_int64 *ptr, npy_intp stride, npy_uintp nlane, npy_int64 fill) +{ + assert(nlane > 0); + if (nlane == 1) { + return vcombine_s64(vld1_s64((const int64_t*)ptr), vdup_n_s64(fill)); + } + return npyv_loadn_s64(ptr, stride); +} +// fill zero to rest lanes +NPY_FINLINE npyv_s64 npyv_loadn_tillz_s64(const npy_int64 *ptr, npy_intp stride, npy_uintp nlane) +{ return npyv_loadn_till_s64(ptr, stride, nlane, 0); } + +/********************************* + * Partial store + *********************************/ +//// 32 +NPY_FINLINE void npyv_store_till_s32(npy_int32 *ptr, npy_uintp nlane, npyv_s32 a) +{ + assert(nlane > 0); + switch(nlane) { + case 1: + vst1q_lane_s32((int32_t*)ptr, a, 0); + break; + case 2: + vst1_s32((int32_t*)ptr, vget_low_s32(a)); + break; + case 3: + vst1_s32((int32_t*)ptr, vget_low_s32(a)); + vst1q_lane_s32((int32_t*)ptr + 2, a, 2); + break; + default: + npyv_store_s32(ptr, a); + } +} +//// 64 +NPY_FINLINE void npyv_store_till_s64(npy_int64 *ptr, npy_uintp nlane, npyv_s64 a) +{ + assert(nlane > 0); + if (nlane == 1) { + vst1q_lane_s64((int64_t*)ptr, a, 0); + return; + } + npyv_store_s64(ptr, a); +} +/********************************* + * Non-contiguous partial store + *********************************/ +//// 32 +NPY_FINLINE void npyv_storen_till_s32(npy_int32 *ptr, npy_intp stride, npy_uintp nlane, npyv_s32 a) +{ + assert(nlane > 0); + switch(nlane) { + default: + vst1q_lane_s32((int32_t*)ptr + stride*3, a, 3); + case 3: + vst1q_lane_s32((int32_t*)ptr + stride*2, a, 2); + case 2: + vst1q_lane_s32((int32_t*)ptr + stride, a, 1); + case 1: + vst1q_lane_s32((int32_t*)ptr, a, 0); + break; + } +} +//// 64 +NPY_FINLINE void npyv_storen_till_s64(npy_int64 *ptr, npy_intp stride, npy_uintp nlane, npyv_s64 a) +{ + assert(nlane > 0); + if (nlane == 1) { + vst1q_lane_s64((int64_t*)ptr, a, 0); + return; + } + npyv_storen_s64(ptr, stride, a); +} + +/***************************************************************** + * Implement partial load/store for u32/f32/u64/f64... via casting + *****************************************************************/ +#define NPYV_IMPL_NEON_REST_PARTIAL_TYPES(F_SFX, T_SFX) \ + NPY_FINLINE npyv_##F_SFX npyv_load_till_##F_SFX \ + (const npyv_lanetype_##F_SFX *ptr, npy_uintp nlane, npyv_lanetype_##F_SFX fill) \ + { \ + union { \ + npyv_lanetype_##F_SFX from_##F_SFX; \ + npyv_lanetype_##T_SFX to_##T_SFX; \ + } pun = {.from_##F_SFX = fill}; \ + return npyv_reinterpret_##F_SFX##_##T_SFX(npyv_load_till_##T_SFX( \ + (const npyv_lanetype_##T_SFX *)ptr, nlane, pun.to_##T_SFX \ + )); \ + } \ + NPY_FINLINE npyv_##F_SFX npyv_loadn_till_##F_SFX \ + (const npyv_lanetype_##F_SFX *ptr, npy_intp stride, npy_uintp nlane, \ + npyv_lanetype_##F_SFX fill) \ + { \ + union { \ + npyv_lanetype_##F_SFX from_##F_SFX; \ + npyv_lanetype_##T_SFX to_##T_SFX; \ + } pun = {.from_##F_SFX = fill}; \ + return npyv_reinterpret_##F_SFX##_##T_SFX(npyv_loadn_till_##T_SFX( \ + (const npyv_lanetype_##T_SFX *)ptr, stride, nlane, pun.to_##T_SFX \ + )); \ + } \ + NPY_FINLINE npyv_##F_SFX npyv_load_tillz_##F_SFX \ + (const npyv_lanetype_##F_SFX *ptr, npy_uintp nlane) \ + { \ + return npyv_reinterpret_##F_SFX##_##T_SFX(npyv_load_tillz_##T_SFX( \ + (const npyv_lanetype_##T_SFX *)ptr, nlane \ + )); \ + } \ + NPY_FINLINE npyv_##F_SFX npyv_loadn_tillz_##F_SFX \ + (const npyv_lanetype_##F_SFX *ptr, npy_intp stride, npy_uintp nlane) \ + { \ + return npyv_reinterpret_##F_SFX##_##T_SFX(npyv_loadn_tillz_##T_SFX( \ + (const npyv_lanetype_##T_SFX *)ptr, stride, nlane \ + )); \ + } \ + NPY_FINLINE void npyv_store_till_##F_SFX \ + (npyv_lanetype_##F_SFX *ptr, npy_uintp nlane, npyv_##F_SFX a) \ + { \ + npyv_store_till_##T_SFX( \ + (npyv_lanetype_##T_SFX *)ptr, nlane, \ + npyv_reinterpret_##T_SFX##_##F_SFX(a) \ + ); \ + } \ + NPY_FINLINE void npyv_storen_till_##F_SFX \ + (npyv_lanetype_##F_SFX *ptr, npy_intp stride, npy_uintp nlane, npyv_##F_SFX a) \ + { \ + npyv_storen_till_##T_SFX( \ + (npyv_lanetype_##T_SFX *)ptr, stride, nlane, \ + npyv_reinterpret_##T_SFX##_##F_SFX(a) \ + ); \ + } + +NPYV_IMPL_NEON_REST_PARTIAL_TYPES(u32, s32) +NPYV_IMPL_NEON_REST_PARTIAL_TYPES(f32, s32) +NPYV_IMPL_NEON_REST_PARTIAL_TYPES(u64, s64) +#if NPY_SIMD_F64 +NPYV_IMPL_NEON_REST_PARTIAL_TYPES(f64, s64) +#endif #endif // _NPY_SIMD_NEON_MEMORY_H diff --git a/numpy/core/src/common/simd/simd.h b/numpy/core/src/common/simd/simd.h index 2f39c8427b5d..8804223c9fef 100644 --- a/numpy/core/src/common/simd/simd.h +++ b/numpy/core/src/common/simd/simd.h @@ -49,6 +49,55 @@ typedef double npyv_lanetype_f64; #define NPY_SIMD_WIDTH 0 #define NPY_SIMD_F64 0 #endif +/** + * Some SIMD extensions currently(AVX2, AVX512F) require (de facto) + * a maximum number of strides sizes when dealing with non-contiguous memory access. + * + * Therefore the following functions must be used to check the maximum + * acceptable limit of strides before using any of non-contiguous load/store intrinsics. + * + * For instance: + * npy_intp ld_stride = step[0] / sizeof(float); + * npy_intp st_stride = step[1] / sizeof(float); + * + * if (npyv_loadable_stride_f32(ld_stride) && npyv_storable_stride_f32(st_stride)) { + * for (;;) + * npyv_f32 a = npyv_loadn_f32(ld_pointer, ld_stride); + * // ... + * npyv_storen_f32(st_pointer, st_stride, a); + * } + * else { + * for (;;) + * // C scalars + * } + */ +#ifndef NPY_SIMD_MAXLOAD_STRIDE32 + #define NPY_SIMD_MAXLOAD_STRIDE32 0 +#endif +#ifndef NPY_SIMD_MAXSTORE_STRIDE32 + #define NPY_SIMD_MAXSTORE_STRIDE32 0 +#endif +#ifndef NPY_SIMD_MAXLOAD_STRIDE64 + #define NPY_SIMD_MAXLOAD_STRIDE64 0 +#endif +#ifndef NPY_SIMD_MAXSTORE_STRIDE64 + #define NPY_SIMD_MAXSTORE_STRIDE64 0 +#endif +#define NPYV_IMPL_MAXSTRIDE(SFX, MAXLOAD, MAXSTORE) \ + NPY_FINLINE int npyv_loadable_stride_##SFX(npy_intp stride) \ + { return MAXLOAD > 0 ? llabs(stride) <= MAXLOAD : 1; } \ + NPY_FINLINE int npyv_storable_stride_##SFX(npy_intp stride) \ + { return MAXSTORE > 0 ? llabs(stride) <= MAXSTORE : 1; } +#if NPY_SIMD + NPYV_IMPL_MAXSTRIDE(u32, NPY_SIMD_MAXLOAD_STRIDE32, NPY_SIMD_MAXSTORE_STRIDE32) + NPYV_IMPL_MAXSTRIDE(s32, NPY_SIMD_MAXLOAD_STRIDE32, NPY_SIMD_MAXSTORE_STRIDE32) + NPYV_IMPL_MAXSTRIDE(f32, NPY_SIMD_MAXLOAD_STRIDE32, NPY_SIMD_MAXSTORE_STRIDE32) + NPYV_IMPL_MAXSTRIDE(u64, NPY_SIMD_MAXLOAD_STRIDE64, NPY_SIMD_MAXSTORE_STRIDE64) + NPYV_IMPL_MAXSTRIDE(s64, NPY_SIMD_MAXLOAD_STRIDE64, NPY_SIMD_MAXSTORE_STRIDE64) +#endif +#if NPY_SIMD_F64 + NPYV_IMPL_MAXSTRIDE(f64, NPY_SIMD_MAXLOAD_STRIDE64, NPY_SIMD_MAXSTORE_STRIDE64) +#endif #ifdef __cplusplus } diff --git a/numpy/core/src/common/simd/sse/arithmetic.h b/numpy/core/src/common/simd/sse/arithmetic.h index 717dacd39f8d..8440cc52e5c8 100644 --- a/numpy/core/src/common/simd/sse/arithmetic.h +++ b/numpy/core/src/common/simd/sse/arithmetic.h @@ -147,4 +147,31 @@ NPY_FINLINE __m128i npyv_mul_u8(__m128i a, __m128i b) return npyv_sub_f64(npyv_mul_f64(neg_a, b), c); } #endif // !NPY_HAVE_FMA3 + +// Horizontal add: Calculates the sum of all vector elements. +NPY_FINLINE float npyv_sum_f32(__m128 a) +{ +#ifdef NPY_HAVE_SSE3 + __m128 sum_halves = _mm_hadd_ps(a, a); + return _mm_cvtss_f32(_mm_hadd_ps(sum_halves, sum_halves)); +#else + __m128 t1 = _mm_movehl_ps(a, a); + __m128 t2 = _mm_add_ps(a, t1); + __m128 t3 = _mm_shuffle_ps(t2, t2, 1); + __m128 t4 = _mm_add_ss(t2, t3); + return _mm_cvtss_f32(t4); +#endif +} + +NPY_FINLINE double npyv_sum_f64(__m128d a) +{ +#ifdef NPY_HAVE_SSE3 + return _mm_cvtsd_f64(_mm_hadd_pd(a, a)); +#else + return _mm_cvtsd_f64(_mm_add_pd(a, _mm_unpackhi_pd(a, a))); +#endif +} + #endif // _NPY_SIMD_SSE_ARITHMETIC_H + + diff --git a/numpy/core/src/common/simd/sse/memory.h b/numpy/core/src/common/simd/sse/memory.h index 1a555d6f03de..1074c3b02efe 100644 --- a/numpy/core/src/common/simd/sse/memory.h +++ b/numpy/core/src/common/simd/sse/memory.h @@ -5,6 +5,8 @@ #ifndef _NPY_SIMD_SSE_MEMORY_H #define _NPY_SIMD_SSE_MEMORY_H +#include "misc.h" + /*************************** * load/store ***************************/ @@ -70,5 +72,427 @@ NPYV_IMPL_SSE_MEM_INT(npy_int64, s64) // store higher part #define npyv_storeh_f32(PTR, VEC) npyv_storeh_u32((npy_uint32*)(PTR), _mm_castps_si128(VEC)) #define npyv_storeh_f64(PTR, VEC) npyv_storeh_u32((npy_uint32*)(PTR), _mm_castpd_si128(VEC)) +/*************************** + * Non-contiguous Load + ***************************/ +//// 32 +NPY_FINLINE npyv_s32 npyv_loadn_s32(const npy_int32 *ptr, npy_intp stride) +{ + __m128i a = _mm_cvtsi32_si128(*ptr); +#ifdef NPY_HAVE_SSE41 + a = _mm_insert_epi32(a, ptr[stride], 1); + a = _mm_insert_epi32(a, ptr[stride*2], 2); + a = _mm_insert_epi32(a, ptr[stride*3], 3); +#else + __m128i a1 = _mm_cvtsi32_si128(ptr[stride]); + __m128i a2 = _mm_cvtsi32_si128(ptr[stride*2]); + __m128i a3 = _mm_cvtsi32_si128(ptr[stride*3]); + a = _mm_unpacklo_epi32(a, a1); + a = _mm_unpacklo_epi64(a, _mm_unpacklo_epi32(a2, a3)); +#endif + return a; +} +NPY_FINLINE npyv_u32 npyv_loadn_u32(const npy_uint32 *ptr, npy_intp stride) +{ return npyv_loadn_s32((const npy_int32*)ptr, stride); } +NPY_FINLINE npyv_f32 npyv_loadn_f32(const float *ptr, npy_intp stride) +{ return _mm_castsi128_ps(npyv_loadn_s32((const npy_int32*)ptr, stride)); } +//// 64 +NPY_FINLINE npyv_f64 npyv_loadn_f64(const double *ptr, npy_intp stride) +{ return _mm_loadh_pd(npyv_loadl_f64(ptr), ptr + stride); } +NPY_FINLINE npyv_u64 npyv_loadn_u64(const npy_uint64 *ptr, npy_intp stride) +{ return _mm_castpd_si128(npyv_loadn_f64((const double*)ptr, stride)); } +NPY_FINLINE npyv_s64 npyv_loadn_s64(const npy_int64 *ptr, npy_intp stride) +{ return _mm_castpd_si128(npyv_loadn_f64((const double*)ptr, stride)); } +/*************************** + * Non-contiguous Store + ***************************/ +//// 32 +NPY_FINLINE void npyv_storen_s32(npy_int32 *ptr, npy_intp stride, npyv_s32 a) +{ + ptr[stride * 0] = _mm_cvtsi128_si32(a); +#ifdef NPY_HAVE_SSE41 + ptr[stride * 1] = _mm_extract_epi32(a, 1); + ptr[stride * 2] = _mm_extract_epi32(a, 2); + ptr[stride * 3] = _mm_extract_epi32(a, 3); +#else + ptr[stride * 1] = _mm_cvtsi128_si32(_mm_shuffle_epi32(a, _MM_SHUFFLE(0, 0, 0, 1))); + ptr[stride * 2] = _mm_cvtsi128_si32(_mm_shuffle_epi32(a, _MM_SHUFFLE(0, 0, 0, 2))); + ptr[stride * 3] = _mm_cvtsi128_si32(_mm_shuffle_epi32(a, _MM_SHUFFLE(0, 0, 0, 3))); +#endif +} +NPY_FINLINE void npyv_storen_u32(npy_uint32 *ptr, npy_intp stride, npyv_u32 a) +{ npyv_storen_s32((npy_int32*)ptr, stride, a); } +NPY_FINLINE void npyv_storen_f32(float *ptr, npy_intp stride, npyv_f32 a) +{ npyv_storen_s32((npy_int32*)ptr, stride, _mm_castps_si128(a)); } +//// 64 +NPY_FINLINE void npyv_storen_f64(double *ptr, npy_intp stride, npyv_f64 a) +{ + _mm_storel_pd(ptr, a); + _mm_storeh_pd(ptr + stride, a); +} +NPY_FINLINE void npyv_storen_u64(npy_uint64 *ptr, npy_intp stride, npyv_u64 a) +{ npyv_storen_f64((double*)ptr, stride, _mm_castsi128_pd(a)); } +NPY_FINLINE void npyv_storen_s64(npy_int64 *ptr, npy_intp stride, npyv_s64 a) +{ npyv_storen_f64((double*)ptr, stride, _mm_castsi128_pd(a)); } + +/********************************* + * Partial Load + *********************************/ +#if defined(__clang__) && __clang_major__ > 7 + /** + * Clang >=8 perform aggressive optimization that tends to + * zero the bits of upper half part of vectors even + * when we try to fill it up with certain scalars, + * which my lead to zero division errors. + */ + #define NPYV__CLANG_ZEROUPPER +#endif +//// 32 +NPY_FINLINE npyv_s32 npyv_load_till_s32(const npy_int32 *ptr, npy_uintp nlane, npy_int32 fill) +{ + assert(nlane > 0); +#ifdef NPYV__CLANG_ZEROUPPER + if (nlane > 3) { + return npyv_load_s32(ptr); + } + npy_int32 NPY_DECL_ALIGNED(16) data[4] = {fill, fill, fill, fill}; + for (npy_uint64 i = 0; i < nlane; ++i) { + data[i] = ptr[i]; + } + return npyv_loada_s32(data); +#else + #ifndef NPY_HAVE_SSE41 + const short *wptr = (const short*)ptr; + #endif + const __m128i vfill = npyv_setall_s32(fill); + __m128i a; + switch(nlane) { + case 2: + return _mm_castpd_si128( + _mm_loadl_pd(_mm_castsi128_pd(vfill), (double*)ptr) + ); + #ifdef NPY_HAVE_SSE41 + case 1: + return _mm_insert_epi32(vfill, ptr[0], 0); + case 3: + a = _mm_loadl_epi64((const __m128i*)ptr); + a = _mm_insert_epi32(a, ptr[2], 2); + a = _mm_insert_epi32(a, fill, 3); + return a; + #else + case 1: + a = _mm_insert_epi16(vfill, wptr[0], 0); + return _mm_insert_epi16(a, wptr[1], 1); + case 3: + a = _mm_loadl_epi64((const __m128i*)ptr); + a = _mm_unpacklo_epi64(a, vfill); + a = _mm_insert_epi16(a, wptr[4], 4); + a = _mm_insert_epi16(a, wptr[5], 5); + return a; + #endif // NPY_HAVE_SSE41 + default: + return npyv_load_s32(ptr); + } +#endif +} +// fill zero to rest lanes +NPY_FINLINE npyv_s32 npyv_load_tillz_s32(const npy_int32 *ptr, npy_uintp nlane) +{ + assert(nlane > 0); + switch(nlane) { + case 1: + return _mm_cvtsi32_si128(*ptr); + case 2: + return _mm_loadl_epi64((const __m128i*)ptr); + case 3:; + npyv_s32 a = _mm_loadl_epi64((const __m128i*)ptr); + #ifdef NPY_HAVE_SSE41 + return _mm_insert_epi32(a, ptr[2], 2); + #else + return _mm_unpacklo_epi64(a, _mm_cvtsi32_si128(ptr[2])); + #endif + default: + return npyv_load_s32(ptr); + } +} +//// 64 +NPY_FINLINE npyv_s64 npyv_load_till_s64(const npy_int64 *ptr, npy_uintp nlane, npy_int64 fill) +{ + assert(nlane > 0); +#ifdef NPYV__CLANG_ZEROUPPER + if (nlane <= 2) { + npy_int64 NPY_DECL_ALIGNED(16) data[2] = {fill, fill}; + for (npy_uint64 i = 0; i < nlane; ++i) { + data[i] = ptr[i]; + } + return npyv_loada_s64(data); + } +#else + if (nlane == 1) { + const __m128i vfill = npyv_setall_s64(fill); + return _mm_castpd_si128( + _mm_loadl_pd(_mm_castsi128_pd(vfill), (double*)ptr) + ); + } +#endif + return npyv_load_s64(ptr); +} +// fill zero to rest lanes +NPY_FINLINE npyv_s64 npyv_load_tillz_s64(const npy_int64 *ptr, npy_uintp nlane) +{ + assert(nlane > 0); + if (nlane == 1) { + return _mm_loadl_epi64((const __m128i*)ptr); + } + return npyv_load_s64(ptr); +} +/********************************* + * Non-contiguous partial load + *********************************/ +//// 32 +NPY_FINLINE npyv_s32 +npyv_loadn_till_s32(const npy_int32 *ptr, npy_intp stride, npy_uintp nlane, npy_int32 fill) +{ + assert(nlane > 0); +#ifdef NPYV__CLANG_ZEROUPPER + if (nlane > 3) { + return npyv_loadn_s32(ptr, stride); + } + npy_int32 NPY_DECL_ALIGNED(16) data[4] = {fill, fill, fill, fill}; + for (npy_uint64 i = 0; i < nlane; ++i) { + data[i] = ptr[stride*i]; + } + return npyv_loada_s32(data); +#else + __m128i vfill = npyv_setall_s32(fill); + #ifndef NPY_HAVE_SSE41 + const short *wptr = (const short*)ptr; + #endif + switch(nlane) { + #ifdef NPY_HAVE_SSE41 + case 3: + vfill = _mm_insert_epi32(vfill, ptr[stride*2], 2); + case 2: + vfill = _mm_insert_epi32(vfill, ptr[stride], 1); + case 1: + vfill = _mm_insert_epi32(vfill, ptr[0], 0); + break; + #else + case 3: + vfill = _mm_unpacklo_epi32(_mm_cvtsi32_si128(ptr[stride*2]), vfill); + case 2: + vfill = _mm_unpacklo_epi64(_mm_unpacklo_epi32( + _mm_cvtsi32_si128(*ptr), _mm_cvtsi32_si128(ptr[stride]) + ), vfill); + break; + case 1: + vfill = _mm_insert_epi16(vfill, wptr[0], 0); + vfill = _mm_insert_epi16(vfill, wptr[1], 1); + break; + #endif // NPY_HAVE_SSE41 + default: + return npyv_loadn_s32(ptr, stride); + } // switch + return vfill; +#endif +} +// fill zero to rest lanes +NPY_FINLINE npyv_s32 +npyv_loadn_tillz_s32(const npy_int32 *ptr, npy_intp stride, npy_uintp nlane) +{ + assert(nlane > 0); + switch(nlane) { + case 1: + return _mm_cvtsi32_si128(ptr[0]); + case 2:; + npyv_s32 a = _mm_cvtsi32_si128(ptr[0]); +#ifdef NPY_HAVE_SSE41 + return _mm_insert_epi32(a, ptr[stride], 1); +#else + return _mm_unpacklo_epi32(a, _mm_cvtsi32_si128(ptr[stride])); +#endif // NPY_HAVE_SSE41 + case 3:; + a = _mm_cvtsi32_si128(ptr[0]); +#ifdef NPY_HAVE_SSE41 + a = _mm_insert_epi32(a, ptr[stride], 1); + a = _mm_insert_epi32(a, ptr[stride*2], 2); + return a; +#else + a = _mm_unpacklo_epi32(a, _mm_cvtsi32_si128(ptr[stride])); + a = _mm_unpacklo_epi64(a, _mm_cvtsi32_si128(ptr[stride*2])); + return a; +#endif // NPY_HAVE_SSE41 + default: + return npyv_loadn_s32(ptr, stride); + } +} +//// 64 +NPY_FINLINE npyv_s64 +npyv_loadn_till_s64(const npy_int64 *ptr, npy_intp stride, npy_uintp nlane, npy_int64 fill) +{ + assert(nlane > 0); +#ifdef NPYV__CLANG_ZEROUPPER + if (nlane <= 2) { + npy_int64 NPY_DECL_ALIGNED(16) data[2] = {fill, fill}; + for (npy_uint64 i = 0; i < nlane; ++i) { + data[i] = ptr[i*stride]; + } + return npyv_loada_s64(data); + } +#else + if (nlane == 1) { + const __m128i vfill = npyv_setall_s64(fill); + return _mm_castpd_si128( + _mm_loadl_pd(_mm_castsi128_pd(vfill), (double*)ptr) + ); + } +#endif + return npyv_loadn_s64(ptr, stride); +} +// fill zero to rest lanes +NPY_FINLINE npyv_s64 npyv_loadn_tillz_s64(const npy_int64 *ptr, npy_intp stride, npy_uintp nlane) +{ + assert(nlane > 0); + if (nlane == 1) { + return _mm_loadl_epi64((const __m128i*)ptr); + } + return npyv_loadn_s64(ptr, stride); +} +/********************************* + * Partial store + *********************************/ +//// 32 +NPY_FINLINE void npyv_store_till_s32(npy_int32 *ptr, npy_uintp nlane, npyv_s32 a) +{ + assert(nlane > 0); + switch(nlane) { + case 1: + *ptr = _mm_cvtsi128_si32(a); + break; + case 2: + _mm_storel_epi64((__m128i *)ptr, a); + break; + case 3: + _mm_storel_epi64((__m128i *)ptr, a); + #ifdef NPY_HAVE_SSE41 + ptr[2] = _mm_extract_epi32(a, 2); + #else + ptr[2] = _mm_cvtsi128_si32(_mm_shuffle_epi32(a, _MM_SHUFFLE(0, 0, 0, 2))); + #endif + break; + default: + npyv_store_s32(ptr, a); + } +} +//// 64 +NPY_FINLINE void npyv_store_till_s64(npy_int64 *ptr, npy_uintp nlane, npyv_s64 a) +{ + assert(nlane > 0); + if (nlane == 1) { + _mm_storel_epi64((__m128i *)ptr, a); + return; + } + npyv_store_s64(ptr, a); +} +/********************************* + * Non-contiguous partial store + *********************************/ +//// 32 +NPY_FINLINE void npyv_storen_till_s32(npy_int32 *ptr, npy_intp stride, npy_uintp nlane, npyv_s32 a) +{ + assert(nlane > 0); + switch(nlane) { +#ifdef NPY_HAVE_SSE41 + default: + ptr[stride*3] = _mm_extract_epi32(a, 3); + case 3: + ptr[stride*2] = _mm_extract_epi32(a, 2); + case 2: + ptr[stride*1] = _mm_extract_epi32(a, 1); +#else + default: + ptr[stride*3] = _mm_cvtsi128_si32(_mm_shuffle_epi32(a, _MM_SHUFFLE(0, 0, 0, 3))); + case 3: + ptr[stride*2] = _mm_cvtsi128_si32(_mm_shuffle_epi32(a, _MM_SHUFFLE(0, 0, 0, 2))); + case 2: + ptr[stride*1] = _mm_cvtsi128_si32(_mm_shuffle_epi32(a, _MM_SHUFFLE(0, 0, 0, 1))); +#endif + case 1: + ptr[stride*0] = _mm_cvtsi128_si32(a); + break; + } +} +//// 64 +NPY_FINLINE void npyv_storen_till_s64(npy_int64 *ptr, npy_intp stride, npy_uintp nlane, npyv_s64 a) +{ + assert(nlane > 0); + if (nlane == 1) { + _mm_storel_epi64((__m128i *)ptr, a); + return; + } + npyv_storen_s64(ptr, stride, a); +} +/***************************************************************** + * Implement partial load/store for u32/f32/u64/f64... via casting + *****************************************************************/ +#define NPYV_IMPL_SSE_REST_PARTIAL_TYPES(F_SFX, T_SFX) \ + NPY_FINLINE npyv_##F_SFX npyv_load_till_##F_SFX \ + (const npyv_lanetype_##F_SFX *ptr, npy_uintp nlane, npyv_lanetype_##F_SFX fill) \ + { \ + union { \ + npyv_lanetype_##F_SFX from_##F_SFX; \ + npyv_lanetype_##T_SFX to_##T_SFX; \ + } pun = {.from_##F_SFX = fill}; \ + return npyv_reinterpret_##F_SFX##_##T_SFX(npyv_load_till_##T_SFX( \ + (const npyv_lanetype_##T_SFX *)ptr, nlane, pun.to_##T_SFX \ + )); \ + } \ + NPY_FINLINE npyv_##F_SFX npyv_loadn_till_##F_SFX \ + (const npyv_lanetype_##F_SFX *ptr, npy_intp stride, npy_uintp nlane, \ + npyv_lanetype_##F_SFX fill) \ + { \ + union { \ + npyv_lanetype_##F_SFX from_##F_SFX; \ + npyv_lanetype_##T_SFX to_##T_SFX; \ + } pun = {.from_##F_SFX = fill}; \ + return npyv_reinterpret_##F_SFX##_##T_SFX(npyv_loadn_till_##T_SFX( \ + (const npyv_lanetype_##T_SFX *)ptr, stride, nlane, pun.to_##T_SFX \ + )); \ + } \ + NPY_FINLINE npyv_##F_SFX npyv_load_tillz_##F_SFX \ + (const npyv_lanetype_##F_SFX *ptr, npy_uintp nlane) \ + { \ + return npyv_reinterpret_##F_SFX##_##T_SFX(npyv_load_tillz_##T_SFX( \ + (const npyv_lanetype_##T_SFX *)ptr, nlane \ + )); \ + } \ + NPY_FINLINE npyv_##F_SFX npyv_loadn_tillz_##F_SFX \ + (const npyv_lanetype_##F_SFX *ptr, npy_intp stride, npy_uintp nlane) \ + { \ + return npyv_reinterpret_##F_SFX##_##T_SFX(npyv_loadn_tillz_##T_SFX( \ + (const npyv_lanetype_##T_SFX *)ptr, stride, nlane \ + )); \ + } \ + NPY_FINLINE void npyv_store_till_##F_SFX \ + (npyv_lanetype_##F_SFX *ptr, npy_uintp nlane, npyv_##F_SFX a) \ + { \ + npyv_store_till_##T_SFX( \ + (npyv_lanetype_##T_SFX *)ptr, nlane, \ + npyv_reinterpret_##T_SFX##_##F_SFX(a) \ + ); \ + } \ + NPY_FINLINE void npyv_storen_till_##F_SFX \ + (npyv_lanetype_##F_SFX *ptr, npy_intp stride, npy_uintp nlane, npyv_##F_SFX a) \ + { \ + npyv_storen_till_##T_SFX( \ + (npyv_lanetype_##T_SFX *)ptr, stride, nlane, \ + npyv_reinterpret_##T_SFX##_##F_SFX(a) \ + ); \ + } + +NPYV_IMPL_SSE_REST_PARTIAL_TYPES(u32, s32) +NPYV_IMPL_SSE_REST_PARTIAL_TYPES(f32, s32) +NPYV_IMPL_SSE_REST_PARTIAL_TYPES(u64, s64) +NPYV_IMPL_SSE_REST_PARTIAL_TYPES(f64, s64) #endif // _NPY_SIMD_SSE_MEMORY_H diff --git a/numpy/core/src/common/simd/vsx/arithmetic.h b/numpy/core/src/common/simd/vsx/arithmetic.h index 6ef007676d03..2f6762e636b4 100644 --- a/numpy/core/src/common/simd/vsx/arithmetic.h +++ b/numpy/core/src/common/simd/vsx/arithmetic.h @@ -116,4 +116,16 @@ #define npyv_nmulsub_f32 vec_nmadd // equivalent to -(a*b + c) #define npyv_nmulsub_f64 vec_nmadd +// Horizontal add: Calculates the sum of all vector elements. +NPY_FINLINE float npyv_sum_f32(npyv_f32 a) +{ + npyv_f32 sum = vec_add(a, npyv_combineh_f32(a, a)); + return vec_extract(sum, 0) + vec_extract(sum, 1); +} + +NPY_FINLINE double npyv_sum_f64(npyv_f64 a) +{ + return vec_extract(a, 0) + vec_extract(a, 1); +} + #endif // _NPY_SIMD_VSX_ARITHMETIC_H diff --git a/numpy/core/src/common/simd/vsx/memory.h b/numpy/core/src/common/simd/vsx/memory.h index e0d908bf906b..08a0a9276cc6 100644 --- a/numpy/core/src/common/simd/vsx/memory.h +++ b/numpy/core/src/common/simd/vsx/memory.h @@ -4,147 +4,343 @@ #ifndef _NPY_SIMD_VSX_MEMORY_H #define _NPY_SIMD_VSX_MEMORY_H + +#include "misc.h" + /**************************** - * load/store + * Private utilities ****************************/ // TODO: test load by cast #define VSX__CAST_lOAD 0 #if VSX__CAST_lOAD - #define npyv__load(PTR, T_VEC) (*((T_VEC*)(PTR))) + #define npyv__load(T_VEC, PTR) (*((T_VEC*)(PTR))) #else /** * CLANG fails to load unaligned addresses via vec_xl, vec_xst * so we failback to vec_vsx_ld, vec_vsx_st */ #if (defined(__GNUC__) && !defined(vec_xl)) || (defined(__clang__) && !defined(__IBMC__)) - #define npyv__load(PTR, T_VEC) vec_vsx_ld(0, PTR) + #define npyv__load(T_VEC, PTR) vec_vsx_ld(0, PTR) #else - #define npyv__load(PTR, T_VEC) vec_xl(0, PTR) + #define npyv__load(T_VEC, PTR) vec_xl(0, PTR) #endif #endif -// unaligned load -#define npyv_load_u8(PTR) npyv__load(PTR, npyv_u8) -#define npyv_load_s8(PTR) npyv__load(PTR, npyv_s8) -#define npyv_load_u16(PTR) npyv__load(PTR, npyv_u16) -#define npyv_load_s16(PTR) npyv__load(PTR, npyv_s16) -#define npyv_load_u32(PTR) npyv__load(PTR, npyv_u32) -#define npyv_load_s32(PTR) npyv__load(PTR, npyv_s32) -#define npyv_load_f32(PTR) npyv__load(PTR, npyv_f32) -#define npyv_load_f64(PTR) npyv__load(PTR, npyv_f64) -#if VSX__CAST_lOAD - #define npyv_load_u64(PTR) npyv__load(PTR, npyv_u64) - #define npyv_load_s64(PTR) npyv__load(PTR, npyv_s64) +// unaligned store +#if (defined(__GNUC__) && !defined(vec_xl)) || (defined(__clang__) && !defined(__IBMC__)) + #define npyv__store(PTR, VEC) vec_vsx_st(VEC, 0, PTR) #else - #define npyv_load_u64(PTR) ((npyv_u64)npyv_load_u32((const unsigned int*)PTR)) - #define npyv_load_s64(PTR) ((npyv_s64)npyv_load_s32((const unsigned int*)PTR)) + #define npyv__store(PTR, VEC) vec_xst(VEC, 0, PTR) #endif -// aligned load -#define npyv_loada_u8(PTR) vec_ld(0, PTR) -#define npyv_loada_s8 npyv_loada_u8 -#define npyv_loada_u16 npyv_loada_u8 -#define npyv_loada_s16 npyv_loada_u8 -#define npyv_loada_u32 npyv_loada_u8 -#define npyv_loada_s32 npyv_loada_u8 -#define npyv_loada_u64 npyv_load_u64 -#define npyv_loada_s64 npyv_load_s64 -#define npyv_loada_f32 npyv_loada_u8 -#define npyv_loada_f64 npyv_load_f64 -// stream load -#define npyv_loads_u8 npyv_loada_u8 -#define npyv_loads_s8 npyv_loada_s8 -#define npyv_loads_u16 npyv_loada_u16 -#define npyv_loads_s16 npyv_loada_s16 -#define npyv_loads_u32 npyv_loada_u32 -#define npyv_loads_s32 npyv_loada_s32 -#define npyv_loads_u64 npyv_loada_u64 -#define npyv_loads_s64 npyv_loada_s64 -#define npyv_loads_f32 npyv_loada_f32 -#define npyv_loads_f64 npyv_loada_f64 -// load lower part + // avoid aliasing rules #ifdef __cplusplus template - NPY_FINLINE npy_uint64 *npyv__ptr2u64(T_PTR *ptr) - { return npy_uint64 *ptr64 = (npy_uint64*)ptr; return ptr; } + NPY_FINLINE npy_uint64 *npyv__ptr2u64(const T_PTR *ptr) + { npy_uint64 *ptr64 = (npy_uint64*)ptr; return ptr64; } #else - NPY_FINLINE npy_uint64 *npyv__ptr2u64(void *ptr) - { npy_uint64 *ptr64 = ptr; return ptr64; } + NPY_FINLINE npy_uint64 *npyv__ptr2u64(const void *ptr) + { npy_uint64 *ptr64 = (npy_uint64*)ptr; return ptr64; } #endif // __cplusplus -#if defined(__clang__) && !defined(__IBMC__) - // vec_promote doesn't support doubleword on clang - #define npyv_loadl_u64(PTR) npyv_setall_u64(*npyv__ptr2u64(PTR)) -#else - #define npyv_loadl_u64(PTR) vec_promote(*npyv__ptr2u64(PTR), 0) -#endif -#define npyv_loadl_u8(PTR) ((npyv_u8)npyv_loadl_u64(PTR)) -#define npyv_loadl_s8(PTR) ((npyv_s8)npyv_loadl_u64(PTR)) -#define npyv_loadl_u16(PTR) ((npyv_u16)npyv_loadl_u64(PTR)) -#define npyv_loadl_s16(PTR) ((npyv_s16)npyv_loadl_u64(PTR)) -#define npyv_loadl_u32(PTR) ((npyv_u32)npyv_loadl_u64(PTR)) -#define npyv_loadl_s32(PTR) ((npyv_s32)npyv_loadl_u64(PTR)) -#define npyv_loadl_s64(PTR) ((npyv_s64)npyv_loadl_u64(PTR)) -#define npyv_loadl_f32(PTR) ((npyv_f32)npyv_loadl_u64(PTR)) -#define npyv_loadl_f64(PTR) ((npyv_f64)npyv_loadl_u64(PTR)) -// unaligned store -#if (defined(__GNUC__) && !defined(vec_xl)) || (defined(__clang__) && !defined(__IBMC__)) - #define npyv_store_u8(PTR, VEC) vec_vsx_st(VEC, 0, PTR) -#else - #define npyv_store_u8(PTR, VEC) vec_xst(VEC, 0, PTR) -#endif -#define npyv_store_s8 npyv_store_u8 -#define npyv_store_u16 npyv_store_u8 -#define npyv_store_s16 npyv_store_u8 -#define npyv_store_u32 npyv_store_u8 -#define npyv_store_s32 npyv_store_u8 -#define npyv_store_u64(PTR, VEC) npyv_store_u8((unsigned int*)PTR, (npyv_u32)VEC) -#define npyv_store_s64(PTR, VEC) npyv_store_u8((unsigned int*)PTR, (npyv_u32)VEC) -#define npyv_store_f32 npyv_store_u8 -#define npyv_store_f64 npyv_store_u8 -// aligned store -#define npyv_storea_u8(PTR, VEC) vec_st(VEC, 0, PTR) -#define npyv_storea_s8 npyv_storea_u8 -#define npyv_storea_u16 npyv_storea_u8 -#define npyv_storea_s16 npyv_storea_u8 -#define npyv_storea_u32 npyv_storea_u8 -#define npyv_storea_s32 npyv_storea_u8 -#define npyv_storea_u64 npyv_store_u64 -#define npyv_storea_s64 npyv_store_s64 -#define npyv_storea_f32 npyv_storea_u8 -#define npyv_storea_f64 npyv_store_f64 -// stream store -#define npyv_stores_u8 npyv_storea_u8 -#define npyv_stores_s8 npyv_storea_s8 -#define npyv_stores_u16 npyv_storea_u16 -#define npyv_stores_s16 npyv_storea_s16 -#define npyv_stores_u32 npyv_storea_u32 -#define npyv_stores_s32 npyv_storea_s32 -#define npyv_stores_u64 npyv_storea_u64 -#define npyv_stores_s64 npyv_storea_s64 -#define npyv_stores_f32 npyv_storea_f32 -#define npyv_stores_f64 npyv_storea_f64 + +// load lower part +NPY_FINLINE npyv_u64 npyv__loadl(const void *ptr) +{ + #if defined(__clang__) && !defined(__IBMC__) + // vec_promote doesn't support doubleword on clang + return npyv_setall_u64(*npyv__ptr2u64(ptr)); + #else + return vec_promote(*npyv__ptr2u64(ptr), 0); + #endif +} // store lower part -#define npyv_storel_u8(PTR, VEC) \ +#define npyv__storel(PTR, VEC) \ *npyv__ptr2u64(PTR) = vec_extract(((npyv_u64)VEC), 0) -#define npyv_storel_s8 npyv_storel_u8 -#define npyv_storel_u16 npyv_storel_u8 -#define npyv_storel_s16 npyv_storel_u8 -#define npyv_storel_u32 npyv_storel_u8 -#define npyv_storel_s32 npyv_storel_u8 -#define npyv_storel_s64 npyv_storel_u8 -#define npyv_storel_u64 npyv_storel_u8 -#define npyv_storel_f32 npyv_storel_u8 -#define npyv_storel_f64 npyv_storel_u8 -// store higher part -#define npyv_storeh_u8(PTR, VEC) \ + +#define npyv__storeh(PTR, VEC) \ *npyv__ptr2u64(PTR) = vec_extract(((npyv_u64)VEC), 1) -#define npyv_storeh_s8 npyv_storeh_u8 -#define npyv_storeh_u16 npyv_storeh_u8 -#define npyv_storeh_s16 npyv_storeh_u8 -#define npyv_storeh_u32 npyv_storeh_u8 -#define npyv_storeh_s32 npyv_storeh_u8 -#define npyv_storeh_s64 npyv_storeh_u8 -#define npyv_storeh_u64 npyv_storeh_u8 -#define npyv_storeh_f32 npyv_storeh_u8 -#define npyv_storeh_f64 npyv_storeh_u8 + +/**************************** + * load/store + ****************************/ +#define NPYV_IMPL_VSX_MEM(SFX, DW_CAST) \ + NPY_FINLINE npyv_##SFX npyv_load_##SFX(const npyv_lanetype_##SFX *ptr) \ + { return (npyv_##SFX)npyv__load(npyv_##SFX, (const npyv_lanetype_##DW_CAST*)ptr); } \ + NPY_FINLINE npyv_##SFX npyv_loada_##SFX(const npyv_lanetype_##SFX *ptr) \ + { return (npyv_##SFX)vec_ld(0, (const npyv_lanetype_u32*)ptr); } \ + NPY_FINLINE npyv_##SFX npyv_loads_##SFX(const npyv_lanetype_##SFX *ptr) \ + { return npyv_loada_##SFX(ptr); } \ + NPY_FINLINE npyv_##SFX npyv_loadl_##SFX(const npyv_lanetype_##SFX *ptr) \ + { return (npyv_##SFX)npyv__loadl(ptr); } \ + NPY_FINLINE void npyv_store_##SFX(npyv_lanetype_##SFX *ptr, npyv_##SFX vec) \ + { npyv__store((npyv_lanetype_##DW_CAST*)ptr, (npyv_##DW_CAST)vec); } \ + NPY_FINLINE void npyv_storea_##SFX(npyv_lanetype_##SFX *ptr, npyv_##SFX vec) \ + { vec_st((npyv_u32)vec, 0, (npyv_lanetype_u32*)ptr); } \ + NPY_FINLINE void npyv_stores_##SFX(npyv_lanetype_##SFX *ptr, npyv_##SFX vec) \ + { npyv_storea_##SFX(ptr, vec); } \ + NPY_FINLINE void npyv_storel_##SFX(npyv_lanetype_##SFX *ptr, npyv_##SFX vec) \ + { npyv__storel(ptr, vec); } \ + NPY_FINLINE void npyv_storeh_##SFX(npyv_lanetype_##SFX *ptr, npyv_##SFX vec) \ + { npyv__storeh(ptr, vec); } + +NPYV_IMPL_VSX_MEM(u8, u8) +NPYV_IMPL_VSX_MEM(s8, s8) +NPYV_IMPL_VSX_MEM(u16, u16) +NPYV_IMPL_VSX_MEM(s16, s16) +NPYV_IMPL_VSX_MEM(u32, u32) +NPYV_IMPL_VSX_MEM(s32, s32) +NPYV_IMPL_VSX_MEM(u64, f64) +NPYV_IMPL_VSX_MEM(s64, f64) +NPYV_IMPL_VSX_MEM(f32, f32) +NPYV_IMPL_VSX_MEM(f64, f64) + +/*************************** + * Non-contiguous Load + ***************************/ +//// 32 +NPY_FINLINE npyv_u32 npyv_loadn_u32(const npy_uint32 *ptr, npy_intp stride) +{ + return npyv_set_u32( + ptr[stride * 0], ptr[stride * 1], + ptr[stride * 2], ptr[stride * 3] + ); +} +NPY_FINLINE npyv_s32 npyv_loadn_s32(const npy_int32 *ptr, npy_intp stride) +{ return (npyv_s32)npyv_loadn_u32((const npy_uint32*)ptr, stride); } +NPY_FINLINE npyv_f32 npyv_loadn_f32(const float *ptr, npy_intp stride) +{ return (npyv_f32)npyv_loadn_u32((const npy_uint32*)ptr, stride); } +//// 64 +NPY_FINLINE npyv_u64 npyv_loadn_u64(const npy_uint64 *ptr, npy_intp stride) +{ return npyv_set_u64(ptr[0], ptr[stride]); } +NPY_FINLINE npyv_s64 npyv_loadn_s64(const npy_int64 *ptr, npy_intp stride) +{ return npyv_set_s64(ptr[0], ptr[stride]); } +NPY_FINLINE npyv_f64 npyv_loadn_f64(const double *ptr, npy_intp stride) +{ return npyv_set_f64(ptr[0], ptr[stride]); } +/*************************** + * Non-contiguous Store + ***************************/ +//// 32 +NPY_FINLINE void npyv_storen_u32(npy_uint32 *ptr, npy_intp stride, npyv_u32 a) +{ + ptr[stride * 0] = vec_extract(a, 0); + ptr[stride * 1] = vec_extract(a, 1); + ptr[stride * 2] = vec_extract(a, 2); + ptr[stride * 3] = vec_extract(a, 3); +} +NPY_FINLINE void npyv_storen_s32(npy_int32 *ptr, npy_intp stride, npyv_s32 a) +{ npyv_storen_u32((npy_uint32*)ptr, stride, (npyv_u32)a); } +NPY_FINLINE void npyv_storen_f32(float *ptr, npy_intp stride, npyv_f32 a) +{ npyv_storen_u32((npy_uint32*)ptr, stride, (npyv_u32)a); } +//// 64 +NPY_FINLINE void npyv_storen_u64(npy_uint64 *ptr, npy_intp stride, npyv_u64 a) +{ + ptr[stride * 0] = vec_extract(a, 0); + ptr[stride * 1] = vec_extract(a, 1); +} +NPY_FINLINE void npyv_storen_s64(npy_int64 *ptr, npy_intp stride, npyv_s64 a) +{ npyv_storen_u64((npy_uint64*)ptr, stride, (npyv_u64)a); } +NPY_FINLINE void npyv_storen_f64(double *ptr, npy_intp stride, npyv_f64 a) +{ npyv_storen_u64((npy_uint64*)ptr, stride, (npyv_u64)a); } + +/********************************* + * Partial Load + *********************************/ +//// 32 +NPY_FINLINE npyv_s32 npyv_load_till_s32(const npy_int32 *ptr, npy_uintp nlane, npy_int32 fill) +{ + assert(nlane > 0); + npyv_s32 vfill = npyv_setall_s32(fill); + switch(nlane) { + case 1: + return vec_insert(ptr[0], vfill, 0); + case 2: + return (npyv_s32)vec_insert( + *npyv__ptr2u64(ptr), (npyv_u64)vfill, 0 + ); + case 3: + vfill = vec_insert(ptr[2], vfill, 2); + return (npyv_s32)vec_insert( + *npyv__ptr2u64(ptr), (npyv_u64)vfill, 0 + ); + default: + return npyv_load_s32(ptr); + } +} +// fill zero to rest lanes +NPY_FINLINE npyv_s32 npyv_load_tillz_s32(const npy_int32 *ptr, npy_uintp nlane) +{ return npyv_load_till_s32(ptr, nlane, 0); } +//// 64 +NPY_FINLINE npyv_s64 npyv_load_till_s64(const npy_int64 *ptr, npy_uintp nlane, npy_int64 fill) +{ + assert(nlane > 0); + if (nlane == 1) { + return npyv_set_s64(ptr[0], fill); + } + return npyv_load_s64(ptr); +} +// fill zero to rest lanes +NPY_FINLINE npyv_s64 npyv_load_tillz_s64(const npy_int64 *ptr, npy_uintp nlane) +{ return npyv_load_till_s64(ptr, nlane, 0); } +/********************************* + * Non-contiguous partial load + *********************************/ +//// 32 +NPY_FINLINE npyv_s32 +npyv_loadn_till_s32(const npy_int32 *ptr, npy_intp stride, npy_uintp nlane, npy_int32 fill) +{ + assert(nlane > 0); + npyv_s32 vfill = npyv_setall_s32(fill); + switch(nlane) { + case 3: + vfill = vec_insert(ptr[stride*2], vfill, 2); + case 2: + vfill = vec_insert(ptr[stride], vfill, 1); + case 1: + vfill = vec_insert(*ptr, vfill, 0); + break; + default: + return npyv_loadn_s32(ptr, stride); + } // switch + return vfill; +} +// fill zero to rest lanes +NPY_FINLINE npyv_s32 +npyv_loadn_tillz_s32(const npy_int32 *ptr, npy_intp stride, npy_uintp nlane) +{ return npyv_loadn_till_s32(ptr, stride, nlane, 0); } +//// 64 +NPY_FINLINE npyv_s64 +npyv_loadn_till_s64(const npy_int64 *ptr, npy_intp stride, npy_uintp nlane, npy_int64 fill) +{ + assert(nlane > 0); + if (nlane == 1) { + return npyv_set_s64(*ptr, fill); + } + return npyv_loadn_s64(ptr, stride); +} +// fill zero to rest lanes +NPY_FINLINE npyv_s64 npyv_loadn_tillz_s64(const npy_int64 *ptr, npy_intp stride, npy_uintp nlane) +{ return npyv_loadn_till_s64(ptr, stride, nlane, 0); } +/********************************* + * Partial store + *********************************/ +//// 32 +NPY_FINLINE void npyv_store_till_s32(npy_int32 *ptr, npy_uintp nlane, npyv_s32 a) +{ + assert(nlane > 0); + switch(nlane) { + case 1: + *ptr = vec_extract(a, 0); + break; + case 2: + npyv_storel_s32(ptr, a); + break; + case 3: + npyv_storel_s32(ptr, a); + ptr[2] = vec_extract(a, 2); + break; + default: + npyv_store_s32(ptr, a); + } +} +//// 64 +NPY_FINLINE void npyv_store_till_s64(npy_int64 *ptr, npy_uintp nlane, npyv_s64 a) +{ + assert(nlane > 0); + if (nlane == 1) { + npyv_storel_s64(ptr, a); + return; + } + npyv_store_s64(ptr, a); +} +/********************************* + * Non-contiguous partial store + *********************************/ +//// 32 +NPY_FINLINE void npyv_storen_till_s32(npy_int32 *ptr, npy_intp stride, npy_uintp nlane, npyv_s32 a) +{ + assert(nlane > 0); + switch(nlane) { + default: + ptr[stride*3] = vec_extract(a, 3); + case 3: + ptr[stride*2] = vec_extract(a, 2); + case 2: + ptr[stride*1] = vec_extract(a, 1); + case 1: + ptr[stride*0] = vec_extract(a, 0); + break; + } +} +//// 64 +NPY_FINLINE void npyv_storen_till_s64(npy_int64 *ptr, npy_intp stride, npy_uintp nlane, npyv_s64 a) +{ + assert(nlane > 0); + if (nlane == 1) { + npyv_storel_s64(ptr, a); + return; + } + npyv_storen_s64(ptr, stride, a); +} +/***************************************************************** + * Implement partial load/store for u32/f32/u64/f64... via casting + *****************************************************************/ +#define NPYV_IMPL_VSX_REST_PARTIAL_TYPES(F_SFX, T_SFX) \ + NPY_FINLINE npyv_##F_SFX npyv_load_till_##F_SFX \ + (const npyv_lanetype_##F_SFX *ptr, npy_uintp nlane, npyv_lanetype_##F_SFX fill) \ + { \ + union { \ + npyv_lanetype_##F_SFX from_##F_SFX; \ + npyv_lanetype_##T_SFX to_##T_SFX; \ + } pun = {.from_##F_SFX = fill}; \ + return npyv_reinterpret_##F_SFX##_##T_SFX(npyv_load_till_##T_SFX( \ + (const npyv_lanetype_##T_SFX *)ptr, nlane, pun.to_##T_SFX \ + )); \ + } \ + NPY_FINLINE npyv_##F_SFX npyv_loadn_till_##F_SFX \ + (const npyv_lanetype_##F_SFX *ptr, npy_intp stride, npy_uintp nlane, \ + npyv_lanetype_##F_SFX fill) \ + { \ + union { \ + npyv_lanetype_##F_SFX from_##F_SFX; \ + npyv_lanetype_##T_SFX to_##T_SFX; \ + } pun = {.from_##F_SFX = fill}; \ + return npyv_reinterpret_##F_SFX##_##T_SFX(npyv_loadn_till_##T_SFX( \ + (const npyv_lanetype_##T_SFX *)ptr, stride, nlane, pun.to_##T_SFX \ + )); \ + } \ + NPY_FINLINE npyv_##F_SFX npyv_load_tillz_##F_SFX \ + (const npyv_lanetype_##F_SFX *ptr, npy_uintp nlane) \ + { \ + return npyv_reinterpret_##F_SFX##_##T_SFX(npyv_load_tillz_##T_SFX( \ + (const npyv_lanetype_##T_SFX *)ptr, nlane \ + )); \ + } \ + NPY_FINLINE npyv_##F_SFX npyv_loadn_tillz_##F_SFX \ + (const npyv_lanetype_##F_SFX *ptr, npy_intp stride, npy_uintp nlane) \ + { \ + return npyv_reinterpret_##F_SFX##_##T_SFX(npyv_loadn_tillz_##T_SFX( \ + (const npyv_lanetype_##T_SFX *)ptr, stride, nlane \ + )); \ + } \ + NPY_FINLINE void npyv_store_till_##F_SFX \ + (npyv_lanetype_##F_SFX *ptr, npy_uintp nlane, npyv_##F_SFX a) \ + { \ + npyv_store_till_##T_SFX( \ + (npyv_lanetype_##T_SFX *)ptr, nlane, \ + npyv_reinterpret_##T_SFX##_##F_SFX(a) \ + ); \ + } \ + NPY_FINLINE void npyv_storen_till_##F_SFX \ + (npyv_lanetype_##F_SFX *ptr, npy_intp stride, npy_uintp nlane, npyv_##F_SFX a) \ + { \ + npyv_storen_till_##T_SFX( \ + (npyv_lanetype_##T_SFX *)ptr, stride, nlane, \ + npyv_reinterpret_##T_SFX##_##F_SFX(a) \ + ); \ + } + +NPYV_IMPL_VSX_REST_PARTIAL_TYPES(u32, s32) +NPYV_IMPL_VSX_REST_PARTIAL_TYPES(f32, s32) +NPYV_IMPL_VSX_REST_PARTIAL_TYPES(u64, s64) +NPYV_IMPL_VSX_REST_PARTIAL_TYPES(f64, s64) #endif // _NPY_SIMD_VSX_MEMORY_H diff --git a/numpy/core/src/multiarray/_datetime.h b/numpy/core/src/multiarray/_datetime.h index 4e7ade5edacc..421b03f93c5a 100644 --- a/numpy/core/src/multiarray/_datetime.h +++ b/numpy/core/src/multiarray/_datetime.h @@ -200,17 +200,15 @@ convert_pyobject_to_datetime_metadata(PyObject *obj, PyArray_DatetimeMetaData *out_meta); /* - * 'ret' is a PyUString containing the datetime string, and this - * function appends the metadata string to it. + * Returns datetime metadata as a new reference a Unicode object. + * Returns NULL on error. * * If 'skip_brackets' is true, skips the '[]'. * - * This function steals the reference 'ret' */ NPY_NO_EXPORT PyObject * -append_metastr_to_string(PyArray_DatetimeMetaData *meta, - int skip_brackets, - PyObject *ret); +metastr_to_unicode(PyArray_DatetimeMetaData *meta, int skip_brackets); + /* * Tests for and converts a Python datetime.datetime or datetime.date diff --git a/numpy/core/src/multiarray/_multiarray_tests.c.src b/numpy/core/src/multiarray/_multiarray_tests.c.src index ea04c82bdf2e..5b6b6dc78fe0 100644 --- a/numpy/core/src/multiarray/_multiarray_tests.c.src +++ b/numpy/core/src/multiarray/_multiarray_tests.c.src @@ -176,17 +176,20 @@ test_neighborhood_iterator(PyObject* NPY_UNUSED(self), PyObject* args) /* Compute boundaries for the neighborhood iterator */ for (i = 0; i < 2 * PyArray_NDIM(ax); ++i) { PyObject* bound; + bound = PySequence_GetItem(b, i); if (bound == NULL) { goto clean_itx; } - if (!PyInt_Check(bound)) { + /* PyLong_AsSsize checks for PyLong */ + bounds[i] = PyLong_AsSsize_t(bound); + if (error_converting(bounds[i])) { + PyErr_Clear(); PyErr_SetString(PyExc_ValueError, - "bound not long"); + "bound is invalid"); Py_DECREF(bound); goto clean_itx; } - bounds[i] = PyLong_AsSsize_t(bound); Py_DECREF(bound); } @@ -335,17 +338,20 @@ test_neighborhood_iterator_oob(PyObject* NPY_UNUSED(self), PyObject* args) /* Compute boundaries for the neighborhood iterator */ for (i = 0; i < 2 * PyArray_NDIM(ax); ++i) { PyObject* bound; + bound = PySequence_GetItem(b1, i); if (bound == NULL) { goto clean_itx; } - if (!PyInt_Check(bound)) { + /* PyLong_AsSsize checks for PyLong */ + bounds[i] = PyLong_AsSsize_t(bound); + if (error_converting(bounds[i])) { + PyErr_Clear(); PyErr_SetString(PyExc_ValueError, - "bound not long"); + "bound is invalid"); Py_DECREF(bound); goto clean_itx; } - bounds[i] = PyLong_AsSsize_t(bound); Py_DECREF(bound); } @@ -359,17 +365,20 @@ test_neighborhood_iterator_oob(PyObject* NPY_UNUSED(self), PyObject* args) for (i = 0; i < 2 * PyArray_NDIM(ax); ++i) { PyObject* bound; + bound = PySequence_GetItem(b2, i); if (bound == NULL) { goto clean_itx; } - if (!PyInt_Check(bound)) { + /* PyLong_AsSsize checks for PyLong */ + bounds[i] = PyLong_AsSsize_t(bound); + if (error_converting(bounds[i])) { + PyErr_Clear(); PyErr_SetString(PyExc_ValueError, - "bound not long"); + "bound is invalid"); Py_DECREF(bound); goto clean_itx; } - bounds[i] = PyLong_AsSsize_t(bound); Py_DECREF(bound); } @@ -610,6 +619,71 @@ fromstring_null_term_c_api(PyObject *dummy, PyObject *byte_obj) } +/* + * Create a custom field dtype from an existing void one (and test some errors). + * The dtypes created by this function may be not be usable (or even crash + * while using). + */ +static PyObject * +create_custom_field_dtype(PyObject *NPY_UNUSED(mod), PyObject *args) +{ + PyArray_Descr *dtype; + PyTypeObject *scalar_type; + PyTypeObject *original_type = NULL; + int error_path; + + if (!PyArg_ParseTuple(args, "O!O!i", + &PyArrayDescr_Type, &dtype, + &PyType_Type, &scalar_type, + &error_path)) { + return NULL; + } + /* check that the result should be more or less valid */ + if (dtype->type_num != NPY_VOID || dtype->fields == NULL || + !PyDict_CheckExact(dtype->fields) || + PyTuple_Size(dtype->names) != 1 || + !PyDataType_REFCHK(dtype) || + dtype->elsize != sizeof(PyObject *)) { + PyErr_SetString(PyExc_ValueError, + "Bad dtype passed to test function, must be an object " + "containing void with a single field."); + return NULL; + } + + /* Copy and then appropriate this dtype */ + original_type = Py_TYPE(dtype); + dtype = PyArray_DescrNew(dtype); + if (dtype == NULL) { + return NULL; + } + + Py_INCREF(scalar_type); + Py_SETREF(dtype->typeobj, scalar_type); + if (error_path == 1) { + /* Test that we reject this, if fields was not already set */ + Py_SETREF(dtype->fields, NULL); + } + else if (error_path == 2) { + /* + * Test that we reject this if the type is not set to something that + * we are pretty sure can be safely replaced. + */ + Py_SET_TYPE(dtype, scalar_type); + } + else if (error_path != 0) { + PyErr_SetString(PyExc_ValueError, + "invalid error argument to test function."); + } + if (PyArray_RegisterDataType(dtype) < 0) { + /* Fix original type in the error_path == 2 case. */ + Py_SET_TYPE(dtype, original_type); + return NULL; + } + Py_INCREF(dtype); + return (PyObject *)dtype; +} + + /* check no elison for avoided increfs */ static PyObject * incref_elide(PyObject *dummy, PyObject *args) @@ -2081,6 +2155,9 @@ static PyMethodDef Multiarray_TestsMethods[] = { {"fromstring_null_term_c_api", fromstring_null_term_c_api, METH_O, NULL}, + {"create_custom_field_dtype", + create_custom_field_dtype, + METH_VARARGS, NULL}, {"incref_elide", incref_elide, METH_VARARGS, NULL}, diff --git a/numpy/core/src/multiarray/array_coercion.c b/numpy/core/src/multiarray/array_coercion.c index 3f3fd1387b64..ffb9bdbe848e 100644 --- a/numpy/core/src/multiarray/array_coercion.c +++ b/numpy/core/src/multiarray/array_coercion.c @@ -128,7 +128,9 @@ _prime_global_pytype_to_type_dict(void) /** - * Add a new mapping from a python type to the DType class. + * Add a new mapping from a python type to the DType class. For a user + * defined legacy dtype, this function does nothing unless the pytype + * subclass from `np.generic`. * * This assumes that the DType class is guaranteed to hold on the * python type (this assumption is guaranteed). @@ -145,21 +147,29 @@ _PyArray_MapPyTypeToDType( { PyObject *Dtype_obj = (PyObject *)DType; - if (userdef) { + if (userdef && !PyObject_IsSubclass( + (PyObject *)pytype, (PyObject *)&PyGenericArrType_Type)) { /* - * It seems we did not strictly enforce this in the legacy dtype - * API, but assume that it is always true. Further, this could be - * relaxed in the future. In particular we should have a new - * superclass of ``np.generic`` in order to note enforce the array - * scalar behaviour. + * We expect that user dtypes (for now) will subclass some numpy + * scalar class to allow automatic discovery. */ - if (!PyObject_IsSubclass((PyObject *)pytype, (PyObject *)&PyGenericArrType_Type)) { - PyErr_Format(PyExc_RuntimeError, - "currently it is only possible to register a DType " - "for scalars deriving from `np.generic`, got '%S'.", - (PyObject *)pytype); - return -1; + if (DType->legacy) { + /* + * For legacy user dtypes, discovery relied on subclassing, but + * arbitrary type objects are supported, so do nothing. + */ + return 0; } + /* + * We currently enforce that user DTypes subclass from `np.generic` + * (this should become a `np.generic` base class and may be lifted + * entirely). + */ + PyErr_Format(PyExc_RuntimeError, + "currently it is only possible to register a DType " + "for scalars deriving from `np.generic`, got '%S'.", + (PyObject *)pytype); + return -1; } /* Create the global dictionary if it does not exist */ @@ -288,7 +298,7 @@ discover_dtype_from_pyobject( Py_INCREF(DType); Py_DECREF(legacy_descr); /* TODO: Enable warning about subclass handling */ - if (0 && !((*flags) & GAVE_SUBCLASS_WARNING)) { + if ((0) && !((*flags) & GAVE_SUBCLASS_WARNING)) { if (DEPRECATE_FUTUREWARNING( "in the future NumPy will not automatically find the " "dtype for subclasses of scalars known to NumPy (i.e. " @@ -306,51 +316,6 @@ discover_dtype_from_pyobject( } -/* - * This function should probably become public API eventually. At this - * time it is implemented by falling back to `PyArray_AdaptFlexibleDType`. - * We will use `CastingImpl[from, to].adjust_descriptors(...)` to implement - * this logic. - */ -static NPY_INLINE PyArray_Descr * -cast_descriptor_to_fixed_dtype( - PyArray_Descr *descr, PyArray_DTypeMeta *fixed_DType) -{ - if (fixed_DType == NULL) { - /* Nothing to do, we only need to promote the new dtype */ - Py_INCREF(descr); - return descr; - } - - if (!fixed_DType->parametric) { - /* - * Don't actually do anything, the default is always the result - * of any cast. - */ - return fixed_DType->default_descr(fixed_DType); - } - if (PyObject_TypeCheck((PyObject *)descr, (PyTypeObject *)fixed_DType)) { - Py_INCREF(descr); - return descr; - } - /* - * TODO: When this is implemented for all dtypes, the special cases - * can be removed... - */ - if (fixed_DType->legacy && fixed_DType->parametric && - NPY_DTYPE(descr)->legacy) { - PyArray_Descr *flex_dtype = PyArray_DescrFromType(fixed_DType->type_num); - return PyArray_AdaptFlexibleDType(descr, flex_dtype); - } - - PyErr_SetString(PyExc_NotImplementedError, - "Must use casting to find the correct dtype, this is " - "not yet implemented! " - "(It should not be possible to hit this code currently!)"); - return NULL; -} - - /** * Discover the correct descriptor from a known DType class and scalar. * If the fixed DType can discover a dtype instance/descr all is fine, @@ -392,7 +357,7 @@ find_scalar_descriptor( return descr; } - Py_SETREF(descr, cast_descriptor_to_fixed_dtype(descr, fixed_DType)); + Py_SETREF(descr, PyArray_CastDescrToDType(descr, fixed_DType)); return descr; } @@ -583,7 +548,7 @@ npy_new_coercion_cache( cache = _coercion_cache_cache[_coercion_cache_num]; } else { - cache = PyObject_MALLOC(sizeof(coercion_cache_obj)); + cache = PyMem_Malloc(sizeof(coercion_cache_obj)); } if (cache == NULL) { PyErr_NoMemory(); @@ -615,7 +580,7 @@ npy_unlink_coercion_cache(coercion_cache_obj *current) _coercion_cache_num++; } else { - PyObject_FREE(current); + PyMem_Free(current); } return next; } @@ -727,8 +692,13 @@ find_descriptor_from_array( enum _dtype_discovery_flags flags = 0; *out_descr = NULL; - if (NPY_UNLIKELY(DType != NULL && DType->parametric && - PyArray_ISOBJECT(arr))) { + if (DType == NULL) { + *out_descr = PyArray_DESCR(arr); + Py_INCREF(*out_descr); + return 0; + } + + if (NPY_UNLIKELY(DType->parametric && PyArray_ISOBJECT(arr))) { /* * We have one special case, if (and only if) the input array is of * object DType and the dtype is not fixed already but parametric. @@ -777,7 +747,7 @@ find_descriptor_from_array( } Py_DECREF(iter); } - else if (DType != NULL && NPY_UNLIKELY(DType->type_num == NPY_DATETIME) && + else if (NPY_UNLIKELY(DType->type_num == NPY_DATETIME) && PyArray_ISSTRING(arr)) { /* * TODO: This branch should be deprecated IMO, the workaround is @@ -806,8 +776,7 @@ find_descriptor_from_array( * If this is not an object array figure out the dtype cast, * or simply use the returned DType. */ - *out_descr = cast_descriptor_to_fixed_dtype( - PyArray_DESCR(arr), DType); + *out_descr = PyArray_CastDescrToDType(PyArray_DESCR(arr), DType); if (*out_descr == NULL) { return -1; } @@ -1325,15 +1294,9 @@ PyArray_DiscoverDTypeAndShape( * the correct default. */ if (fixed_DType != NULL) { - if (fixed_DType->default_descr == NULL) { - Py_INCREF(fixed_DType->singleton); - *out_descr = fixed_DType->singleton; - } - else { - *out_descr = fixed_DType->default_descr(fixed_DType); - if (*out_descr == NULL) { - goto fail; - } + *out_descr = fixed_DType->default_descr(fixed_DType); + if (*out_descr == NULL) { + goto fail; } } } diff --git a/numpy/core/src/multiarray/arrayfunction_override.c b/numpy/core/src/multiarray/arrayfunction_override.c index 613fe6b3f09f..8e3bde78f714 100644 --- a/numpy/core/src/multiarray/arrayfunction_override.c +++ b/numpy/core/src/multiarray/arrayfunction_override.c @@ -388,15 +388,18 @@ array_implement_c_array_function_creation( PyObject *numpy_module = PyImport_Import(npy_ma_str_numpy); if (numpy_module == NULL) { + Py_DECREF(relevant_args); return NULL; } PyObject *public_api = PyObject_GetAttrString(numpy_module, function_name); Py_DECREF(numpy_module); if (public_api == NULL) { + Py_DECREF(relevant_args); return NULL; } if (!PyCallable_Check(public_api)) { + Py_DECREF(relevant_args); Py_DECREF(public_api); return PyErr_Format(PyExc_RuntimeError, "numpy.%s is not callable.", @@ -406,6 +409,7 @@ array_implement_c_array_function_creation( PyObject* result = array_implement_array_function_internal( public_api, relevant_args, args, kwargs); + Py_DECREF(relevant_args); Py_DECREF(public_api); return result; } diff --git a/numpy/core/src/multiarray/buffer.c b/numpy/core/src/multiarray/buffer.c index af40cdc2c911..3b3bba663aac 100644 --- a/numpy/core/src/multiarray/buffer.c +++ b/numpy/core/src/multiarray/buffer.c @@ -456,7 +456,7 @@ static PyObject *_buffer_info_cache = NULL; /* Fill in the info structure */ static _buffer_info_t* -_buffer_info_new(PyObject *obj) +_buffer_info_new(PyObject *obj, int flags) { /* * Note that the buffer info is cached as PyLongObjects making them appear @@ -504,25 +504,64 @@ _buffer_info_new(PyObject *obj) info->shape = (npy_intp *)((char *)info + sizeof(_buffer_info_t)); assert((size_t)info->shape % sizeof(npy_intp) == 0); info->strides = info->shape + PyArray_NDIM(arr); - for (k = 0; k < PyArray_NDIM(arr); ++k) { - info->shape[k] = PyArray_DIMS(arr)[k]; - info->strides[k] = PyArray_STRIDES(arr)[k]; + +#if NPY_RELAXED_STRIDES_CHECKING + /* + * When NPY_RELAXED_STRIDES_CHECKING is used, some buffer users + * may expect a contiguous buffer to have well formatted strides + * also when a dimension is 1, but we do not guarantee this + * internally. Thus, recalculate strides for contiguous arrays. + * (This is unnecessary, but has no effect in the case where + * NPY_RELAXED_STRIDES CHECKING is disabled.) + */ + int f_contiguous = (flags & PyBUF_F_CONTIGUOUS) == PyBUF_F_CONTIGUOUS; + if (PyArray_IS_C_CONTIGUOUS(arr) && !( + f_contiguous && PyArray_IS_F_CONTIGUOUS(arr))) { + Py_ssize_t sd = PyArray_ITEMSIZE(arr); + for (k = info->ndim-1; k >= 0; --k) { + info->shape[k] = PyArray_DIMS(arr)[k]; + info->strides[k] = sd; + sd *= info->shape[k]; + } + } + else if (PyArray_IS_F_CONTIGUOUS(arr)) { + Py_ssize_t sd = PyArray_ITEMSIZE(arr); + for (k = 0; k < info->ndim; ++k) { + info->shape[k] = PyArray_DIMS(arr)[k]; + info->strides[k] = sd; + sd *= info->shape[k]; + } + } + else { +#else /* NPY_RELAXED_STRIDES_CHECKING */ + /* We can always use the arrays strides directly */ + { +#endif + + for (k = 0; k < PyArray_NDIM(arr); ++k) { + info->shape[k] = PyArray_DIMS(arr)[k]; + info->strides[k] = PyArray_STRIDES(arr)[k]; + } } } Py_INCREF(descr); } /* Fill in format */ - err = _buffer_format_string(descr, &fmt, obj, NULL, NULL); - Py_DECREF(descr); - if (err != 0) { - goto fail; + if ((flags & PyBUF_FORMAT) == PyBUF_FORMAT) { + err = _buffer_format_string(descr, &fmt, obj, NULL, NULL); + Py_DECREF(descr); + if (err != 0) { + goto fail; + } + if (_append_char(&fmt, '\0') < 0) { + goto fail; + } + info->format = fmt.s; } - if (_append_char(&fmt, '\0') < 0) { - goto fail; + else { + info->format = NULL; } - info->format = fmt.s; - return info; fail: @@ -538,9 +577,10 @@ _buffer_info_cmp(_buffer_info_t *a, _buffer_info_t *b) Py_ssize_t c; int k; - c = strcmp(a->format, b->format); - if (c != 0) return c; - + if (a->format != NULL && b->format != NULL) { + c = strcmp(a->format, b->format); + if (c != 0) return c; + } c = a->ndim - b->ndim; if (c != 0) return c; @@ -565,7 +605,7 @@ _buffer_info_free(_buffer_info_t *info) /* Get buffer info from the global dictionary */ static _buffer_info_t* -_buffer_get_info(PyObject *obj) +_buffer_get_info(PyObject *obj, int flags) { PyObject *key = NULL, *item_list = NULL, *item = NULL; _buffer_info_t *info = NULL, *old_info = NULL; @@ -578,7 +618,7 @@ _buffer_get_info(PyObject *obj) } /* Compute information */ - info = _buffer_info_new(obj); + info = _buffer_info_new(obj, flags); if (info == NULL) { return NULL; } @@ -591,12 +631,44 @@ _buffer_get_info(PyObject *obj) item_list = PyDict_GetItem(_buffer_info_cache, key); if (item_list != NULL) { + Py_ssize_t item_list_length = PyList_GET_SIZE(item_list); Py_INCREF(item_list); - if (PyList_GET_SIZE(item_list) > 0) { - item = PyList_GetItem(item_list, PyList_GET_SIZE(item_list) - 1); + if (item_list_length > 0) { + item = PyList_GetItem(item_list, item_list_length - 1); old_info = (_buffer_info_t*)PyLong_AsVoidPtr(item); + if (_buffer_info_cmp(info, old_info) != 0) { + old_info = NULL; /* Can't use this one, but possibly next */ + + if (item_list_length > 1 && info->ndim > 1) { + /* + * Some arrays are C- and F-contiguous and if they have more + * than one dimension, the buffer-info may differ between + * the two due to RELAXED_STRIDES_CHECKING. + * If we export both buffers, the first stored one may be + * the one for the other contiguity, so check both. + * This is generally very unlikely in all other cases, since + * in all other cases the first one will match unless array + * metadata was modified in-place (which is discouraged). + */ + item = PyList_GetItem(item_list, item_list_length - 2); + old_info = (_buffer_info_t*)PyLong_AsVoidPtr(item); + if (_buffer_info_cmp(info, old_info) != 0) { + old_info = NULL; + } + } + } - if (_buffer_info_cmp(info, old_info) == 0) { + if (old_info != NULL) { + /* + * The two info->format are considered equal if one of them + * has no format set (meaning the format is arbitrary and can + * be modified). If the new info has a format, but we reuse + * the old one, this transfers the ownership to the old one. + */ + if (old_info->format == NULL) { + old_info->format = info->format; + info->format = NULL; + } _buffer_info_free(info); info = old_info; } @@ -706,7 +778,7 @@ array_getbuffer(PyObject *obj, Py_buffer *view, int flags) } /* Fill in information */ - info = _buffer_get_info(obj); + info = _buffer_get_info(obj, flags); if (info == NULL) { goto fail; } @@ -742,35 +814,6 @@ array_getbuffer(PyObject *obj, Py_buffer *view, int flags) } if ((flags & PyBUF_STRIDES) == PyBUF_STRIDES) { view->strides = info->strides; - -#ifdef NPY_RELAXED_STRIDES_CHECKING - /* - * If NPY_RELAXED_STRIDES_CHECKING is on, the array may be - * contiguous, but it won't look that way to Python when it - * tries to determine contiguity by looking at the strides - * (since one of the elements may be -1). In that case, just - * regenerate strides from shape. - */ - if (PyArray_CHKFLAGS(self, NPY_ARRAY_C_CONTIGUOUS) && - !((flags & PyBUF_F_CONTIGUOUS) == PyBUF_F_CONTIGUOUS)) { - Py_ssize_t sd = view->itemsize; - int i; - - for (i = view->ndim-1; i >= 0; --i) { - view->strides[i] = sd; - sd *= view->shape[i]; - } - } - else if (PyArray_CHKFLAGS(self, NPY_ARRAY_F_CONTIGUOUS)) { - Py_ssize_t sd = view->itemsize; - int i; - - for (i = 0; i < view->ndim; ++i) { - view->strides[i] = sd; - sd *= view->shape[i]; - } - } -#endif } else { view->strides = NULL; @@ -800,7 +843,7 @@ void_getbuffer(PyObject *self, Py_buffer *view, int flags) } /* Fill in information */ - info = _buffer_get_info(self); + info = _buffer_get_info(self, flags); if (info == NULL) { goto fail; } diff --git a/numpy/core/src/multiarray/common.c b/numpy/core/src/multiarray/common.c index 6af71f351ab0..841ed799db54 100644 --- a/numpy/core/src/multiarray/common.c +++ b/numpy/core/src/multiarray/common.c @@ -233,7 +233,6 @@ NPY_NO_EXPORT PyObject * convert_shape_to_string(npy_intp n, npy_intp const *vals, char *ending) { npy_intp i; - PyObject *ret, *tmp; /* * Negative dimension indicates "newaxis", which can @@ -245,14 +244,14 @@ convert_shape_to_string(npy_intp n, npy_intp const *vals, char *ending) if (i == n) { return PyUnicode_FromFormat("()%s", ending); } - else { - ret = PyUnicode_FromFormat("(%" NPY_INTP_FMT, vals[i++]); - if (ret == NULL) { - return NULL; - } - } + PyObject *ret = PyUnicode_FromFormat("%" NPY_INTP_FMT, vals[i++]); + if (ret == NULL) { + return NULL; + } for (; i < n; ++i) { + PyObject *tmp; + if (vals[i] < 0) { tmp = PyUnicode_FromString(",newaxis"); } @@ -264,19 +263,19 @@ convert_shape_to_string(npy_intp n, npy_intp const *vals, char *ending) return NULL; } - PyUString_ConcatAndDel(&ret, tmp); + Py_SETREF(ret, PyUnicode_Concat(ret, tmp)); + Py_DECREF(tmp); if (ret == NULL) { return NULL; } } if (i == 1) { - tmp = PyUnicode_FromFormat(",)%s", ending); + Py_SETREF(ret, PyUnicode_FromFormat("(%S,)%s", ret, ending)); } else { - tmp = PyUnicode_FromFormat(")%s", ending); + Py_SETREF(ret, PyUnicode_FromFormat("(%S)%s", ret, ending)); } - PyUString_ConcatAndDel(&ret, tmp); return ret; } diff --git a/numpy/core/src/multiarray/convert_datatype.c b/numpy/core/src/multiarray/convert_datatype.c index d9121707b21a..f700bdc990df 100644 --- a/numpy/core/src/multiarray/convert_datatype.c +++ b/numpy/core/src/multiarray/convert_datatype.c @@ -1019,7 +1019,7 @@ promote_types(PyArray_Descr *type1, PyArray_Descr *type2, * Returns a new reference to type if it is already NBO, otherwise * returns a copy converted to NBO. */ -static PyArray_Descr * +NPY_NO_EXPORT PyArray_Descr * ensure_dtype_nbo(PyArray_Descr *type) { if (PyArray_ISNBO(type->byteorder)) { @@ -1031,327 +1031,148 @@ ensure_dtype_nbo(PyArray_Descr *type) } } -/*NUMPY_API - * Produces the smallest size and lowest kind type to which both - * input types can be cast. + +/** + * This function should possibly become public API eventually. At this + * time it is implemented by falling back to `PyArray_AdaptFlexibleDType`. + * We will use `CastingImpl[from, to].adjust_descriptors(...)` to implement + * this logic. + * Before that, the API needs to be reviewed though. + * + * WARNING: This function currently does not guarantee that `descr` can + * actually be cast to the given DType. + * + * @param descr The dtype instance to adapt "cast" + * @param given_DType The DType class for which we wish to find an instance able + * to represent `descr`. + * @returns Instance of `given_DType`. If `given_DType` is parametric the + * descr may be adapted to hold it. */ NPY_NO_EXPORT PyArray_Descr * -PyArray_PromoteTypes(PyArray_Descr *type1, PyArray_Descr *type2) +PyArray_CastDescrToDType(PyArray_Descr *descr, PyArray_DTypeMeta *given_DType) { - int type_num1, type_num2, ret_type_num; - - /* - * Fast path for identical dtypes. - * - * Non-native-byte-order types are converted to native ones below, so we - * can't quit early. - */ - if (type1 == type2 && PyArray_ISNBO(type1->byteorder)) { - Py_INCREF(type1); - return type1; + if (NPY_DTYPE(descr) == given_DType) { + Py_INCREF(descr); + return descr; } - - type_num1 = type1->type_num; - type_num2 = type2->type_num; - - /* If they're built-in types, use the promotion table */ - if (type_num1 < NPY_NTYPES && type_num2 < NPY_NTYPES) { - ret_type_num = _npy_type_promotion_table[type_num1][type_num2]; + if (!given_DType->parametric) { /* - * The table doesn't handle string/unicode/void/datetime/timedelta, - * so check the result + * Don't actually do anything, the default is always the result + * of any cast. */ - if (ret_type_num >= 0) { - return PyArray_DescrFromType(ret_type_num); - } + return given_DType->default_descr(given_DType); + } + if (PyObject_TypeCheck((PyObject *)descr, (PyTypeObject *)given_DType)) { + Py_INCREF(descr); + return descr; } - /* If one or both are user defined, calculate it */ - else { - int skind1 = NPY_NOSCALAR, skind2 = NPY_NOSCALAR, skind; - if (PyArray_CanCastTo(type2, type1)) { - /* Promoted types are always native byte order */ - return ensure_dtype_nbo(type1); - } - else if (PyArray_CanCastTo(type1, type2)) { - /* Promoted types are always native byte order */ - return ensure_dtype_nbo(type2); - } + if (!given_DType->legacy) { + PyErr_SetString(PyExc_NotImplementedError, + "Must use casting to find the correct DType for a parametric " + "user DType. This is not yet implemented (this error should be " + "unreachable)."); + return NULL; + } - /* Convert the 'kind' char into a scalar kind */ - switch (type1->kind) { - case 'b': - skind1 = NPY_BOOL_SCALAR; - break; - case 'u': - skind1 = NPY_INTPOS_SCALAR; - break; - case 'i': - skind1 = NPY_INTNEG_SCALAR; - break; - case 'f': - skind1 = NPY_FLOAT_SCALAR; - break; - case 'c': - skind1 = NPY_COMPLEX_SCALAR; - break; - } - switch (type2->kind) { - case 'b': - skind2 = NPY_BOOL_SCALAR; - break; - case 'u': - skind2 = NPY_INTPOS_SCALAR; - break; - case 'i': - skind2 = NPY_INTNEG_SCALAR; - break; - case 'f': - skind2 = NPY_FLOAT_SCALAR; - break; - case 'c': - skind2 = NPY_COMPLEX_SCALAR; - break; - } + PyArray_Descr *flex_dtype = PyArray_DescrNew(given_DType->singleton); + return PyArray_AdaptFlexibleDType(descr, flex_dtype); +} - /* If both are scalars, there may be a promotion possible */ - if (skind1 != NPY_NOSCALAR && skind2 != NPY_NOSCALAR) { - /* Start with the larger scalar kind */ - skind = (skind1 > skind2) ? skind1 : skind2; - ret_type_num = _npy_smallest_type_of_kind_table[skind]; +/** + * This function defines the common DType operator. + * + * Note that the common DType will not be "object" (unless one of the dtypes + * is object), even though object can technically represent all values + * correctly. + * + * TODO: Before exposure, we should review the return value (e.g. no error + * when no common DType is found). + * + * @param dtype1 DType class to find the common type for. + * @param dtype2 Second DType class. + * @return The common DType or NULL with an error set + */ +NPY_NO_EXPORT PyArray_DTypeMeta * +PyArray_CommonDType(PyArray_DTypeMeta *dtype1, PyArray_DTypeMeta *dtype2) +{ + if (dtype1 == dtype2) { + Py_INCREF(dtype1); + return dtype1; + } - for (;;) { + PyArray_DTypeMeta *common_dtype; - /* If there is no larger type of this kind, try a larger kind */ - if (ret_type_num < 0) { - ++skind; - /* Use -1 to signal no promoted type found */ - if (skind < NPY_NSCALARKINDS) { - ret_type_num = _npy_smallest_type_of_kind_table[skind]; - } - else { - break; - } - } + common_dtype = dtype1->common_dtype(dtype1, dtype2); + if (common_dtype == (PyArray_DTypeMeta *)Py_NotImplemented) { + Py_DECREF(common_dtype); + common_dtype = dtype2->common_dtype(dtype2, dtype1); + } + if (common_dtype == NULL) { + return NULL; + } + if (common_dtype == (PyArray_DTypeMeta *)Py_NotImplemented) { + Py_DECREF(Py_NotImplemented); + PyErr_Format(PyExc_TypeError, + "The DTypes %S and %S do not have a common DType. " + "For example they cannot be stored in a single array unless " + "the dtype is `object`.", dtype1, dtype2); + return NULL; + } + return common_dtype; +} - /* If we found a type to which we can promote both, done! */ - if (PyArray_CanCastSafely(type_num1, ret_type_num) && - PyArray_CanCastSafely(type_num2, ret_type_num)) { - return PyArray_DescrFromType(ret_type_num); - } - /* Try the next larger type of this kind */ - ret_type_num = _npy_next_larger_type_table[ret_type_num]; - } +/*NUMPY_API + * Produces the smallest size and lowest kind type to which both + * input types can be cast. + */ +NPY_NO_EXPORT PyArray_Descr * +PyArray_PromoteTypes(PyArray_Descr *type1, PyArray_Descr *type2) +{ + PyArray_DTypeMeta *common_dtype; + PyArray_Descr *res; - } + /* Fast path for identical inputs (NOTE: This path preserves metadata!) */ + if (type1 == type2 && PyArray_ISNBO(type1->byteorder)) { + Py_INCREF(type1); + return type1; + } - PyErr_SetString(PyExc_TypeError, - "invalid type promotion with custom data type"); + common_dtype = PyArray_CommonDType(NPY_DTYPE(type1), NPY_DTYPE(type2)); + if (common_dtype == NULL) { return NULL; } - switch (type_num1) { - /* BOOL can convert to anything except datetime/void */ - case NPY_BOOL: - if (type_num2 == NPY_STRING || type_num2 == NPY_UNICODE) { - int char_size = 1; - if (type_num2 == NPY_UNICODE) { - char_size = 4; - } - if (type2->elsize < 5 * char_size) { - PyArray_Descr *ret = NULL; - PyArray_Descr *temp = PyArray_DescrNew(type2); - ret = ensure_dtype_nbo(temp); - ret->elsize = 5 * char_size; - Py_DECREF(temp); - return ret; - } - return ensure_dtype_nbo(type2); - } - else if (type_num2 != NPY_DATETIME && type_num2 != NPY_VOID) { - return ensure_dtype_nbo(type2); - } - break; - /* For strings and unicodes, take the larger size */ - case NPY_STRING: - if (type_num2 == NPY_STRING) { - if (type1->elsize > type2->elsize) { - return ensure_dtype_nbo(type1); - } - else { - return ensure_dtype_nbo(type2); - } - } - else if (type_num2 == NPY_UNICODE) { - if (type2->elsize >= type1->elsize * 4) { - return ensure_dtype_nbo(type2); - } - else { - PyArray_Descr *d = PyArray_DescrNewFromType(NPY_UNICODE); - if (d == NULL) { - return NULL; - } - d->elsize = type1->elsize * 4; - return d; - } - } - /* Allow NUMBER -> STRING */ - else if (PyTypeNum_ISNUMBER(type_num2)) { - PyArray_Descr *ret = NULL; - PyArray_Descr *temp = PyArray_DescrNew(type1); - PyDataType_MAKEUNSIZED(temp); - - temp = PyArray_AdaptFlexibleDType(type2, temp); - if (temp == NULL) { - return NULL; - } - if (temp->elsize > type1->elsize) { - ret = ensure_dtype_nbo(temp); - } - else { - ret = ensure_dtype_nbo(type1); - } - Py_DECREF(temp); - return ret; - } - break; - case NPY_UNICODE: - if (type_num2 == NPY_UNICODE) { - if (type1->elsize > type2->elsize) { - return ensure_dtype_nbo(type1); - } - else { - return ensure_dtype_nbo(type2); - } - } - else if (type_num2 == NPY_STRING) { - if (type1->elsize >= type2->elsize * 4) { - return ensure_dtype_nbo(type1); - } - else { - PyArray_Descr *d = PyArray_DescrNewFromType(NPY_UNICODE); - if (d == NULL) { - return NULL; - } - d->elsize = type2->elsize * 4; - return d; - } - } - /* Allow NUMBER -> UNICODE */ - else if (PyTypeNum_ISNUMBER(type_num2)) { - PyArray_Descr *ret = NULL; - PyArray_Descr *temp = PyArray_DescrNew(type1); - PyDataType_MAKEUNSIZED(temp); - temp = PyArray_AdaptFlexibleDType(type2, temp); - if (temp == NULL) { - return NULL; - } - if (temp->elsize > type1->elsize) { - ret = ensure_dtype_nbo(temp); - } - else { - ret = ensure_dtype_nbo(type1); - } - Py_DECREF(temp); - return ret; - } - break; - case NPY_DATETIME: - case NPY_TIMEDELTA: - if (type_num2 == NPY_DATETIME || type_num2 == NPY_TIMEDELTA) { - return datetime_type_promotion(type1, type2); - } - break; + if (!common_dtype->parametric) { + res = common_dtype->default_descr(common_dtype); + Py_DECREF(common_dtype); + return res; } - switch (type_num2) { - /* BOOL can convert to almost anything */ - case NPY_BOOL: - if (type_num2 == NPY_STRING || type_num2 == NPY_UNICODE) { - int char_size = 1; - if (type_num2 == NPY_UNICODE) { - char_size = 4; - } - if (type2->elsize < 5 * char_size) { - PyArray_Descr *ret = NULL; - PyArray_Descr *temp = PyArray_DescrNew(type2); - ret = ensure_dtype_nbo(temp); - ret->elsize = 5 * char_size; - Py_DECREF(temp); - return ret; - } - return ensure_dtype_nbo(type2); - } - else if (type_num1 != NPY_DATETIME && type_num1 != NPY_TIMEDELTA && - type_num1 != NPY_VOID) { - return ensure_dtype_nbo(type1); - } - break; - case NPY_STRING: - /* Allow NUMBER -> STRING */ - if (PyTypeNum_ISNUMBER(type_num1)) { - PyArray_Descr *ret = NULL; - PyArray_Descr *temp = PyArray_DescrNew(type2); - PyDataType_MAKEUNSIZED(temp); - temp = PyArray_AdaptFlexibleDType(type1, temp); - if (temp == NULL) { - return NULL; - } - if (temp->elsize > type2->elsize) { - ret = ensure_dtype_nbo(temp); - } - else { - ret = ensure_dtype_nbo(type2); - } - Py_DECREF(temp); - return ret; - } - break; - case NPY_UNICODE: - /* Allow NUMBER -> UNICODE */ - if (PyTypeNum_ISNUMBER(type_num1)) { - PyArray_Descr *ret = NULL; - PyArray_Descr *temp = PyArray_DescrNew(type2); - PyDataType_MAKEUNSIZED(temp); - temp = PyArray_AdaptFlexibleDType(type1, temp); - if (temp == NULL) { - return NULL; - } - if (temp->elsize > type2->elsize) { - ret = ensure_dtype_nbo(temp); - } - else { - ret = ensure_dtype_nbo(type2); - } - Py_DECREF(temp); - return ret; - } - break; - case NPY_TIMEDELTA: - if (PyTypeNum_ISSIGNED(type_num1)) { - return ensure_dtype_nbo(type2); - } - break; + /* Cast the input types to the common DType if necessary */ + type1 = PyArray_CastDescrToDType(type1, common_dtype); + if (type1 == NULL) { + Py_DECREF(common_dtype); + return NULL; } - - /* For types equivalent up to endianness, can return either */ - if (PyArray_CanCastTypeTo(type1, type2, NPY_EQUIV_CASTING)) { - return ensure_dtype_nbo(type1); + type2 = PyArray_CastDescrToDType(type2, common_dtype); + if (type2 == NULL) { + Py_DECREF(type1); + Py_DECREF(common_dtype); + return NULL; } - /* TODO: Also combine fields, subarrays, strings, etc */ - /* - printf("invalid type promotion: "); - PyObject_Print(type1, stdout, 0); - printf(" "); - PyObject_Print(type2, stdout, 0); - printf("\n"); - */ - PyErr_SetString(PyExc_TypeError, "invalid type promotion"); - return NULL; + * And find the common instance of the two inputs + * NOTE: Common instance preserves metadata (normally and of one input) + */ + res = common_dtype->common_instance(type1, type2); + Py_DECREF(type1); + Py_DECREF(type2); + Py_DECREF(common_dtype); + return res; } /* diff --git a/numpy/core/src/multiarray/convert_datatype.h b/numpy/core/src/multiarray/convert_datatype.h index 9b7f39db2e33..a2b36b497fbe 100644 --- a/numpy/core/src/multiarray/convert_datatype.h +++ b/numpy/core/src/multiarray/convert_datatype.h @@ -10,6 +10,9 @@ PyArray_ObjectType(PyObject *op, int minimum_type); NPY_NO_EXPORT PyArrayObject ** PyArray_ConvertToCommonType(PyObject *op, int *retn); +NPY_NO_EXPORT PyArray_DTypeMeta * +PyArray_CommonDType(PyArray_DTypeMeta *dtype1, PyArray_DTypeMeta *dtype2); + NPY_NO_EXPORT int PyArray_ValidType(int type); @@ -18,6 +21,9 @@ NPY_NO_EXPORT npy_bool can_cast_scalar_to(PyArray_Descr *scal_type, char *scal_data, PyArray_Descr *to, NPY_CASTING casting); +NPY_NO_EXPORT PyArray_Descr * +ensure_dtype_nbo(PyArray_Descr *type); + NPY_NO_EXPORT int should_use_min_scalar(npy_intp narrs, PyArrayObject **arr, npy_intp ndtypes, PyArray_Descr **dtypes); @@ -49,4 +55,7 @@ npy_set_invalid_cast_error( NPY_NO_EXPORT PyArray_Descr * PyArray_AdaptFlexibleDType(PyArray_Descr *data_dtype, PyArray_Descr *flex_dtype); +NPY_NO_EXPORT PyArray_Descr * +PyArray_CastDescrToDType(PyArray_Descr *descr, PyArray_DTypeMeta *given_DType); + #endif diff --git a/numpy/core/src/multiarray/ctors.c b/numpy/core/src/multiarray/ctors.c index 956dfd3bbf9e..ff262369bff6 100644 --- a/numpy/core/src/multiarray/ctors.c +++ b/numpy/core/src/multiarray/ctors.c @@ -610,6 +610,7 @@ PyArray_AssignFromCache(PyArrayObject *self, coercion_cache_obj *cache) { PyErr_SetString(PyExc_RuntimeError, "Inconsistent object during array creation? " "Content of sequences changed (cache not consumed)."); + npy_free_coercion_cache(cache); return -1; } return 0; @@ -1367,6 +1368,160 @@ PyArray_GetArrayParamsFromObject(PyObject *NPY_UNUSED(op), } +/* + * This function is a legacy implementation to retain subarray dtype + * behaviour in array coercion. The behaviour here makes sense if tuples + * of matching dimensionality are being coerced. Due to the difficulty + * that the result is ill-defined for lists of array-likes, this is deprecated. + * + * WARNING: Do not use this function, it exists purely to support a deprecated + * code path. + */ +static int +setArrayFromSequence(PyArrayObject *a, PyObject *s, + int dim, PyArrayObject * dst) +{ + Py_ssize_t i, slen; + int res = -1; + + /* first recursion, view equal destination */ + if (dst == NULL) + dst = a; + + /* + * This code is to ensure that the sequence access below will + * return a lower-dimensional sequence. + */ + + /* INCREF on entry DECREF on exit */ + Py_INCREF(s); + + PyObject *seq = NULL; + + if (PyArray_Check(s)) { + if (!(PyArray_CheckExact(s))) { + /* + * make sure a base-class array is used so that the dimensionality + * reduction assumption is correct. + */ + /* This will DECREF(s) if replaced */ + s = PyArray_EnsureArray(s); + if (s == NULL) { + goto fail; + } + } + + /* dst points to correct array subsection */ + if (PyArray_CopyInto(dst, (PyArrayObject *)s) < 0) { + goto fail; + } + + Py_DECREF(s); + return 0; + } + + if (dim > PyArray_NDIM(a)) { + PyErr_Format(PyExc_ValueError, + "setArrayFromSequence: sequence/array dimensions mismatch."); + goto fail; + } + + /* Try __array__ before using s as a sequence */ + PyObject *tmp = _array_from_array_like(s, NULL, 0, NULL); + if (tmp == NULL) { + goto fail; + } + else if (tmp == Py_NotImplemented) { + Py_DECREF(tmp); + } + else { + int r = PyArray_CopyInto(dst, (PyArrayObject *)tmp); + Py_DECREF(tmp); + if (r < 0) { + goto fail; + } + Py_DECREF(s); + return 0; + } + + seq = PySequence_Fast(s, "Could not convert object to sequence"); + if (seq == NULL) { + goto fail; + } + slen = PySequence_Fast_GET_SIZE(seq); + + /* + * Either the dimensions match, or the sequence has length 1 and can + * be broadcast to the destination. + */ + if (slen != PyArray_DIMS(a)[dim] && slen != 1) { + PyErr_Format(PyExc_ValueError, + "cannot copy sequence with size %zd to array axis " + "with dimension %" NPY_INTP_FMT, slen, PyArray_DIMS(a)[dim]); + goto fail; + } + + /* Broadcast the one element from the sequence to all the outputs */ + if (slen == 1) { + PyObject *o = PySequence_Fast_GET_ITEM(seq, 0); + npy_intp alen = PyArray_DIM(a, dim); + + for (i = 0; i < alen; i++) { + if ((PyArray_NDIM(a) - dim) > 1) { + PyArrayObject * tmp = + (PyArrayObject *)array_item_asarray(dst, i); + if (tmp == NULL) { + goto fail; + } + + res = setArrayFromSequence(a, o, dim+1, tmp); + Py_DECREF(tmp); + } + else { + char * b = (PyArray_BYTES(dst) + i * PyArray_STRIDES(dst)[0]); + res = PyArray_SETITEM(dst, b, o); + } + if (res < 0) { + goto fail; + } + } + } + /* Copy element by element */ + else { + for (i = 0; i < slen; i++) { + PyObject * o = PySequence_Fast_GET_ITEM(seq, i); + if ((PyArray_NDIM(a) - dim) > 1) { + PyArrayObject * tmp = + (PyArrayObject *)array_item_asarray(dst, i); + if (tmp == NULL) { + goto fail; + } + + res = setArrayFromSequence(a, o, dim+1, tmp); + Py_DECREF(tmp); + } + else { + char * b = (PyArray_BYTES(dst) + i * PyArray_STRIDES(dst)[0]); + res = PyArray_SETITEM(dst, b, o); + } + if (res < 0) { + goto fail; + } + } + } + + Py_DECREF(seq); + Py_DECREF(s); + return 0; + + fail: + Py_XDECREF(seq); + Py_DECREF(s); + return res; +} + + + /*NUMPY_API * Does not check for NPY_ARRAY_ENSURECOPY and NPY_ARRAY_NOTSWAPPED in flags * Steals a reference to newtype --- which can be NULL @@ -1407,6 +1562,71 @@ PyArray_FromAny(PyObject *op, PyArray_Descr *newtype, int min_depth, if (ndim < 0) { return NULL; } + + if (NPY_UNLIKELY(fixed_descriptor != NULL && PyDataType_HASSUBARRAY(dtype))) { + /* + * When a subarray dtype was passed in, its dimensions are appended + * to the array dimension (causing a dimension mismatch). + * There is a problem with that, because if we coerce from non-arrays + * we do this correctly by element (as defined by tuples), but for + * arrays we first append the dimensions and then assign to the base + * dtype and then assign which causes the problem. + * + * Thus, we check if there is an array included, in that case we + * give a FutureWarning. + * When the warning is removed, PyArray_Pack will have to ensure + * that that it does not append the dimensions when creating the + * subarrays to assign `arr[0] = obj[0]`. + */ + int includes_array = 0; + if (cache != NULL) { + /* This is not ideal, but it is a pretty special case */ + coercion_cache_obj *next = cache; + while (next != NULL) { + if (!next->sequence) { + includes_array = 1; + break; + } + next = next->next; + } + } + if (includes_array) { + npy_free_coercion_cache(cache); + + ret = (PyArrayObject *) PyArray_NewFromDescr( + &PyArray_Type, dtype, ndim, dims, NULL, NULL, + flags & NPY_ARRAY_F_CONTIGUOUS, NULL); + if (ret == NULL) { + return NULL; + } + assert(PyArray_NDIM(ret) != ndim); + + /* NumPy 1.20, 2020-10-01 */ + if (DEPRECATE_FUTUREWARNING( + "creating an array with a subarray dtype will behave " + "differently when the `np.array()` (or `asarray`, etc.) " + "call includes an array or array object.\n" + "If you are converting a single array or a list of arrays," + "you can opt-in to the future behaviour using:\n" + " np.array(arr, dtype=np.dtype(['f', dtype]))['f']\n" + " np.array([arr1, arr2], dtype=np.dtype(['f', dtype]))['f']\n" + "\n" + "By including a new field and indexing it after the " + "conversion.\n" + "This may lead to a different result or to current failures " + "succeeding. (FutureWarning since NumPy 1.20)") < 0) { + Py_DECREF(ret); + return NULL; + } + + if (setArrayFromSequence(ret, op, 0, NULL) < 0) { + Py_DECREF(ret); + return NULL; + } + return (PyObject *)ret; + } + } + if (dtype == NULL) { dtype = PyArray_DescrFromType(NPY_DEFAULT_TYPE); } @@ -1460,6 +1680,31 @@ PyArray_FromAny(PyObject *op, PyArray_Descr *newtype, int min_depth, ((PyVoidScalarObject *)op)->flags, NULL, op); } + else if (cache == 0 && newtype != NULL && + PyDataType_ISSIGNED(newtype) && PyArray_IsScalar(op, Generic)) { + assert(ndim == 0); + /* + * This is an (possible) inconsistency where: + * + * np.array(np.float64(np.nan), dtype=np.int64) + * + * behaves differently from: + * + * np.array([np.float64(np.nan)], dtype=np.int64) + * arr1d_int64[0] = np.float64(np.nan) + * np.array(np.array(np.nan), dtype=np.int64) + * + * by not raising an error instead of using typical casting. + * The error is desirable, but to always error seems like a + * larger change to be considered at some other time and it is + * undesirable that 0-D arrays behave differently from scalars. + * This retains the behaviour, largely due to issues in pandas + * which relied on a try/except (although hopefully that will + * have a better solution at some point): + * https://github.com/pandas-dev/pandas/issues/35481 + */ + return PyArray_FromScalar(op, dtype); + } /* There was no array (or array-like) passed in directly. */ if ((flags & NPY_ARRAY_WRITEBACKIFCOPY) || @@ -1467,28 +1712,57 @@ PyArray_FromAny(PyObject *op, PyArray_Descr *newtype, int min_depth, PyErr_SetString(PyExc_TypeError, "WRITEBACKIFCOPY used for non-array input."); Py_DECREF(dtype); + npy_free_coercion_cache(cache); return NULL; } /* Create a new array and copy the data */ + Py_INCREF(dtype); /* hold on in case of a subarray that is replaced */ ret = (PyArrayObject *)PyArray_NewFromDescr( &PyArray_Type, dtype, ndim, dims, NULL, NULL, flags&NPY_ARRAY_F_CONTIGUOUS, NULL); if (ret == NULL) { + npy_free_coercion_cache(cache); + Py_DECREF(dtype); return NULL; } + if (ndim == PyArray_NDIM(ret)) { + /* + * Appending of dimensions did not occur, so use the actual dtype + * below. This is relevant for S0 or U0 which can be replaced with + * S1 or U1, although that should likely change. + */ + Py_SETREF(dtype, PyArray_DESCR(ret)); + Py_INCREF(dtype); + } + if (cache == NULL) { /* This is a single item. Set it directly. */ assert(ndim == 0); - if (PyArray_Pack(PyArray_DESCR(ret), PyArray_DATA(ret), op) < 0) { + + if (PyArray_Pack(dtype, PyArray_BYTES(ret), op) < 0) { + Py_DECREF(dtype); Py_DECREF(ret); return NULL; } + Py_DECREF(dtype); return (PyObject *)ret; } assert(ndim != 0); assert(op == cache->converted_obj); - if (PyArray_AssignFromCache(ret, cache) < 0) { + + /* Decrease the number of dimensions to the detected ones */ + int out_ndim = PyArray_NDIM(ret); + PyArray_Descr *out_descr = PyArray_DESCR(ret); + ((PyArrayObject_fields *)ret)->nd = ndim; + ((PyArrayObject_fields *)ret)->descr = dtype; + + int success = PyArray_AssignFromCache(ret, cache); + + ((PyArrayObject_fields *)ret)->nd = out_ndim; + ((PyArrayObject_fields *)ret)->descr = out_descr; + Py_DECREF(dtype); + if (success < 0) { Py_DECREF(ret); return NULL; } diff --git a/numpy/core/src/multiarray/datetime.c b/numpy/core/src/multiarray/datetime.c index f2225809a904..4afc45fb63e8 100644 --- a/numpy/core/src/multiarray/datetime.c +++ b/numpy/core/src/multiarray/datetime.c @@ -1434,18 +1434,20 @@ raise_if_datetime64_metadata_cast_error(char *object_type, return 0; } else { - PyObject *errmsg; - errmsg = PyUnicode_FromFormat("Cannot cast %s " - "from metadata ", object_type); - errmsg = append_metastr_to_string(src_meta, 0, errmsg); - PyUString_ConcatAndDel(&errmsg, - PyUnicode_FromString(" to ")); - errmsg = append_metastr_to_string(dst_meta, 0, errmsg); - PyUString_ConcatAndDel(&errmsg, - PyUnicode_FromFormat(" according to the rule %s", - npy_casting_to_string(casting))); - PyErr_SetObject(PyExc_TypeError, errmsg); - Py_DECREF(errmsg); + PyObject *src = metastr_to_unicode(src_meta, 0); + if (src == NULL) { + return -1; + } + PyObject *dst = metastr_to_unicode(dst_meta, 0); + if (dst == NULL) { + Py_DECREF(src); + return -1; + } + PyErr_Format(PyExc_TypeError, + "Cannot cast %s from metadata %S to %S according to the rule %s", + object_type, src, dst, npy_casting_to_string(casting)); + Py_DECREF(src); + Py_DECREF(dst); return -1; } } @@ -1466,18 +1468,20 @@ raise_if_timedelta64_metadata_cast_error(char *object_type, return 0; } else { - PyObject *errmsg; - errmsg = PyUnicode_FromFormat("Cannot cast %s " - "from metadata ", object_type); - errmsg = append_metastr_to_string(src_meta, 0, errmsg); - PyUString_ConcatAndDel(&errmsg, - PyUnicode_FromString(" to ")); - errmsg = append_metastr_to_string(dst_meta, 0, errmsg); - PyUString_ConcatAndDel(&errmsg, - PyUnicode_FromFormat(" according to the rule %s", - npy_casting_to_string(casting))); - PyErr_SetObject(PyExc_TypeError, errmsg); - Py_DECREF(errmsg); + PyObject *src = metastr_to_unicode(src_meta, 0); + if (src == NULL) { + return -1; + } + PyObject *dst = metastr_to_unicode(dst_meta, 0); + if (dst == NULL) { + Py_DECREF(src); + return -1; + } + PyErr_Format(PyExc_TypeError, + "Cannot cast %s from metadata %S to %S according to the rule %s", + object_type, src, dst, npy_casting_to_string(casting)); + Py_DECREF(src); + Py_DECREF(dst); return -1; } } @@ -1600,32 +1604,38 @@ compute_datetime_metadata_greatest_common_divisor( return 0; incompatible_units: { - PyObject *errmsg; - errmsg = PyUnicode_FromString("Cannot get " - "a common metadata divisor for " - "NumPy datetime metadata "); - errmsg = append_metastr_to_string(meta1, 0, errmsg); - PyUString_ConcatAndDel(&errmsg, - PyUnicode_FromString(" and ")); - errmsg = append_metastr_to_string(meta2, 0, errmsg); - PyUString_ConcatAndDel(&errmsg, - PyUnicode_FromString(" because they have " - "incompatible nonlinear base time units")); - PyErr_SetObject(PyExc_TypeError, errmsg); - Py_DECREF(errmsg); + PyObject *umeta1 = metastr_to_unicode(meta1, 0); + if (umeta1 == NULL) { + return -1; + } + PyObject *umeta2 = metastr_to_unicode(meta2, 0); + if (umeta2 == NULL) { + Py_DECREF(umeta1); + return -1; + } + PyErr_Format(PyExc_TypeError, + "Cannot get a common metadata divisor for Numpy datatime " + "metadata %S and %S because they have incompatible nonlinear " + "base time units.", umeta1, umeta2); + Py_DECREF(umeta1); + Py_DECREF(umeta2); return -1; } units_overflow: { - PyObject *errmsg; - errmsg = PyUnicode_FromString("Integer overflow " - "getting a common metadata divisor for " - "NumPy datetime metadata "); - errmsg = append_metastr_to_string(meta1, 0, errmsg); - PyUString_ConcatAndDel(&errmsg, - PyUnicode_FromString(" and ")); - errmsg = append_metastr_to_string(meta2, 0, errmsg); - PyErr_SetObject(PyExc_OverflowError, errmsg); - Py_DECREF(errmsg); + PyObject *umeta1 = metastr_to_unicode(meta1, 0); + if (umeta1 == NULL) { + return -1; + } + PyObject *umeta2 = metastr_to_unicode(meta2, 0); + if (umeta2 == NULL) { + Py_DECREF(umeta1); + return -1; + } + PyErr_Format(PyExc_OverflowError, + "Integer overflow getting a common metadata divisor for " + "NumPy datetime metadata %S and %S.", umeta1, umeta2); + Py_DECREF(umeta1); + Py_DECREF(umeta2); return -1; } } @@ -1950,35 +1960,27 @@ convert_pyobject_to_datetime_metadata(PyObject *obj, } /* - * 'ret' is a PyUString containing the datetime string, and this - * function appends the metadata string to it. + * Return the datetime metadata as a Unicode object. + * + * Returns new reference, NULL on error. * * If 'skip_brackets' is true, skips the '[]'. * - * This function steals the reference 'ret' */ NPY_NO_EXPORT PyObject * -append_metastr_to_string(PyArray_DatetimeMetaData *meta, - int skip_brackets, - PyObject *ret) +metastr_to_unicode(PyArray_DatetimeMetaData *meta, int skip_brackets) { - PyObject *res; int num; char const *basestr; - if (ret == NULL) { - return NULL; - } - if (meta->base == NPY_FR_GENERIC) { /* Without brackets, give a string "generic" */ if (skip_brackets) { - PyUString_ConcatAndDel(&ret, PyUnicode_FromString("generic")); - return ret; + return PyUnicode_FromString("generic"); } - /* But with brackets, append nothing */ + /* But with brackets, return nothing */ else { - return ret; + return PyUnicode_FromString(""); } } @@ -1994,25 +1996,23 @@ append_metastr_to_string(PyArray_DatetimeMetaData *meta, if (num == 1) { if (skip_brackets) { - res = PyUnicode_FromFormat("%s", basestr); + return PyUnicode_FromFormat("%s", basestr); } else { - res = PyUnicode_FromFormat("[%s]", basestr); + return PyUnicode_FromFormat("[%s]", basestr); } } else { if (skip_brackets) { - res = PyUnicode_FromFormat("%d%s", num, basestr); + return PyUnicode_FromFormat("%d%s", num, basestr); } else { - res = PyUnicode_FromFormat("[%d%s]", num, basestr); + return PyUnicode_FromFormat("[%d%s]", num, basestr); } } - - PyUString_ConcatAndDel(&ret, res); - return ret; } + /* * Adjusts a datetimestruct based on a seconds offset. Assumes * the current values are valid. diff --git a/numpy/core/src/multiarray/datetime_busday.c b/numpy/core/src/multiarray/datetime_busday.c index d3cce8a370da..2cf1575515bc 100644 --- a/numpy/core/src/multiarray/datetime_busday.c +++ b/numpy/core/src/multiarray/datetime_busday.c @@ -834,24 +834,23 @@ static int PyArray_BusDayRollConverter(PyObject *roll_in, NPY_BUSDAY_ROLL *roll) { PyObject *obj = roll_in; - char *str; - Py_ssize_t len; - /* Make obj into an ASCII string */ - Py_INCREF(obj); - if (PyUnicode_Check(obj)) { - /* accept unicode input */ - PyObject *obj_str; - obj_str = PyUnicode_AsASCIIString(obj); + /* Make obj into an UTF8 string */ + if (PyBytes_Check(obj)) { + /* accept bytes input */ + PyObject *obj_str = PyUnicode_FromEncodedObject(obj, NULL, NULL); if (obj_str == NULL) { - Py_DECREF(obj); return 0; } - Py_DECREF(obj); obj = obj_str; } + else { + Py_INCREF(obj); + } - if (PyBytes_AsStringAndSize(obj, &str, &len) < 0) { + Py_ssize_t len; + char const *str = PyUnicode_AsUTF8AndSize(obj, &len); + if (str == NULL) { Py_DECREF(obj); return 0; } diff --git a/numpy/core/src/multiarray/datetime_busdaycal.c b/numpy/core/src/multiarray/datetime_busdaycal.c index 2374eaa63453..d48141d4cb7d 100644 --- a/numpy/core/src/multiarray/datetime_busdaycal.c +++ b/numpy/core/src/multiarray/datetime_busdaycal.c @@ -30,33 +30,31 @@ PyArray_WeekMaskConverter(PyObject *weekmask_in, npy_bool *weekmask) { PyObject *obj = weekmask_in; - /* Make obj into an ASCII string if it is UNICODE */ - Py_INCREF(obj); - if (PyUnicode_Check(obj)) { - /* accept unicode input */ - PyObject *obj_str; - obj_str = PyUnicode_AsASCIIString(obj); + /* Make obj into an UTF8 string */ + if (PyBytes_Check(obj)) { + /* accept bytes input */ + PyObject *obj_str = PyUnicode_FromEncodedObject(obj, NULL, NULL); if (obj_str == NULL) { - Py_DECREF(obj); return 0; } - Py_DECREF(obj); obj = obj_str; } + else { + Py_INCREF(obj); + } - if (PyBytes_Check(obj)) { - char *str; - Py_ssize_t len; - int i; - if (PyBytes_AsStringAndSize(obj, &str, &len) < 0) { + if (PyUnicode_Check(obj)) { + Py_ssize_t len; + char const *str = PyUnicode_AsUTF8AndSize(obj, &len); + if (str == NULL) { Py_DECREF(obj); return 0; } /* Length 7 is a string like "1111100" */ if (len == 7) { - for (i = 0; i < 7; ++i) { + for (int i = 0; i < 7; ++i) { switch(str[i]) { case '0': weekmask[i] = 0; @@ -75,7 +73,7 @@ PyArray_WeekMaskConverter(PyObject *weekmask_in, npy_bool *weekmask) general_weekmask_string: /* a string like "SatSun" or "Mon Tue Wed" */ memset(weekmask, 0, 7); - for (i = 0; i < len; i += 3) { + for (Py_ssize_t i = 0; i < len; i += 3) { while (isspace(str[i])) ++i; diff --git a/numpy/core/src/multiarray/datetime_strings.c b/numpy/core/src/multiarray/datetime_strings.c index f847c7ea8e9c..360868568478 100644 --- a/numpy/core/src/multiarray/datetime_strings.c +++ b/numpy/core/src/multiarray/datetime_strings.c @@ -1385,21 +1385,23 @@ array_datetime_as_string(PyObject *NPY_UNUSED(self), PyObject *args, /* Parse the input unit if provided */ if (unit_in != NULL && unit_in != Py_None) { PyObject *strobj; - char *str = NULL; - Py_ssize_t len = 0; - if (PyUnicode_Check(unit_in)) { - strobj = PyUnicode_AsASCIIString(unit_in); - if (strobj == NULL) { - goto fail; + if (PyBytes_Check(unit_in)) { + /* accept bytes input */ + PyObject *obj_str = PyUnicode_FromEncodedObject(unit_in, NULL, NULL); + if (obj_str == NULL) { + return 0; } + strobj = obj_str; } else { + Py_INCREF(unit_in); strobj = unit_in; - Py_INCREF(strobj); } - if (PyBytes_AsStringAndSize(strobj, &str, &len) < 0) { + Py_ssize_t len; + char const *str = PyUnicode_AsUTF8AndSize(strobj, &len); + if (str == NULL) { Py_DECREF(strobj); goto fail; } @@ -1434,24 +1436,27 @@ array_datetime_as_string(PyObject *NPY_UNUSED(self), PyObject *args, /* Get the input time zone */ if (timezone_obj != NULL) { - /* Convert to ASCII if it's unicode */ - if (PyUnicode_Check(timezone_obj)) { - /* accept unicode input */ - PyObject *obj_str; - obj_str = PyUnicode_AsASCIIString(timezone_obj); + PyObject *strobj; + if (PyBytes_Check(timezone_obj)) { + /* accept bytes input */ + PyObject *obj_str = PyUnicode_FromEncodedObject(timezone_obj, NULL, NULL); if (obj_str == NULL) { goto fail; } - Py_DECREF(timezone_obj); - timezone_obj = obj_str; + strobj = obj_str; } + else { + Py_INCREF(timezone_obj); + strobj = timezone_obj; + } + + Py_SETREF(timezone_obj, strobj); /* Check for the supported string inputs */ - if (PyBytes_Check(timezone_obj)) { - char *str; + if (PyUnicode_Check(timezone_obj)) { Py_ssize_t len; - - if (PyBytes_AsStringAndSize(timezone_obj, &str, &len) < 0) { + char const *str = PyUnicode_AsUTF8AndSize(timezone_obj, &len); + if (str == NULL) { goto fail; } diff --git a/numpy/core/src/multiarray/descriptor.c b/numpy/core/src/multiarray/descriptor.c index 6e378f626b6a..a8d575248bb7 100644 --- a/numpy/core/src/multiarray/descriptor.c +++ b/numpy/core/src/multiarray/descriptor.c @@ -1497,15 +1497,36 @@ _convert_from_any(PyObject *obj, int align) } else if (PyTuple_Check(obj)) { /* or a tuple */ - return _convert_from_tuple(obj, align); + if (Py_EnterRecursiveCall( + " while trying to convert the given data type from" + " a tuple object" ) != 0) { + return NULL; + } + PyArray_Descr *ret = _convert_from_tuple(obj, align); + Py_LeaveRecursiveCall(); + return ret; } else if (PyList_Check(obj)) { /* or a list */ - return _convert_from_array_descr(obj, align); + if (Py_EnterRecursiveCall( + " while trying to convert the given data type from" + " a list object" ) != 0) { + return NULL; + } + PyArray_Descr *ret = _convert_from_array_descr(obj, align); + Py_LeaveRecursiveCall(); + return ret; } else if (PyDict_Check(obj) || PyDictProxy_Check(obj)) { /* or a dictionary */ - return _convert_from_dict(obj, align); + if (Py_EnterRecursiveCall( + " while trying to convert the given data type from" + " a dict object" ) != 0) { + return NULL; + } + PyArray_Descr *ret = _convert_from_dict(obj, align); + Py_LeaveRecursiveCall(); + return ret; } else if (PyArray_Check(obj)) { PyErr_SetString(PyExc_TypeError, "Cannot construct a dtype from an array"); @@ -1892,18 +1913,26 @@ arraydescr_protocol_typestr_get(PyArray_Descr *self) else { ret = PyUnicode_FromFormat("%c%c%d", endian, basic_, size); } + if (ret == NULL) { + return NULL; + } + if (PyDataType_ISDATETIME(self)) { PyArray_DatetimeMetaData *meta; - meta = get_datetime_metadata_from_dtype(self); if (meta == NULL) { Py_DECREF(ret); return NULL; } + PyObject *umeta = metastr_to_unicode(meta, 0); + if (umeta == NULL) { + Py_DECREF(ret); + return NULL; + } - ret = append_metastr_to_string(meta, 0, ret); + Py_SETREF(ret, PyUnicode_Concat(ret, umeta)); + Py_DECREF(umeta); } - return ret; } @@ -2890,14 +2919,13 @@ arraydescr_setstate(PyArray_Descr *self, PyObject *args) } if (PyDataType_ISDATETIME(self) && (metadata != NULL)) { - PyObject *old_metadata, *errmsg; + PyObject *old_metadata; PyArray_DatetimeMetaData temp_dt_data; if ((! PyTuple_Check(metadata)) || (PyTuple_Size(metadata) != 2)) { - errmsg = PyUnicode_FromString("Invalid datetime dtype (metadata, c_metadata): "); - PyUString_ConcatAndDel(&errmsg, PyObject_Repr(metadata)); - PyErr_SetObject(PyExc_ValueError, errmsg); - Py_DECREF(errmsg); + PyErr_Format(PyExc_ValueError, + "Invalid datetime dtype (metadata, c_metadata): %R", + metadata); return NULL; } diff --git a/numpy/core/src/multiarray/dtype_transfer.c b/numpy/core/src/multiarray/dtype_transfer.c index 42c66ee7f6e7..af4e6c22e601 100644 --- a/numpy/core/src/multiarray/dtype_transfer.c +++ b/numpy/core/src/multiarray/dtype_transfer.c @@ -1322,95 +1322,21 @@ get_unicode_to_datetime_transfer_function(int aligned, return NPY_SUCCEED; } + static int -get_nbo_cast_transfer_function(int aligned, - npy_intp src_stride, npy_intp dst_stride, - PyArray_Descr *src_dtype, PyArray_Descr *dst_dtype, - int move_references, - PyArray_StridedUnaryOp **out_stransfer, - NpyAuxData **out_transferdata, - int *out_needs_api, - int *out_needs_wrap) +get_legacy_dtype_cast_function( + int aligned, npy_intp src_stride, npy_intp dst_stride, + PyArray_Descr *src_dtype, PyArray_Descr *dst_dtype, + int move_references, + PyArray_StridedUnaryOp **out_stransfer, NpyAuxData **out_transferdata, + int *out_needs_api, int *out_needs_wrap) { _strided_cast_data *data; PyArray_VectorUnaryFunc *castfunc; PyArray_Descr *tmp_dtype; - npy_intp shape = 1, src_itemsize = src_dtype->elsize, - dst_itemsize = dst_dtype->elsize; - - if (PyTypeNum_ISNUMBER(src_dtype->type_num) && - PyTypeNum_ISNUMBER(dst_dtype->type_num)) { - *out_needs_wrap = !PyArray_ISNBO(src_dtype->byteorder) || - !PyArray_ISNBO(dst_dtype->byteorder); - return get_nbo_cast_numeric_transfer_function(aligned, - src_stride, dst_stride, - src_dtype->type_num, dst_dtype->type_num, - out_stransfer, out_transferdata); - } - - if (src_dtype->type_num == NPY_DATETIME || - src_dtype->type_num == NPY_TIMEDELTA || - dst_dtype->type_num == NPY_DATETIME || - dst_dtype->type_num == NPY_TIMEDELTA) { - /* A parameterized type, datetime->datetime sometimes needs casting */ - if ((src_dtype->type_num == NPY_DATETIME && - dst_dtype->type_num == NPY_DATETIME) || - (src_dtype->type_num == NPY_TIMEDELTA && - dst_dtype->type_num == NPY_TIMEDELTA)) { - *out_needs_wrap = !PyArray_ISNBO(src_dtype->byteorder) || - !PyArray_ISNBO(dst_dtype->byteorder); - return get_nbo_cast_datetime_transfer_function(aligned, - src_stride, dst_stride, - src_dtype, dst_dtype, - out_stransfer, out_transferdata); - } - - /* - * Datetime <-> string conversions can be handled specially. - * The functions may raise an error if the strings have no - * space, or can't be parsed properly. - */ - if (src_dtype->type_num == NPY_DATETIME) { - switch (dst_dtype->type_num) { - case NPY_STRING: - *out_needs_api = 1; - *out_needs_wrap = !PyArray_ISNBO(src_dtype->byteorder); - return get_nbo_datetime_to_string_transfer_function( - aligned, - src_stride, dst_stride, - src_dtype, dst_dtype, - out_stransfer, out_transferdata); - - case NPY_UNICODE: - return get_datetime_to_unicode_transfer_function( - aligned, - src_stride, dst_stride, - src_dtype, dst_dtype, - out_stransfer, out_transferdata, - out_needs_api); - } - } - else if (dst_dtype->type_num == NPY_DATETIME) { - switch (src_dtype->type_num) { - case NPY_STRING: - *out_needs_api = 1; - *out_needs_wrap = !PyArray_ISNBO(dst_dtype->byteorder); - return get_nbo_string_to_datetime_transfer_function( - aligned, - src_stride, dst_stride, - src_dtype, dst_dtype, - out_stransfer, out_transferdata); - - case NPY_UNICODE: - return get_unicode_to_datetime_transfer_function( - aligned, - src_stride, dst_stride, - src_dtype, dst_dtype, - out_stransfer, out_transferdata, - out_needs_api); - } - } - } + npy_intp shape = 1; + npy_intp src_itemsize = src_dtype->elsize; + npy_intp dst_itemsize = dst_dtype->elsize; *out_needs_wrap = !aligned || !PyArray_ISNBO(src_dtype->byteorder) || @@ -1543,6 +1469,167 @@ get_nbo_cast_transfer_function(int aligned, return NPY_SUCCEED; } + +static int +get_nbo_cast_transfer_function(int aligned, + npy_intp src_stride, npy_intp dst_stride, + PyArray_Descr *src_dtype, PyArray_Descr *dst_dtype, + int move_references, + PyArray_StridedUnaryOp **out_stransfer, + NpyAuxData **out_transferdata, + int *out_needs_api, + int *out_needs_wrap) +{ + if (PyTypeNum_ISNUMBER(src_dtype->type_num) && + PyTypeNum_ISNUMBER(dst_dtype->type_num)) { + *out_needs_wrap = !PyArray_ISNBO(src_dtype->byteorder) || + !PyArray_ISNBO(dst_dtype->byteorder); + return get_nbo_cast_numeric_transfer_function(aligned, + src_stride, dst_stride, + src_dtype->type_num, dst_dtype->type_num, + out_stransfer, out_transferdata); + } + + if (src_dtype->type_num == NPY_DATETIME || + src_dtype->type_num == NPY_TIMEDELTA || + dst_dtype->type_num == NPY_DATETIME || + dst_dtype->type_num == NPY_TIMEDELTA) { + /* A parameterized type, datetime->datetime sometimes needs casting */ + if ((src_dtype->type_num == NPY_DATETIME && + dst_dtype->type_num == NPY_DATETIME) || + (src_dtype->type_num == NPY_TIMEDELTA && + dst_dtype->type_num == NPY_TIMEDELTA)) { + *out_needs_wrap = !PyArray_ISNBO(src_dtype->byteorder) || + !PyArray_ISNBO(dst_dtype->byteorder); + return get_nbo_cast_datetime_transfer_function(aligned, + src_stride, dst_stride, + src_dtype, dst_dtype, + out_stransfer, out_transferdata); + } + + /* + * Datetime <-> string conversions can be handled specially. + * The functions may raise an error if the strings have no + * space, or can't be parsed properly. + */ + if (src_dtype->type_num == NPY_DATETIME) { + switch (dst_dtype->type_num) { + case NPY_STRING: + *out_needs_api = 1; + *out_needs_wrap = !PyArray_ISNBO(src_dtype->byteorder); + return get_nbo_datetime_to_string_transfer_function( + aligned, + src_stride, dst_stride, + src_dtype, dst_dtype, + out_stransfer, out_transferdata); + + case NPY_UNICODE: + return get_datetime_to_unicode_transfer_function( + aligned, + src_stride, dst_stride, + src_dtype, dst_dtype, + out_stransfer, out_transferdata, + out_needs_api); + } + } + else if (dst_dtype->type_num == NPY_DATETIME) { + switch (src_dtype->type_num) { + case NPY_STRING: + *out_needs_api = 1; + *out_needs_wrap = !PyArray_ISNBO(dst_dtype->byteorder); + return get_nbo_string_to_datetime_transfer_function( + aligned, + src_stride, dst_stride, + src_dtype, dst_dtype, + out_stransfer, out_transferdata); + + case NPY_UNICODE: + return get_unicode_to_datetime_transfer_function( + aligned, + src_stride, dst_stride, + src_dtype, dst_dtype, + out_stransfer, out_transferdata, + out_needs_api); + } + } + } + + return get_legacy_dtype_cast_function( + aligned, src_stride, dst_stride, src_dtype, dst_dtype, + move_references, out_stransfer, out_transferdata, + out_needs_api, out_needs_wrap); +} + + +static int +wrap_aligned_contig_transfer_function_with_copyswapn( + int aligned, npy_intp src_stride, npy_intp dst_stride, + PyArray_Descr *src_dtype, PyArray_Descr *dst_dtype, + PyArray_StridedUnaryOp **out_stransfer, NpyAuxData **out_transferdata, + int *out_needs_api, + PyArray_StridedUnaryOp *caststransfer, NpyAuxData *castdata) +{ + NpyAuxData *todata = NULL, *fromdata = NULL; + PyArray_StridedUnaryOp *tobuffer, *frombuffer; + npy_intp src_itemsize = src_dtype->elsize; + npy_intp dst_itemsize = dst_dtype->elsize; + + /* Get the copy/swap operation from src */ + PyArray_GetDTypeCopySwapFn( + aligned, src_stride, src_itemsize, src_dtype, &tobuffer, &todata); + + if (!PyDataType_REFCHK(dst_dtype)) { + /* Copying from buffer is a simple copy/swap operation */ + PyArray_GetDTypeCopySwapFn( + aligned, dst_itemsize, dst_stride, dst_dtype, + &frombuffer, &fromdata); + } + else { + /* + * Since the buffer is initialized to NULL, need to move the + * references in order to DECREF the existing data. + */ + /* Object types cannot be byte swapped */ + assert(PyDataType_ISNOTSWAPPED(dst_dtype)); + /* The loop already needs the python api if this is reached */ + assert(*out_needs_api); + + if (PyArray_GetDTypeTransferFunction( + aligned, dst_itemsize, dst_stride, + dst_dtype, dst_dtype, 1, + &frombuffer, &fromdata, out_needs_api) != NPY_SUCCEED) { + return NPY_FAIL; + } + } + + if (frombuffer == NULL || tobuffer == NULL) { + NPY_AUXDATA_FREE(castdata); + NPY_AUXDATA_FREE(todata); + NPY_AUXDATA_FREE(fromdata); + return NPY_FAIL; + } + + *out_stransfer = caststransfer; + + /* Wrap it all up in a new transfer function + data */ + if (wrap_aligned_contig_transfer_function( + src_itemsize, dst_itemsize, + tobuffer, todata, + frombuffer, fromdata, + caststransfer, castdata, + PyDataType_FLAGCHK(dst_dtype, NPY_NEEDS_INIT), + *out_needs_api, + out_stransfer, out_transferdata) != NPY_SUCCEED) { + NPY_AUXDATA_FREE(castdata); + NPY_AUXDATA_FREE(todata); + NPY_AUXDATA_FREE(fromdata); + return NPY_FAIL; + } + + return NPY_SUCCEED; +} + + static int get_cast_transfer_function(int aligned, npy_intp src_stride, npy_intp dst_stride, @@ -1553,10 +1640,8 @@ get_cast_transfer_function(int aligned, int *out_needs_api) { PyArray_StridedUnaryOp *caststransfer; - NpyAuxData *castdata, *todata = NULL, *fromdata = NULL; + NpyAuxData *castdata; int needs_wrap = 0; - npy_intp src_itemsize = src_dtype->elsize, - dst_itemsize = dst_dtype->elsize; if (get_nbo_cast_transfer_function(aligned, src_stride, dst_stride, @@ -1581,64 +1666,10 @@ get_cast_transfer_function(int aligned, } /* Otherwise, we have to copy and/or swap to aligned temporaries */ else { - PyArray_StridedUnaryOp *tobuffer, *frombuffer; - - /* Get the copy/swap operation from src */ - PyArray_GetDTypeCopySwapFn(aligned, - src_stride, src_itemsize, - src_dtype, - &tobuffer, &todata); - - if (!PyDataType_REFCHK(dst_dtype)) { - /* Copying from buffer is a simple copy/swap operation */ - PyArray_GetDTypeCopySwapFn(aligned, - dst_itemsize, dst_stride, - dst_dtype, - &frombuffer, &fromdata); - } - else { - /* - * Since the buffer is initialized to NULL, need to move the - * references in order to DECREF the existing data. - */ - /* Object types cannot be byte swapped */ - assert(PyDataType_ISNOTSWAPPED(dst_dtype)); - /* The loop already needs the python api if this is reached */ - assert(*out_needs_api); - - if (PyArray_GetDTypeTransferFunction( - aligned, dst_itemsize, dst_stride, - dst_dtype, dst_dtype, 1, - &frombuffer, &fromdata, out_needs_api) != NPY_SUCCEED) { - return NPY_FAIL; - } - } - - if (frombuffer == NULL || tobuffer == NULL) { - NPY_AUXDATA_FREE(castdata); - NPY_AUXDATA_FREE(todata); - NPY_AUXDATA_FREE(fromdata); - return NPY_FAIL; - } - - *out_stransfer = caststransfer; - - /* Wrap it all up in a new transfer function + data */ - if (wrap_aligned_contig_transfer_function( - src_itemsize, dst_itemsize, - tobuffer, todata, - frombuffer, fromdata, - caststransfer, castdata, - PyDataType_FLAGCHK(dst_dtype, NPY_NEEDS_INIT), - *out_needs_api, - out_stransfer, out_transferdata) != NPY_SUCCEED) { - NPY_AUXDATA_FREE(castdata); - NPY_AUXDATA_FREE(todata); - NPY_AUXDATA_FREE(fromdata); - return NPY_FAIL; - } - - return NPY_SUCCEED; + return wrap_aligned_contig_transfer_function_with_copyswapn( + aligned, src_stride, dst_stride, src_dtype, dst_dtype, + out_stransfer, out_transferdata, out_needs_api, + caststransfer, castdata); } } diff --git a/numpy/core/src/multiarray/dtypemeta.c b/numpy/core/src/multiarray/dtypemeta.c index d07dc700dccd..af14bb7e5d5d 100644 --- a/numpy/core/src/multiarray/dtypemeta.c +++ b/numpy/core/src/multiarray/dtypemeta.c @@ -15,6 +15,9 @@ #include "dtypemeta.h" #include "_datetime.h" #include "array_coercion.h" +#include "scalartypes.h" +#include "convert_datatype.h" +#include "usertypes.h" static void @@ -194,7 +197,39 @@ discover_datetime_and_timedelta_from_pyobject( static PyArray_Descr * -flexible_default_descr(PyArray_DTypeMeta *cls) +nonparametric_default_descr(PyArray_DTypeMeta *cls) +{ + Py_INCREF(cls->singleton); + return cls->singleton; +} + + +/* Ensure a copy of the singleton (just in case we do adapt it somewhere) */ +static PyArray_Descr * +datetime_and_timedelta_default_descr(PyArray_DTypeMeta *cls) +{ + return PyArray_DescrNew(cls->singleton); +} + + +static PyArray_Descr * +void_default_descr(PyArray_DTypeMeta *cls) +{ + PyArray_Descr *res = PyArray_DescrNew(cls->singleton); + if (res == NULL) { + return NULL; + } + /* + * The legacy behaviour for `np.array([], dtype="V")` is to use "V8". + * This is because `[]` uses `float64` as dtype, and then that is used + * for the size of the requested void. + */ + res->elsize = 8; + return res; +} + +static PyArray_Descr * +string_and_unicode_default_descr(PyArray_DTypeMeta *cls) { PyArray_Descr *res = PyArray_DescrNewFromType(cls->type_num); if (res == NULL) { @@ -208,6 +243,34 @@ flexible_default_descr(PyArray_DTypeMeta *cls) } +static PyArray_Descr * +string_unicode_common_instance(PyArray_Descr *descr1, PyArray_Descr *descr2) +{ + if (descr1->elsize >= descr2->elsize) { + return ensure_dtype_nbo(descr1); + } + else { + return ensure_dtype_nbo(descr2); + } +} + + +static PyArray_Descr * +void_common_instance(PyArray_Descr *descr1, PyArray_Descr *descr2) +{ + /* + * We currently do not support promotion of void types unless they + * are equivalent. + */ + if (!PyArray_CanCastTypeTo(descr1, descr2, NPY_EQUIV_CASTING)) { + PyErr_SetString(PyExc_TypeError, + "invalid type promotion with structured or void datatype(s)."); + return NULL; + } + Py_INCREF(descr1); + return descr1; +} + static int python_builtins_are_known_scalar_types( PyArray_DTypeMeta *NPY_UNUSED(cls), PyTypeObject *pytype) @@ -241,6 +304,18 @@ python_builtins_are_known_scalar_types( } +static int +signed_integers_is_known_scalar_types( + PyArray_DTypeMeta *cls, PyTypeObject *pytype) +{ + if (python_builtins_are_known_scalar_types(cls, pytype)) { + return 1; + } + /* Convert our scalars (raise on too large unsigned and NaN, etc.) */ + return PyType_IsSubtype(pytype, &PyGenericArrType_Type); +} + + static int datetime_known_scalar_types( PyArray_DTypeMeta *cls, PyTypeObject *pytype) @@ -281,6 +356,86 @@ string_known_scalar_types( } +/* + * The following set of functions define the common dtype operator for + * the builtin types. + */ +static PyArray_DTypeMeta * +default_builtin_common_dtype(PyArray_DTypeMeta *cls, PyArray_DTypeMeta *other) +{ + assert(cls->type_num < NPY_NTYPES); + if (!other->legacy || other->type_num > cls->type_num) { + /* Let the more generic (larger type number) DType handle this */ + Py_INCREF(Py_NotImplemented); + return (PyArray_DTypeMeta *)Py_NotImplemented; + } + + /* + * Note: The use of the promotion table should probably be revised at + * some point. It may be most useful to remove it entirely and then + * consider adding a fast path/cache `PyArray_CommonDType()` itself. + */ + int common_num = _npy_type_promotion_table[cls->type_num][other->type_num]; + if (common_num < 0) { + Py_INCREF(Py_NotImplemented); + return (PyArray_DTypeMeta *)Py_NotImplemented; + } + return PyArray_DTypeFromTypeNum(common_num); +} + + +static PyArray_DTypeMeta * +string_unicode_common_dtype(PyArray_DTypeMeta *cls, PyArray_DTypeMeta *other) +{ + assert(cls->type_num < NPY_NTYPES); + if (!other->legacy || other->type_num > cls->type_num || + other->type_num == NPY_OBJECT) { + /* Let the more generic (larger type number) DType handle this */ + Py_INCREF(Py_NotImplemented); + return (PyArray_DTypeMeta *)Py_NotImplemented; + } + /* + * The builtin types are ordered by complexity (aside from object) here. + * Arguably, we should not consider numbers and strings "common", but + * we currently do. + */ + Py_INCREF(cls); + return cls; +} + +static PyArray_DTypeMeta * +datetime_common_dtype(PyArray_DTypeMeta *cls, PyArray_DTypeMeta *other) +{ + if (cls->type_num == NPY_DATETIME && other->type_num == NPY_TIMEDELTA) { + /* + * TODO: We actually currently do allow promotion here. This is + * currently relied on within `np.add(datetime, timedelta)`, + * while for concatenation the cast step will fail. + */ + Py_INCREF(cls); + return cls; + } + return default_builtin_common_dtype(cls, other); +} + + + +static PyArray_DTypeMeta * +object_common_dtype( + PyArray_DTypeMeta *cls, PyArray_DTypeMeta *NPY_UNUSED(other)) +{ + /* + * The object DType is special in that it can represent everything, + * including all potential user DTypes. + * One reason to defer (or error) here might be if the other DType + * does not support scalars so that e.g. `arr1d[0]` returns a 0-D array + * and `arr.astype(object)` would fail. But object casts are special. + */ + Py_INCREF(cls); + return cls; +} + + /** * This function takes a PyArray_Descr and replaces its base class with * a newly created dtype subclass (DTypeMeta instances). @@ -312,10 +467,28 @@ string_known_scalar_types( NPY_NO_EXPORT int dtypemeta_wrap_legacy_descriptor(PyArray_Descr *descr) { - if (Py_TYPE(descr) != &PyArrayDescr_Type) { + int has_type_set = Py_TYPE(descr) == &PyArrayDescr_Type; + + if (!has_type_set) { + /* Accept if the type was filled in from an existing builtin dtype */ + for (int i = 0; i < NPY_NTYPES; i++) { + PyArray_Descr *builtin = PyArray_DescrFromType(i); + has_type_set = Py_TYPE(descr) == Py_TYPE(builtin); + Py_DECREF(builtin); + if (has_type_set) { + break; + } + } + } + if (!has_type_set) { PyErr_Format(PyExc_RuntimeError, "During creation/wrapping of legacy DType, the original class " - "was not PyArrayDescr_Type (it is replaced in this step)."); + "was not of PyArrayDescr_Type (it is replaced in this step). " + "The extension creating a custom DType for type %S must be " + "modified to ensure `Py_TYPE(descr) == &PyArrayDescr_Type` or " + "that of an existing dtype (with the assumption it is just " + "copied over and can be replaced).", + descr->typeobj, Py_TYPE(descr)); return -1; } @@ -398,36 +571,54 @@ dtypemeta_wrap_legacy_descriptor(PyArray_Descr *descr) dtype_class->f = descr->f; dtype_class->kind = descr->kind; - /* Strings and voids have (strange) logic around scalars. */ + /* Set default functions (correct for most dtypes, override below) */ + dtype_class->default_descr = nonparametric_default_descr; + dtype_class->discover_descr_from_pyobject = ( + nonparametric_discover_descr_from_pyobject); dtype_class->is_known_scalar_type = python_builtins_are_known_scalar_types; + dtype_class->common_dtype = default_builtin_common_dtype; + dtype_class->common_instance = NULL; + + if (PyTypeNum_ISSIGNED(dtype_class->type_num)) { + /* Convert our scalars (raise on too large unsigned and NaN, etc.) */ + dtype_class->is_known_scalar_type = signed_integers_is_known_scalar_types; + } - if (PyTypeNum_ISDATETIME(descr->type_num)) { + if (PyTypeNum_ISUSERDEF(descr->type_num)) { + dtype_class->common_dtype = legacy_userdtype_common_dtype_function; + } + else if (descr->type_num == NPY_OBJECT) { + dtype_class->common_dtype = object_common_dtype; + } + else if (PyTypeNum_ISDATETIME(descr->type_num)) { /* Datetimes are flexible, but were not considered previously */ dtype_class->parametric = NPY_TRUE; + dtype_class->default_descr = datetime_and_timedelta_default_descr; dtype_class->discover_descr_from_pyobject = ( discover_datetime_and_timedelta_from_pyobject); + dtype_class->common_dtype = datetime_common_dtype; + dtype_class->common_instance = datetime_type_promotion; if (descr->type_num == NPY_DATETIME) { dtype_class->is_known_scalar_type = datetime_known_scalar_types; } } else if (PyTypeNum_ISFLEXIBLE(descr->type_num)) { dtype_class->parametric = NPY_TRUE; - dtype_class->default_descr = flexible_default_descr; if (descr->type_num == NPY_VOID) { + dtype_class->default_descr = void_default_descr; dtype_class->discover_descr_from_pyobject = ( void_discover_descr_from_pyobject); + dtype_class->common_instance = void_common_instance; } else { + dtype_class->default_descr = string_and_unicode_default_descr; dtype_class->is_known_scalar_type = string_known_scalar_types; dtype_class->discover_descr_from_pyobject = ( string_discover_descr_from_pyobject); + dtype_class->common_dtype = string_unicode_common_dtype; + dtype_class->common_instance = string_unicode_common_instance; } } - else { - /* nonparametric case */ - dtype_class->discover_descr_from_pyobject = ( - nonparametric_discover_descr_from_pyobject); - } if (_PyArray_MapPyTypeToDType(dtype_class, descr->typeobj, PyTypeNum_ISUSERDEF(dtype_class->type_num)) < 0) { diff --git a/numpy/core/src/multiarray/dtypemeta.h b/numpy/core/src/multiarray/dtypemeta.h index e0909a7eb4b2..83cf7c07e944 100644 --- a/numpy/core/src/multiarray/dtypemeta.h +++ b/numpy/core/src/multiarray/dtypemeta.h @@ -2,6 +2,22 @@ #define _NPY_DTYPEMETA_H #define NPY_DTYPE(descr) ((PyArray_DTypeMeta *)Py_TYPE(descr)) +/* + * This function will hopefully be phased out or replaced, but was convenient + * for incremental implementation of new DTypes based on DTypeMeta. + * (Error checking is not required for DescrFromType, assuming that the + * type is valid.) + */ +static NPY_INLINE PyArray_DTypeMeta * +PyArray_DTypeFromTypeNum(int typenum) +{ + PyArray_Descr *descr = PyArray_DescrFromType(typenum); + PyArray_DTypeMeta *dtype = NPY_DTYPE(descr); + Py_INCREF(dtype); + Py_DECREF(descr); + return dtype; +} + NPY_NO_EXPORT int dtypemeta_wrap_legacy_descriptor(PyArray_Descr *dtypem); diff --git a/numpy/core/src/multiarray/hashdescr.c b/numpy/core/src/multiarray/hashdescr.c index c596a7098a97..e9a99cc8fa8f 100644 --- a/numpy/core/src/multiarray/hashdescr.c +++ b/numpy/core/src/multiarray/hashdescr.c @@ -165,7 +165,7 @@ static int _array_descr_walk_fields(PyObject *names, PyObject* fields, PyObject* } foffset = PyTuple_GET_ITEM(value, 1); - if (!PyInt_Check(foffset)) { + if (!PyLong_Check(foffset)) { PyErr_SetString(PyExc_SystemError, "(Hash) Second item in compound dtype tuple not an int ???"); return -1; @@ -208,7 +208,7 @@ static int _array_descr_walk_subarray(PyArray_ArrayDescr* adescr, PyObject *l) PyList_Append(l, item); } } - else if (PyInt_Check(adescr->shape)) { + else if (PyLong_Check(adescr->shape)) { PyList_Append(l, adescr->shape); } else { diff --git a/numpy/core/src/multiarray/iterators.c b/numpy/core/src/multiarray/iterators.c index 31795b2d0faf..3ebd4c858974 100644 --- a/numpy/core/src/multiarray/iterators.c +++ b/numpy/core/src/multiarray/iterators.c @@ -597,7 +597,7 @@ iter_subscript(PyArrayIterObject *self, PyObject *ind) } /* Check for Integer or Slice */ - if (PyLong_Check(ind) || PyInt_Check(ind) || PySlice_Check(ind)) { + if (PyLong_Check(ind) || PySlice_Check(ind)) { start = parse_index_entry(ind, &step_size, &n_steps, self->size, 0, 1); if (start == -1) { diff --git a/numpy/core/src/multiarray/mapping.c b/numpy/core/src/multiarray/mapping.c index 0998a6b495c8..cb5c3823dccf 100644 --- a/numpy/core/src/multiarray/mapping.c +++ b/numpy/core/src/multiarray/mapping.c @@ -1418,10 +1418,7 @@ _get_field_view(PyArrayObject *arr, PyObject *ind, PyArrayObject **view) return 0; } else if (tup == NULL){ - PyObject *errmsg = PyUnicode_FromString("no field of name "); - PyUString_Concat(&errmsg, ind); - PyErr_SetObject(PyExc_ValueError, errmsg); - Py_DECREF(errmsg); + PyErr_Format(PyExc_ValueError, "no field of name %S", ind); return 0; } if (_unpack_field(tup, &fieldtype, &offset) < 0) { @@ -2345,7 +2342,6 @@ mapiter_fill_info(PyArrayMapIterObject *mit, npy_index_info *indices, int consec_status = -1; int axis, broadcast_axis; npy_intp dimension; - PyObject *errmsg, *tmp; for (i = 0; i < mit->nd_fancy; i++) { mit->dimensions[i] = 1; @@ -2433,35 +2429,38 @@ mapiter_fill_info(PyArrayMapIterObject *mit, npy_index_info *indices, return 0; - broadcast_error: +broadcast_error: ; // Declarations cannot follow labels, add empty statement. /* * Attempt to set a meaningful exception. Could also find out * if a boolean index was converted. */ - errmsg = PyUnicode_FromString("shape mismatch: indexing arrays could not " - "be broadcast together with shapes "); + PyObject *errmsg = PyUnicode_FromString(""); if (errmsg == NULL) { return -1; } - for (i = 0; i < index_num; i++) { if (!(indices[i].type & HAS_FANCY)) { continue; } - tmp = convert_shape_to_string( - PyArray_NDIM((PyArrayObject *)indices[i].object), - PyArray_SHAPE((PyArrayObject *)indices[i].object), - " "); + + int ndim = PyArray_NDIM((PyArrayObject *)indices[i].object); + npy_intp *shape = PyArray_SHAPE((PyArrayObject *)indices[i].object); + PyObject *tmp = convert_shape_to_string(ndim, shape, " "); if (tmp == NULL) { + Py_DECREF(errmsg); return -1; } - PyUString_ConcatAndDel(&errmsg, tmp); + + Py_SETREF(errmsg, PyUnicode_Concat(errmsg, tmp)); + Py_DECREF(tmp); if (errmsg == NULL) { return -1; } } - PyErr_SetObject(PyExc_IndexError, errmsg); + PyErr_Format(PyExc_IndexError, + "shape mismatch: indexing arrays could not " + "be broadcast together with shapes %S", errmsg); Py_DECREF(errmsg); return -1; } @@ -2653,7 +2652,6 @@ PyArray_MapIterNew(npy_index_info *indices , int index_num, int index_type, npy_uint32 extra_op_flags, PyArrayObject *extra_op, PyArray_Descr *extra_op_dtype) { - PyObject *errmsg, *tmp; /* For shape reporting on error */ PyArrayObject *original_extra_op = extra_op; @@ -3183,45 +3181,29 @@ PyArray_MapIterNew(npy_index_info *indices , int index_num, int index_type, goto finish; broadcast_error: - errmsg = PyUnicode_FromString("shape mismatch: value array " - "of shape "); - if (errmsg == NULL) { - goto finish; - } - /* Report the shape of the original array if it exists */ if (original_extra_op == NULL) { original_extra_op = extra_op; } - tmp = convert_shape_to_string(PyArray_NDIM(original_extra_op), - PyArray_DIMS(original_extra_op), " "); - if (tmp == NULL) { - goto finish; - } - PyUString_ConcatAndDel(&errmsg, tmp); - if (errmsg == NULL) { + int extra_ndim = PyArray_NDIM(original_extra_op); + npy_intp *extra_dims = PyArray_DIMS(original_extra_op); + PyObject *shape1 = convert_shape_to_string(extra_ndim, extra_dims, " "); + if (shape1 == NULL) { goto finish; } - tmp = PyUnicode_FromString("could not be broadcast to indexing " - "result of shape "); - PyUString_ConcatAndDel(&errmsg, tmp); - if (errmsg == NULL) { + PyObject *shape2 = convert_shape_to_string(mit->nd, mit->dimensions, ""); + if (shape2 == NULL) + Py_DECREF(shape1); goto finish; - } - tmp = convert_shape_to_string(mit->nd, mit->dimensions, ""); - if (tmp == NULL) { - goto finish; - } - PyUString_ConcatAndDel(&errmsg, tmp); - if (errmsg == NULL) { - goto finish; - } + PyErr_Format(PyExc_ValueError, + "shape mismatch: value array of shape %S could not be broadcast " + "to indexing result of shape %S", shape1, shape2); - PyErr_SetObject(PyExc_ValueError, errmsg); - Py_DECREF(errmsg); + Py_DECREF(shape1); + Py_DECREF(shape2); finish: Py_XDECREF(extra_op); @@ -3320,7 +3302,7 @@ PyArray_MapIterArrayCopyIfOverlap(PyArrayObject * a, PyObject * index, Py_XDECREF(a_copy); Py_XDECREF(subspace); Py_XDECREF((PyObject *)mit); - for (i=0; i < index_num; i++) { + for (i = 0; i < index_num; i++) { Py_XDECREF(indices[i].object); } return NULL; diff --git a/numpy/core/src/multiarray/methods.c b/numpy/core/src/multiarray/methods.c index f7cb2185b5b0..084f7cee1782 100644 --- a/numpy/core/src/multiarray/methods.c +++ b/numpy/core/src/multiarray/methods.c @@ -844,6 +844,21 @@ array_astype(PyArrayObject *self, PyObject *args, PyObject *kwds) if (ret == NULL) { return NULL; } + /* NumPy 1.20, 2020-10-01 */ + if ((PyArray_NDIM(self) != PyArray_NDIM(ret)) && + DEPRECATE_FUTUREWARNING( + "casting an array to a subarray dtype " + "will not using broadcasting in the future, but cast each " + "element to the new dtype and then append the dtype's shape " + "to the new array. You can opt-in to the new behaviour, by " + "additional field to the cast: " + "`arr.astype(np.dtype([('f', dtype)]))['f']`.\n" + "This may lead to a different result or to current failures " + "succeeding. " + "(FutureWarning since NumPy 1.20)") < 0) { + Py_DECREF(ret); + return NULL; + } if (PyArray_CopyInto(ret, self) < 0) { Py_DECREF(ret); diff --git a/numpy/core/src/multiarray/multiarraymodule.c b/numpy/core/src/multiarray/multiarraymodule.c index f1d5ab694a80..1aad70dc65bb 100644 --- a/numpy/core/src/multiarray/multiarraymodule.c +++ b/numpy/core/src/multiarray/multiarraymodule.c @@ -2296,6 +2296,7 @@ array_fromiter(PyObject *NPY_UNUSED(ignored), PyObject *args, PyObject *keywds) array_function_result = array_implement_c_array_function_creation( "fromiter", args, keywds); if (array_function_result != Py_NotImplemented) { + Py_DECREF(descr); return array_function_result; } @@ -2942,6 +2943,7 @@ array_arange(PyObject *NPY_UNUSED(ignored), PyObject *args, PyObject *kws) { array_function_result = array_implement_c_array_function_creation( "arange", args, kws); if (array_function_result != Py_NotImplemented) { + Py_XDECREF(typecode); return array_function_result; } @@ -4409,12 +4411,6 @@ setup_scalartypes(PyObject *NPY_UNUSED(dict)) /* Timedelta is an integer with an associated unit */ SINGLE_INHERIT(Timedelta, SignedInteger); - /* - fprintf(stderr, - "tp_free = %p, PyObject_Del = %p, int_tp_free = %p, base.tp_free = %p\n", - PyIntArrType_Type.tp_free, PyObject_Del, PyInt_Type.tp_free, - PySignedIntegerArrType_Type.tp_free); - */ SINGLE_INHERIT(UByte, UnsignedInteger); SINGLE_INHERIT(UShort, UnsignedInteger); SINGLE_INHERIT(UInt, UnsignedInteger); diff --git a/numpy/core/src/multiarray/nditer_constr.c b/numpy/core/src/multiarray/nditer_constr.c index 4bc6d2ca1c5c..b379a28ac3ae 100644 --- a/numpy/core/src/multiarray/nditer_constr.c +++ b/numpy/core/src/multiarray/nditer_constr.c @@ -1750,73 +1750,70 @@ npyiter_fill_axisdata(NpyIter *iter, npy_uint32 flags, npyiter_opitflags *op_itf return 1; broadcast_error: { - PyObject *errmsg, *tmp; npy_intp remdims[NPY_MAXDIMS]; - char *tmpstr; if (op_axes == NULL) { - errmsg = PyUnicode_FromString("operands could not be broadcast " - "together with shapes "); - if (errmsg == NULL) { + PyObject *shape1 = PyUnicode_FromString(""); + if (shape1 == NULL) { return 0; } for (iop = 0; iop < nop; ++iop) { if (op[iop] != NULL) { - tmp = convert_shape_to_string(PyArray_NDIM(op[iop]), - PyArray_DIMS(op[iop]), - " "); + int ndims = PyArray_NDIM(op[iop]); + npy_intp *dims = PyArray_DIMS(op[iop]); + PyObject *tmp = convert_shape_to_string(ndims, dims, " "); if (tmp == NULL) { - Py_DECREF(errmsg); + Py_DECREF(shape1); return 0; } - PyUString_ConcatAndDel(&errmsg, tmp); - if (errmsg == NULL) { + Py_SETREF(shape1, PyUnicode_Concat(shape1, tmp)); + Py_DECREF(tmp); + if (shape1 == NULL) { return 0; } } } - if (itershape != NULL) { - tmp = PyUnicode_FromString("and requested shape "); - if (tmp == NULL) { - Py_DECREF(errmsg); - return 0; - } - PyUString_ConcatAndDel(&errmsg, tmp); - if (errmsg == NULL) { - return 0; - } - - tmp = convert_shape_to_string(ndim, itershape, ""); - if (tmp == NULL) { - Py_DECREF(errmsg); - return 0; - } - PyUString_ConcatAndDel(&errmsg, tmp); - if (errmsg == NULL) { + if (itershape == NULL) { + PyErr_Format(PyExc_ValueError, + "operands could not be broadcast together with " + "shapes %S", shape1); + Py_DECREF(shape1); + return 0; + } + else { + PyObject *shape2 = convert_shape_to_string(ndim, itershape, ""); + if (shape2 == NULL) { + Py_DECREF(shape1); return 0; } - + PyErr_Format(PyExc_ValueError, + "operands could not be broadcast together with " + "shapes %S and requested shape %S", shape1, shape2); + Py_DECREF(shape1); + Py_DECREF(shape2); + return 0; } - PyErr_SetObject(PyExc_ValueError, errmsg); - Py_DECREF(errmsg); } else { - errmsg = PyUnicode_FromString("operands could not be broadcast " - "together with remapped shapes " - "[original->remapped]: "); + PyObject *shape1 = PyUnicode_FromString(""); + if (shape1 == NULL) { + return 0; + } for (iop = 0; iop < nop; ++iop) { if (op[iop] != NULL) { int *axes = op_axes[iop]; + int ndims = PyArray_NDIM(op[iop]); + npy_intp *dims = PyArray_DIMS(op[iop]); + char *tmpstr = (axes == NULL) ? " " : "->"; - tmpstr = (axes == NULL) ? " " : "->"; - tmp = convert_shape_to_string(PyArray_NDIM(op[iop]), - PyArray_DIMS(op[iop]), - tmpstr); + PyObject *tmp = convert_shape_to_string(ndims, dims, tmpstr); if (tmp == NULL) { + Py_DECREF(shape1); return 0; } - PyUString_ConcatAndDel(&errmsg, tmp); - if (errmsg == NULL) { + Py_SETREF(shape1, PyUnicode_Concat(shape1, tmp)); + Py_DECREF(tmp); + if (shape1 == NULL) { return 0; } @@ -1831,80 +1828,83 @@ broadcast_error: { remdims[idim] = -1; } } - tmp = convert_shape_to_string(ndim, remdims, " "); + PyObject *tmp = convert_shape_to_string(ndim, remdims, " "); if (tmp == NULL) { + Py_DECREF(shape1); return 0; } - PyUString_ConcatAndDel(&errmsg, tmp); - if (errmsg == NULL) { + Py_SETREF(shape1, PyUnicode_Concat(shape1, tmp)); + Py_DECREF(tmp); + if (shape1 == NULL) { return 0; } } } } - if (itershape != NULL) { - tmp = PyUnicode_FromString("and requested shape "); - if (tmp == NULL) { - Py_DECREF(errmsg); - return 0; - } - PyUString_ConcatAndDel(&errmsg, tmp); - if (errmsg == NULL) { - return 0; - } - - tmp = convert_shape_to_string(ndim, itershape, ""); - if (tmp == NULL) { - Py_DECREF(errmsg); - return 0; - } - PyUString_ConcatAndDel(&errmsg, tmp); - if (errmsg == NULL) { + if (itershape == NULL) { + PyErr_Format(PyExc_ValueError, + "operands could not be broadcast together with " + "remapped shapes [original->remapped]: %S", shape1); + Py_DECREF(shape1); + return 0; + } + else { + PyObject *shape2 = convert_shape_to_string(ndim, itershape, ""); + if (shape2 == NULL) { + Py_DECREF(shape1); return 0; } - + PyErr_Format(PyExc_ValueError, + "operands could not be broadcast together with " + "remapped shapes [original->remapped]: %S and " + "requested shape %S", shape1, shape2); + Py_DECREF(shape1); + Py_DECREF(shape2); + return 0; } - PyErr_SetObject(PyExc_ValueError, errmsg); - Py_DECREF(errmsg); } - - return 0; } operand_different_than_broadcast: { - npy_intp remdims[NPY_MAXDIMS]; - PyObject *errmsg, *tmp; - - /* Start of error message */ - if (op_flags[iop] & NPY_ITER_READONLY) { - errmsg = PyUnicode_FromString("non-broadcastable operand " - "with shape "); - } - else { - errmsg = PyUnicode_FromString("non-broadcastable output " - "operand with shape "); - } - if (errmsg == NULL) { + /* operand shape */ + int ndims = PyArray_NDIM(op[iop]); + npy_intp *dims = PyArray_DIMS(op[iop]); + PyObject *shape1 = convert_shape_to_string(ndims, dims, ""); + if (shape1 == NULL) { return 0; } - /* Operand shape */ - tmp = convert_shape_to_string(PyArray_NDIM(op[iop]), - PyArray_DIMS(op[iop]), ""); - if (tmp == NULL) { + /* Broadcast shape */ + PyObject *shape2 = convert_shape_to_string(ndim, broadcast_shape, ""); + if (shape2 == NULL) { + Py_DECREF(shape1); return 0; } - PyUString_ConcatAndDel(&errmsg, tmp); - if (errmsg == NULL) { + + if (op_axes == NULL || op_axes[iop] == NULL) { + /* operand shape not remapped */ + + if (op_flags[iop] & NPY_ITER_READONLY) { + PyErr_Format(PyExc_ValueError, + "non-broadcastable operand with shape %S doesn't " + "match the broadcast shape %S", shape1, shape2); + } + else { + PyErr_Format(PyExc_ValueError, + "non-broadcastable output operand with shape %S doesn't " + "match the broadcast shape %S", shape1, shape2); + } + Py_DECREF(shape1); + Py_DECREF(shape2); return 0; } - /* Remapped operand shape */ - if (op_axes != NULL && op_axes[iop] != NULL) { - int *axes = op_axes[iop]; + else { + /* operand shape remapped */ + npy_intp remdims[NPY_MAXDIMS]; + int *axes = op_axes[iop]; for (idim = 0; idim < ndim; ++idim) { - npy_intp i = axes[ndim-idim-1]; - + npy_intp i = axes[ndim - idim - 1]; if (i >= 0 && i < PyArray_NDIM(op[iop])) { remdims[idim] = PyArray_DIM(op[iop], i); } @@ -1913,48 +1913,30 @@ operand_different_than_broadcast: { } } - tmp = PyUnicode_FromString(" [remapped to "); - if (tmp == NULL) { - return 0; - } - PyUString_ConcatAndDel(&errmsg, tmp); - if (errmsg == NULL) { + PyObject *shape3 = convert_shape_to_string(ndim, remdims, ""); + if (shape3 == NULL) { + Py_DECREF(shape1); + Py_DECREF(shape2); return 0; } - tmp = convert_shape_to_string(ndim, remdims, "]"); - if (tmp == NULL) { - return 0; + if (op_flags[iop] & NPY_ITER_READONLY) { + PyErr_Format(PyExc_ValueError, + "non-broadcastable operand with shape %S " + "[remapped to %S] doesn't match the broadcast shape %S", + shape1, shape3, shape2); } - PyUString_ConcatAndDel(&errmsg, tmp); - if (errmsg == NULL) { - return 0; + else { + PyErr_Format(PyExc_ValueError, + "non-broadcastable output operand with shape %S " + "[remapped to %S] doesn't match the broadcast shape %S", + shape1, shape3, shape2); } - } - - tmp = PyUnicode_FromString(" doesn't match the broadcast shape "); - if (tmp == NULL) { - return 0; - } - PyUString_ConcatAndDel(&errmsg, tmp); - if (errmsg == NULL) { + Py_DECREF(shape1); + Py_DECREF(shape2); + Py_DECREF(shape3); return 0; } - - /* Broadcast shape */ - tmp = convert_shape_to_string(ndim, broadcast_shape, ""); - if (tmp == NULL) { - return 0; - } - PyUString_ConcatAndDel(&errmsg, tmp); - if (errmsg == NULL) { - return 0; - } - - PyErr_SetObject(PyExc_ValueError, errmsg); - Py_DECREF(errmsg); - - return 0; } } diff --git a/numpy/core/src/multiarray/nditer_impl.h b/numpy/core/src/multiarray/nditer_impl.h index bb483bb1f554..378d6f7117b3 100644 --- a/numpy/core/src/multiarray/nditer_impl.h +++ b/numpy/core/src/multiarray/nditer_impl.h @@ -269,7 +269,7 @@ struct NpyIter_AD { #define NAD_STRIDES(axisdata) ( \ &(axisdata)->ad_flexdata + 0) #define NAD_PTRS(axisdata) ((char **) \ - &(axisdata)->ad_flexdata + 1*(nop+1)) + (&(axisdata)->ad_flexdata + 1*(nop+1))) #define NAD_NSTRIDES() \ ((nop) + ((itflags&NPY_ITFLAG_HASINDEX) ? 1 : 0)) diff --git a/numpy/core/src/multiarray/number.c b/numpy/core/src/multiarray/number.c index 87c3c9b0a708..a629dfe97fde 100644 --- a/numpy/core/src/multiarray/number.c +++ b/numpy/core/src/multiarray/number.c @@ -397,14 +397,21 @@ is_scalar_with_conversion(PyObject *o2, double* out_exponent) PyObject *temp; const int optimize_fpexps = 1; - if (PyInt_Check(o2)) { - *out_exponent = (double)PyLong_AsLong(o2); + if (PyLong_Check(o2)) { + long tmp = PyLong_AsLong(o2); + if (error_converting(tmp)) { + PyErr_Clear(); + return NPY_NOSCALAR; + } + *out_exponent = (double)tmp; return NPY_INTPOS_SCALAR; } + if (optimize_fpexps && PyFloat_Check(o2)) { *out_exponent = PyFloat_AsDouble(o2); return NPY_FLOAT_SCALAR; } + if (PyArray_Check(o2)) { if ((PyArray_NDIM((PyArrayObject *)o2) == 0) && ((PyArray_ISINTEGER((PyArrayObject *)o2) || @@ -442,7 +449,7 @@ is_scalar_with_conversion(PyObject *o2, double* out_exponent) else if (PyIndex_Check(o2)) { PyObject* value = PyNumber_Index(o2); Py_ssize_t val; - if (value==NULL) { + if (value == NULL) { if (PyErr_Occurred()) { PyErr_Clear(); } diff --git a/numpy/core/src/multiarray/refcount.c b/numpy/core/src/multiarray/refcount.c index 0f84449af012..41dd059b0ac1 100644 --- a/numpy/core/src/multiarray/refcount.c +++ b/numpy/core/src/multiarray/refcount.c @@ -292,20 +292,22 @@ static void _fillobject(char *optr, PyObject *obj, PyArray_Descr *dtype) { if (!PyDataType_FLAGCHK(dtype, NPY_ITEM_REFCOUNT)) { - if ((obj == Py_None) || (PyInt_Check(obj) && PyLong_AsLong(obj)==0)) { + PyObject *arr; + + if ((obj == Py_None) || + (PyLong_Check(obj) && PyLong_AsLong(obj) == 0)) { return; } - else { - PyObject *arr; - Py_INCREF(dtype); - arr = PyArray_NewFromDescr(&PyArray_Type, dtype, - 0, NULL, NULL, NULL, - 0, NULL); - if (arr!=NULL) { - dtype->f->setitem(obj, optr, arr); - } - Py_XDECREF(arr); + /* Clear possible long conversion error */ + PyErr_Clear(); + Py_INCREF(dtype); + arr = PyArray_NewFromDescr(&PyArray_Type, dtype, + 0, NULL, NULL, NULL, + 0, NULL); + if (arr!=NULL) { + dtype->f->setitem(obj, optr, arr); } + Py_XDECREF(arr); } if (dtype->type_num == NPY_OBJECT) { Py_XINCREF(obj); diff --git a/numpy/core/src/multiarray/scalarapi.c b/numpy/core/src/multiarray/scalarapi.c index b918786f22b6..0e93cbbe9f57 100644 --- a/numpy/core/src/multiarray/scalarapi.c +++ b/numpy/core/src/multiarray/scalarapi.c @@ -35,7 +35,7 @@ scalar_value(PyObject *scalar, PyArray_Descr *descr) { int type_num; int align; - npy_intp memloc; + uintptr_t memloc; if (descr == NULL) { descr = PyArray_DescrFromScalar(scalar); type_num = descr->type_num; @@ -168,7 +168,7 @@ scalar_value(PyObject *scalar, PyArray_Descr *descr) * Use the alignment flag to figure out where the data begins * after a PyObject_HEAD */ - memloc = (npy_intp)scalar; + memloc = (uintptr_t)scalar; memloc += sizeof(PyObject); /* now round-up to the nearest alignment value */ align = descr->alignment; @@ -373,7 +373,8 @@ PyArray_FromScalar(PyObject *scalar, PyArray_Descr *outcode) NPY_NO_EXPORT PyObject * PyArray_ScalarFromObject(PyObject *object) { - PyObject *ret=NULL; + PyObject *ret = NULL; + if (PyArray_IsZeroDim(object)) { return PyArray_ToScalar(PyArray_DATA((PyArrayObject *)object), (PyArrayObject *)object); @@ -390,42 +391,49 @@ PyArray_ScalarFromObject(PyObject *object) PyArrayScalar_RETURN_FALSE; } } - else if (PyInt_Check(object)) { - ret = PyArrayScalar_New(Long); - if (ret == NULL) { - return NULL; + else if (PyLong_Check(object)) { + /* Check if fits in long */ + npy_long val_long = PyLong_AsLong(object); + if (!error_converting(val_long)) { + ret = PyArrayScalar_New(Long); + if (ret != NULL) { + PyArrayScalar_VAL(ret, Long) = val_long; + } + return ret; } - PyArrayScalar_VAL(ret, Long) = PyLong_AsLong(object); + PyErr_Clear(); + + /* Check if fits in long long */ + npy_longlong val_longlong = PyLong_AsLongLong(object); + if (!error_converting(val_longlong)) { + ret = PyArrayScalar_New(LongLong); + if (ret != NULL) { + PyArrayScalar_VAL(ret, LongLong) = val_longlong; + } + return ret; + } + PyErr_Clear(); + + return NULL; } else if (PyFloat_Check(object)) { ret = PyArrayScalar_New(Double); - if (ret == NULL) { - return NULL; + if (ret != NULL) { + PyArrayScalar_VAL(ret, Double) = PyFloat_AS_DOUBLE(object); } - PyArrayScalar_VAL(ret, Double) = PyFloat_AS_DOUBLE(object); + return ret; } else if (PyComplex_Check(object)) { ret = PyArrayScalar_New(CDouble); - if (ret == NULL) { - return NULL; + if (ret != NULL) { + PyArrayScalar_VAL(ret, CDouble).real = PyComplex_RealAsDouble(object); + PyArrayScalar_VAL(ret, CDouble).imag = PyComplex_ImagAsDouble(object); } - PyArrayScalar_VAL(ret, CDouble).real = PyComplex_RealAsDouble(object); - PyArrayScalar_VAL(ret, CDouble).imag = PyComplex_ImagAsDouble(object); + return ret; } - else if (PyLong_Check(object)) { - npy_longlong val; - val = PyLong_AsLongLong(object); - if (error_converting(val)) { - PyErr_Clear(); - return NULL; - } - ret = PyArrayScalar_New(LongLong); - if (ret == NULL) { - return NULL; - } - PyArrayScalar_VAL(ret, LongLong) = val; + else { + return NULL; } - return ret; } /*New reference */ diff --git a/numpy/core/src/multiarray/scalartypes.c.src b/numpy/core/src/multiarray/scalartypes.c.src index 5a3f4922a66a..d2ae6ce31b7c 100644 --- a/numpy/core/src/multiarray/scalartypes.c.src +++ b/numpy/core/src/multiarray/scalartypes.c.src @@ -518,21 +518,15 @@ datetimetype_repr(PyObject *self) */ if ((scal->obmeta.num == 1 && scal->obmeta.base != NPY_FR_h) || scal->obmeta.base == NPY_FR_GENERIC) { - ret = PyUnicode_FromString("numpy.datetime64('"); - PyUString_ConcatAndDel(&ret, - PyUnicode_FromString(iso)); - PyUString_ConcatAndDel(&ret, - PyUnicode_FromString("')")); + ret = PyUnicode_FromFormat("numpy.datetime64('%s')", iso); } else { - ret = PyUnicode_FromString("numpy.datetime64('"); - PyUString_ConcatAndDel(&ret, - PyUnicode_FromString(iso)); - PyUString_ConcatAndDel(&ret, - PyUnicode_FromString("','")); - ret = append_metastr_to_string(&scal->obmeta, 1, ret); - PyUString_ConcatAndDel(&ret, - PyUnicode_FromString("')")); + PyObject *meta = metastr_to_unicode(&scal->obmeta, 1); + if (meta == NULL) { + return NULL; + } + ret = PyUnicode_FromFormat("numpy.datetime64('%s','%S')", iso, meta); + Py_DECREF(meta); } return ret; @@ -542,7 +536,7 @@ static PyObject * timedeltatype_repr(PyObject *self) { PyTimedeltaScalarObject *scal; - PyObject *ret; + PyObject *val, *ret; if (!PyArray_IsScalar(self, Timedelta)) { PyErr_SetString(PyExc_RuntimeError, @@ -554,32 +548,34 @@ timedeltatype_repr(PyObject *self) /* The value */ if (scal->obval == NPY_DATETIME_NAT) { - ret = PyUnicode_FromString("numpy.timedelta64('NaT'"); + val = PyUnicode_FromString("'NaT'"); } else { - /* - * Can't use "%lld" if HAVE_LONG_LONG is not defined - */ + /* Can't use "%lld" if HAVE_LONG_LONG is not defined */ #if defined(HAVE_LONG_LONG) - ret = PyUnicode_FromFormat("numpy.timedelta64(%lld", - (long long)scal->obval); + val = PyUnicode_FromFormat("%lld", (long long)scal->obval); #else - ret = PyUnicode_FromFormat("numpy.timedelta64(%ld", - (long)scal->obval); + val = PyUnicode_FromFormat("%ld", (long)scal->obval); #endif } + if (val == NULL) { + return NULL; + } + /* The metadata unit */ if (scal->obmeta.base == NPY_FR_GENERIC) { - PyUString_ConcatAndDel(&ret, - PyUnicode_FromString(")")); + ret = PyUnicode_FromFormat("numpy.timedelta64(%S)", val); } else { - PyUString_ConcatAndDel(&ret, - PyUnicode_FromString(",'")); - ret = append_metastr_to_string(&scal->obmeta, 1, ret); - PyUString_ConcatAndDel(&ret, - PyUnicode_FromString("')")); + PyObject *meta = metastr_to_unicode(&scal->obmeta, 1); + if (meta == NULL) { + Py_DECREF(val); + return NULL; + } + ret = PyUnicode_FromFormat("numpy.timedelta64(%S,'%S')", val, meta); + Py_DECREF(meta); } + Py_DECREF(val); return ret; } @@ -664,14 +660,12 @@ timedeltatype_str(PyObject *self) * Can't use "%lld" if HAVE_LONG_LONG is not defined */ #if defined(HAVE_LONG_LONG) - ret = PyUnicode_FromFormat("%lld ", - (long long)(scal->obval * scal->obmeta.num)); + ret = PyUnicode_FromFormat("%lld %s", + (long long)(scal->obval * scal->obmeta.num), basestr); #else - ret = PyUnicode_FromFormat("%ld ", - (long)(scal->obval * scal->obmeta.num)); + ret = PyUnicode_FromFormat("%ld %s", + (long)(scal->obval * scal->obmeta.num), basestr); #endif - PyUString_ConcatAndDel(&ret, - PyUnicode_FromString(basestr)); } return ret; @@ -890,7 +884,7 @@ static PyObject * static PyObject * c@name@type_@kind@(PyObject *self) { - PyObject *rstr, *istr, *ret; + PyObject *rstr, *istr; npy_c@name@ val = PyArrayScalar_VAL(self, C@Name@); TrimMode trim = TrimMode_DptZeros; @@ -903,16 +897,13 @@ c@name@type_@kind@(PyObject *self) if (istr == NULL) { return NULL; } - - PyUString_ConcatAndDel(&istr, PyUnicode_FromString("j")); - return istr; + PyObject *ret = PyUnicode_FromFormat("%Sj", istr); + Py_DECREF(istr); + return ret; } if (npy_isfinite(val.real)) { rstr = @name@type_@kind@_either(val.real, trim, trim, 0); - if (rstr == NULL) { - return NULL; - } } else if (npy_isnan(val.real)) { rstr = PyUnicode_FromString("nan"); @@ -923,12 +914,12 @@ c@name@type_@kind@(PyObject *self) else { rstr = PyUnicode_FromString("-inf"); } + if (rstr == NULL) { + return NULL; + } if (npy_isfinite(val.imag)) { istr = @name@type_@kind@_either(val.imag, trim, trim, 1); - if (istr == NULL) { - return NULL; - } } else if (npy_isnan(val.imag)) { istr = PyUnicode_FromString("+nan"); @@ -939,11 +930,14 @@ c@name@type_@kind@(PyObject *self) else { istr = PyUnicode_FromString("-inf"); } + if (istr == NULL) { + Py_DECREF(rstr); + return NULL; + } - ret = PyUnicode_FromString("("); - PyUString_ConcatAndDel(&ret, rstr); - PyUString_ConcatAndDel(&ret, istr); - PyUString_ConcatAndDel(&ret, PyUnicode_FromString("j)")); + PyObject *ret = PyUnicode_FromFormat("(%S%Sj)", rstr, istr); + Py_DECREF(rstr); + Py_DECREF(istr); return ret; } @@ -2389,6 +2383,50 @@ static PySequenceMethods voidtype_as_sequence = { }; +/* + * This function implements simple buffer export for user defined subclasses + * of `np.generic`. All other scalar types override the buffer export. + */ +static int +gentype_arrtype_getbuffer(PyObject *self, Py_buffer *view, int flags) +{ + if ((flags & PyBUF_FORMAT) == PyBUF_FORMAT) { + PyErr_Format(PyExc_TypeError, + "NumPy scalar %R can only exported as a buffer without format.", + self); + return -1; + } + PyArray_Descr *descr = PyArray_DescrFromScalar(self); + if (descr == NULL) { + return -1; + } + if (!PyDataType_ISUSERDEF(descr)) { + /* This path would also reject the (hopefully) impossible "object" */ + PyErr_Format(PyExc_TypeError, + "user-defined scalar %R registered for built-in dtype %S? " + "This should be impossible.", + self, descr); + return -1; + } + view->ndim = 0; + view->len = descr->elsize; + view->itemsize = descr->elsize; + view->shape = NULL; + view->strides = NULL; + view->suboffsets = NULL; + Py_INCREF(self); + view->obj = self; + view->buf = scalar_value(self, descr); + Py_DECREF(descr); + view->format = NULL; + return 0; +} + + +static PyBufferProcs gentype_arrtype_as_buffer = { + .bf_getbuffer = (getbufferproc)gentype_arrtype_getbuffer, +}; + /**begin repeat * #name = bool, byte, short, int, long, longlong, ubyte, ushort, uint, ulong, @@ -2952,7 +2990,7 @@ void_arrtype_new(PyTypeObject *type, PyObject *args, PyObject *kwds) * For a VOID scalar first see if obj is an integer or long * and create new memory of that size (filled with 0) for the scalar */ - if (PyLong_Check(obj) || PyInt_Check(obj) || + if (PyLong_Check(obj) || PyArray_IsScalar(obj, Integer) || (PyArray_Check(obj) && PyArray_NDIM((PyArrayObject *)obj)==0 && @@ -3800,6 +3838,7 @@ initialize_numeric_types(void) PyGenericArrType_Type.tp_alloc = gentype_alloc; PyGenericArrType_Type.tp_free = (freefunc)gentype_free; PyGenericArrType_Type.tp_richcompare = gentype_richcompare; + PyGenericArrType_Type.tp_as_buffer = &gentype_arrtype_as_buffer; PyBoolArrType_Type.tp_as_number = &bool_arrtype_as_number; /* diff --git a/numpy/core/src/multiarray/shape.c b/numpy/core/src/multiarray/shape.c index 1a38fe956f4d..02c349759528 100644 --- a/numpy/core/src/multiarray/shape.c +++ b/numpy/core/src/multiarray/shape.c @@ -458,14 +458,12 @@ _attempt_nocopy_reshape(PyArrayObject *self, int newnd, const npy_intp *newdims, static void raise_reshape_size_mismatch(PyArray_Dims *newshape, PyArrayObject *arr) { - PyObject *msg = PyUnicode_FromFormat("cannot reshape array of size %zd " - "into shape ", PyArray_SIZE(arr)); PyObject *tmp = convert_shape_to_string(newshape->len, newshape->ptr, ""); - - PyUString_ConcatAndDel(&msg, tmp); - if (msg != NULL) { - PyErr_SetObject(PyExc_ValueError, msg); - Py_DECREF(msg); + if (tmp != NULL) { + PyErr_Format(PyExc_ValueError, + "cannot reshape array of size %zd into shape %S", + PyArray_SIZE(arr), tmp); + Py_DECREF(tmp); } } @@ -979,55 +977,6 @@ PyArray_Flatten(PyArrayObject *a, NPY_ORDER order) return (PyObject *)ret; } -/* See shape.h for parameters documentation */ -NPY_NO_EXPORT PyObject * -build_shape_string(npy_intp n, npy_intp const *vals) -{ - npy_intp i; - PyObject *ret, *tmp; - - /* - * Negative dimension indicates "newaxis", which can - * be discarded for printing if it's a leading dimension. - * Find the first non-"newaxis" dimension. - */ - i = 0; - while (i < n && vals[i] < 0) { - ++i; - } - - if (i == n) { - return PyUnicode_FromFormat("()"); - } - else { - ret = PyUnicode_FromFormat("(%" NPY_INTP_FMT, vals[i++]); - if (ret == NULL) { - return NULL; - } - } - - for (; i < n; ++i) { - if (vals[i] < 0) { - tmp = PyUnicode_FromString(",newaxis"); - } - else { - tmp = PyUnicode_FromFormat(",%" NPY_INTP_FMT, vals[i]); - } - if (tmp == NULL) { - Py_DECREF(ret); - return NULL; - } - - PyUString_ConcatAndDel(&ret, tmp); - if (ret == NULL) { - return NULL; - } - } - - tmp = PyUnicode_FromFormat(")"); - PyUString_ConcatAndDel(&ret, tmp); - return ret; -} /*NUMPY_API * diff --git a/numpy/core/src/multiarray/shape.h b/numpy/core/src/multiarray/shape.h index d252925569ba..875b5430f2e8 100644 --- a/numpy/core/src/multiarray/shape.h +++ b/numpy/core/src/multiarray/shape.h @@ -1,13 +1,6 @@ #ifndef _NPY_ARRAY_SHAPE_H_ #define _NPY_ARRAY_SHAPE_H_ -/* - * Builds a string representation of the shape given in 'vals'. - * A negative value in 'vals' gets interpreted as newaxis. - */ -NPY_NO_EXPORT PyObject * -build_shape_string(npy_intp n, npy_intp const *vals); - /* * Creates a sorted stride perm matching the KEEPORDER behavior * of the NpyIter object. Because this operates based on multiple diff --git a/numpy/core/src/multiarray/usertypes.c b/numpy/core/src/multiarray/usertypes.c index 6b6c6bd9d057..1404c9b68c41 100644 --- a/numpy/core/src/multiarray/usertypes.c +++ b/numpy/core/src/multiarray/usertypes.c @@ -38,6 +38,7 @@ maintainer email: oliphant.travis@ieee.org #include "usertypes.h" #include "dtypemeta.h" +#include "scalartypes.h" NPY_NO_EXPORT PyArray_Descr **userdescrs=NULL; @@ -127,6 +128,9 @@ PyArray_InitArrFuncs(PyArray_ArrFuncs *f) f->scalarkind = NULL; f->cancastscalarkindto = NULL; f->cancastto = NULL; + f->fastclip = NULL; + f->fastputmask = NULL; + f->fasttake = NULL; } @@ -192,7 +196,7 @@ PyArray_RegisterDataType(PyArray_Descr *descr) } } typenum = NPY_USERDEF + NPY_NUMUSERTYPES; - descr->type_num = typenum; + descr->type_num = -1; if (PyDataType_ISUNSIZED(descr)) { PyErr_SetString(PyExc_ValueError, "cannot register a" \ "flexible data-type"); @@ -211,18 +215,31 @@ PyArray_RegisterDataType(PyArray_Descr *descr) " is missing."); return -1; } - if (descr->flags & (NPY_ITEM_IS_POINTER | NPY_ITEM_REFCOUNT)) { - PyErr_SetString(PyExc_ValueError, - "Legacy user dtypes referencing python objects or generally " - "allocated memory are unsupported. " - "If you see this error in an existing, working code base, " - "please contact the NumPy developers."); - return -1; - } if (descr->typeobj == NULL) { PyErr_SetString(PyExc_ValueError, "missing typeobject"); return -1; } + if (descr->flags & (NPY_ITEM_IS_POINTER | NPY_ITEM_REFCOUNT)) { + /* + * User dtype can't actually do reference counting, however, there + * are existing hacks (e.g. xpress), which use a structured one: + * dtype((xpress.var, [('variable', 'O')])) + * so we have to support this. But such a structure must be constant + * (i.e. fixed at registration time, this is the case for `xpress`). + */ + if (descr->names == NULL || descr->fields == NULL || + !PyDict_CheckExact(descr->fields)) { + PyErr_Format(PyExc_ValueError, + "Failed to register dtype for %S: Legacy user dtypes " + "using `NPY_ITEM_IS_POINTER` or `NPY_ITEM_REFCOUNT` are" + "unsupported. It is possible to create such a dtype only " + "if it is a structured dtype with names and fields " + "hardcoded at registration time.\n" + "Please contact the NumPy developers if this used to work " + "but now fails.", descr->typeobj); + return -1; + } + } if (test_deprecated_arrfuncs_members(f) < 0) { return -1; @@ -234,9 +251,13 @@ PyArray_RegisterDataType(PyArray_Descr *descr) PyErr_SetString(PyExc_MemoryError, "RegisterDataType"); return -1; } + userdescrs[NPY_NUMUSERTYPES++] = descr; + descr->type_num = typenum; if (dtypemeta_wrap_legacy_descriptor(descr) < 0) { + descr->type_num = -1; + NPY_NUMUSERTYPES--; return -1; } @@ -299,7 +320,7 @@ PyArray_RegisterCanCast(PyArray_Descr *descr, int totype, if (!PyTypeNum_ISUSERDEF(descr->type_num) && !PyTypeNum_ISUSERDEF(totype)) { PyErr_SetString(PyExc_ValueError, - "At least one of the types provided to" + "At least one of the types provided to " "RegisterCanCast must be user-defined."); return -1; } @@ -347,3 +368,123 @@ PyArray_RegisterCanCast(PyArray_Descr *descr, int totype, return _append_new(&descr->f->cancastscalarkindto[scalar], totype); } } + + +/* + * Legacy user DTypes implemented the common DType operation + * (as used in type promotion/result_type, and e.g. the type for + * concatenation), by using "safe cast" logic. + * + * New DTypes do have this behaviour generally, but we use can-cast + * when legacy user dtypes are involved. + */ +NPY_NO_EXPORT PyArray_DTypeMeta * +legacy_userdtype_common_dtype_function( + PyArray_DTypeMeta *cls, PyArray_DTypeMeta *other) +{ + int skind1 = NPY_NOSCALAR, skind2 = NPY_NOSCALAR, skind; + + if (!other->legacy) { + /* legacy DTypes can always defer to new style ones */ + Py_INCREF(Py_NotImplemented); + return (PyArray_DTypeMeta *)Py_NotImplemented; + } + /* Defer so that only one of the types handles the cast */ + if (cls->type_num < other->type_num) { + Py_INCREF(Py_NotImplemented); + return (PyArray_DTypeMeta *)Py_NotImplemented; + } + + /* Check whether casting is possible from one type to the other */ + if (PyArray_CanCastSafely(cls->type_num, other->type_num)) { + Py_INCREF(other); + return other; + } + if (PyArray_CanCastSafely(other->type_num, cls->type_num)) { + Py_INCREF(cls); + return cls; + } + + /* + * The following code used to be part of PyArray_PromoteTypes(). + * We can expect that this code is never used. + * In principle, it allows for promotion of two different user dtypes + * to a single NumPy dtype of the same "kind". In practice + * using the same `kind` as NumPy was never possible due to an + * simplification where `PyArray_EquivTypes(descr1, descr2)` will + * return True if both kind and element size match (e.g. bfloat16 and + * float16 would be equivalent). + * The option is also very obscure and not used in the examples. + */ + + /* Convert the 'kind' char into a scalar kind */ + switch (cls->kind) { + case 'b': + skind1 = NPY_BOOL_SCALAR; + break; + case 'u': + skind1 = NPY_INTPOS_SCALAR; + break; + case 'i': + skind1 = NPY_INTNEG_SCALAR; + break; + case 'f': + skind1 = NPY_FLOAT_SCALAR; + break; + case 'c': + skind1 = NPY_COMPLEX_SCALAR; + break; + } + switch (other->kind) { + case 'b': + skind2 = NPY_BOOL_SCALAR; + break; + case 'u': + skind2 = NPY_INTPOS_SCALAR; + break; + case 'i': + skind2 = NPY_INTNEG_SCALAR; + break; + case 'f': + skind2 = NPY_FLOAT_SCALAR; + break; + case 'c': + skind2 = NPY_COMPLEX_SCALAR; + break; + } + + /* If both are scalars, there may be a promotion possible */ + if (skind1 != NPY_NOSCALAR && skind2 != NPY_NOSCALAR) { + + /* Start with the larger scalar kind */ + skind = (skind1 > skind2) ? skind1 : skind2; + int ret_type_num = _npy_smallest_type_of_kind_table[skind]; + + for (;;) { + + /* If there is no larger type of this kind, try a larger kind */ + if (ret_type_num < 0) { + ++skind; + /* Use -1 to signal no promoted type found */ + if (skind < NPY_NSCALARKINDS) { + ret_type_num = _npy_smallest_type_of_kind_table[skind]; + } + else { + break; + } + } + + /* If we found a type to which we can promote both, done! */ + if (PyArray_CanCastSafely(cls->type_num, ret_type_num) && + PyArray_CanCastSafely(other->type_num, ret_type_num)) { + return PyArray_DTypeFromTypeNum(ret_type_num); + } + + /* Try the next larger type of this kind */ + ret_type_num = _npy_next_larger_type_table[ret_type_num]; + } + } + + Py_INCREF(Py_NotImplemented); + return (PyArray_DTypeMeta *)Py_NotImplemented; +} diff --git a/numpy/core/src/multiarray/usertypes.h b/numpy/core/src/multiarray/usertypes.h index b3e386c5c671..1b323d458e88 100644 --- a/numpy/core/src/multiarray/usertypes.h +++ b/numpy/core/src/multiarray/usertypes.h @@ -17,4 +17,8 @@ NPY_NO_EXPORT int PyArray_RegisterCastFunc(PyArray_Descr *descr, int totype, PyArray_VectorUnaryFunc *castfunc); +NPY_NO_EXPORT PyArray_DTypeMeta * +legacy_userdtype_common_dtype_function( + PyArray_DTypeMeta *cls, PyArray_DTypeMeta *other); + #endif diff --git a/numpy/core/src/npysort/selection.c.src b/numpy/core/src/npysort/selection.c.src index 4fd955200404..0e285b320b91 100644 --- a/numpy/core/src/npysort/selection.c.src +++ b/numpy/core/src/npysort/selection.c.src @@ -323,7 +323,8 @@ NPY_NO_EXPORT int store_pivot(kth, kth, pivots, npiv); return 0; } - else if (@inexact@ && kth == num - 1) { + // Parenthesis around @inexact@ tells clang dead code as intentional + else if ((@inexact@) && kth == num - 1) { /* useful to check if NaN present via partition(d, (x, -1)) */ npy_intp k; npy_intp maxidx = low; diff --git a/numpy/core/src/umath/_rational_tests.c.src b/numpy/core/src/umath/_rational_tests.c.src index 08c259d98f25..7b1e5627ae7f 100644 --- a/numpy/core/src/umath/_rational_tests.c.src +++ b/numpy/core/src/umath/_rational_tests.c.src @@ -663,7 +663,7 @@ static PyGetSetDef pyrational_getset[] = { static PyTypeObject PyRational_Type = { PyVarObject_HEAD_INIT(NULL, 0) - "rational", /* tp_name */ + "numpy.core._rational_tests.rational", /* tp_name */ sizeof(PyRational), /* tp_basicsize */ 0, /* tp_itemsize */ 0, /* tp_dealloc */ diff --git a/numpy/core/src/umath/_umath_tests.c.src b/numpy/core/src/umath/_umath_tests.c.src index 660c296d6ddb..750fbeb92a7b 100644 --- a/numpy/core/src/umath/_umath_tests.c.src +++ b/numpy/core/src/umath/_umath_tests.c.src @@ -461,6 +461,15 @@ addUfuncs(PyObject *dictionary) { PyDict_SetItemString(dictionary, "cross1d", f); Py_DECREF(f); + f = PyUFunc_FromFuncAndDataAndSignature(NULL, NULL, + NULL, 0, 0, 0, PyUFunc_None, "_pickleable_module_global.ufunc", + "A dotted name for pickle testing, does nothing.", 0, NULL); + if (f == NULL) { + return -1; + } + PyDict_SetItemString(dictionary, "_pickleable_module_global_ufunc", f); + Py_DECREF(f); + return 0; } diff --git a/numpy/core/src/umath/loops.c.src b/numpy/core/src/umath/loops.c.src index d9591ab3353b..ef3d5a21a413 100644 --- a/numpy/core/src/umath/loops.c.src +++ b/numpy/core/src/umath/loops.c.src @@ -2647,7 +2647,8 @@ pairwise_sum_@TYPE@(@ftype@ *rr, @ftype@ * ri, char * a, npy_intp n, NPY_NO_EXPORT void @TYPE@_@kind@(char **args, npy_intp const *dimensions, npy_intp const *steps, void *NPY_UNUSED(func)) { - if (IS_BINARY_REDUCE && @PW@) { + // Parenthesis around @PW@ tells clang dead code is intentional + if (IS_BINARY_REDUCE && (@PW@)) { npy_intp n = dimensions[0]; @ftype@ * or = ((@ftype@ *)args[0]); @ftype@ * oi = ((@ftype@ *)args[0]) + 1; diff --git a/numpy/core/src/umath/ufunc_object.c b/numpy/core/src/umath/ufunc_object.c index f693eb5c246f..8660ee413ee9 100644 --- a/numpy/core/src/umath/ufunc_object.c +++ b/numpy/core/src/umath/ufunc_object.c @@ -5977,6 +5977,7 @@ _typecharfromnum(int num) { return ret; } + static PyObject * ufunc_get_doc(PyUFuncObject *ufunc) { @@ -5997,18 +5998,18 @@ ufunc_get_doc(PyUFuncObject *ufunc) * introspection on name and nin + nout to automate the first part * of it the doc string shouldn't need the calling convention */ - doc = PyObject_CallFunctionObjArgs( - _sig_formatter, (PyObject *)ufunc, NULL); + doc = PyObject_CallFunctionObjArgs(_sig_formatter, + (PyObject *)ufunc, NULL); if (doc == NULL) { return NULL; } if (ufunc->doc != NULL) { - PyUString_ConcatAndDel(&doc, - PyUnicode_FromFormat("\n\n%s", ufunc->doc)); + Py_SETREF(doc, PyUnicode_FromFormat("%S\n\n%s", doc, ufunc->doc)); } return doc; } + static PyObject * ufunc_get_nin(PyUFuncObject *ufunc) { diff --git a/numpy/core/src/umath/ufunc_type_resolution.c b/numpy/core/src/umath/ufunc_type_resolution.c index aa6f34d59119..3abeb2c5a202 100644 --- a/numpy/core/src/umath/ufunc_type_resolution.c +++ b/numpy/core/src/umath/ufunc_type_resolution.c @@ -236,21 +236,6 @@ PyUFunc_ValidateCasting(PyUFuncObject *ufunc, return 0; } -/* - * Returns a new reference to type if it is already NBO, otherwise - * returns a copy converted to NBO. - */ -static PyArray_Descr * -ensure_dtype_nbo(PyArray_Descr *type) -{ - if (PyArray_ISNBO(type->byteorder)) { - Py_INCREF(type); - return type; - } - else { - return PyArray_DescrNewByteorder(type, NPY_NATIVE); - } -} /*UFUNC_API * diff --git a/numpy/core/tests/examples/setup.py b/numpy/core/tests/examples/setup.py index 9860bf5f7c75..6e34aa7787ad 100644 --- a/numpy/core/tests/examples/setup.py +++ b/numpy/core/tests/examples/setup.py @@ -9,12 +9,11 @@ from setuptools.extension import Extension import os -here = os.path.dirname(__file__) macros = [("NPY_NO_DEPRECATED_API", 0)] checks = Extension( "checks", - sources=[os.path.join(here, "checks.pyx")], + sources=[os.path.join('.', "checks.pyx")], include_dirs=[np.get_include()], define_macros=macros, ) diff --git a/numpy/core/tests/test_array_coercion.py b/numpy/core/tests/test_array_coercion.py index a6c8cc8b2156..78def936022d 100644 --- a/numpy/core/tests/test_array_coercion.py +++ b/numpy/core/tests/test_array_coercion.py @@ -309,6 +309,13 @@ def test_scalar_coercion_same_as_cast_and_assignment(self, cast_to): # coercion should also raise (error type may change) with pytest.raises(Exception): np.array(scalar, dtype=dtype) + + if (isinstance(scalar, rational) and + np.issubdtype(dtype, np.signedinteger)): + return + + with pytest.raises(Exception): + np.array([scalar], dtype=dtype) # assignment should also raise res = np.zeros((), dtype=dtype) with pytest.raises(Exception): @@ -324,6 +331,46 @@ def test_scalar_coercion_same_as_cast_and_assignment(self, cast_to): ass[()] = scalar assert_array_equal(ass, cast) + @pytest.mark.parametrize("dtype_char", np.typecodes["All"]) + def test_default_dtype_instance(self, dtype_char): + if dtype_char in "SU": + dtype = np.dtype(dtype_char + "1") + elif dtype_char == "V": + # Legacy behaviour was to use V8. The reason was float64 being the + # default dtype and that having 8 bytes. + dtype = np.dtype("V8") + else: + dtype = np.dtype(dtype_char) + + discovered_dtype, _ = _discover_array_parameters([], type(dtype)) + + assert discovered_dtype == dtype + assert discovered_dtype.itemsize == dtype.itemsize + + @pytest.mark.parametrize("dtype", np.typecodes["Integer"]) + def test_scalar_to_int_coerce_does_not_cast(self, dtype): + """ + Signed integers are currently different in that they do not cast other + NumPy scalar, but instead use scalar.__int__(). The harcoded + exception to this rule is `np.array(scalar, dtype=integer)`. + """ + dtype = np.dtype(dtype) + invalid_int = np.ulonglong(-1) + + float_nan = np.float64(np.nan) + + for scalar in [float_nan, invalid_int]: + # This is a special case using casting logic and thus not failing: + coerced = np.array(scalar, dtype=dtype) + cast = np.array(scalar).astype(dtype) + assert_array_equal(coerced, cast) + + # However these fail: + with pytest.raises((ValueError, OverflowError)): + np.array([scalar], dtype=dtype) + with pytest.raises((ValueError, OverflowError)): + cast[()] = scalar + class TestTimeScalars: @pytest.mark.parametrize("dtype", [np.int64, np.float32]) @@ -333,13 +380,21 @@ class TestTimeScalars: param(np.datetime64("NaT", "generic"), id="datetime64[generic](NaT)"), param(np.datetime64(1, "D"), id="datetime64[D]")],) def test_coercion_basic(self, dtype, scalar): + # Note the `[scalar]` is there because np.array(scalar) uses stricter + # `scalar.__int__()` rules for backward compatibility right now. arr = np.array(scalar, dtype=dtype) cast = np.array(scalar).astype(dtype) - ass = np.ones((), dtype=dtype) - ass[()] = scalar # raises, as would np.array([scalar], dtype=dtype) - assert_array_equal(arr, cast) - assert_array_equal(cast, cast) + + ass = np.ones((), dtype=dtype) + if issubclass(dtype, np.integer): + with pytest.raises(TypeError): + # raises, as would np.array([scalar], dtype=dtype), this is + # conversion from times, but behaviour of integers. + ass[()] = scalar + else: + ass[()] = scalar + assert_array_equal(ass, cast) @pytest.mark.parametrize("dtype", [np.int64, np.float32]) @pytest.mark.parametrize("scalar", @@ -619,3 +674,18 @@ def __array__(self): assert arr[()] is ArrayLike arr = np.array([ArrayLike]) assert arr[0] is ArrayLike + + @pytest.mark.skipif( + np.dtype(np.intp).itemsize < 8, reason="Needs 64bit platform") + def test_too_large_array_error_paths(self): + """Test the error paths, including for memory leaks""" + arr = np.array(0, dtype="uint8") + # Guarantees that a contiguous copy won't work: + arr = np.broadcast_to(arr, 2**62) + + for i in range(5): + # repeat, to ensure caching cannot have an effect: + with pytest.raises(MemoryError): + np.array(arr) + with pytest.raises(MemoryError): + np.array([arr]) diff --git a/numpy/core/tests/test_cython.py b/numpy/core/tests/test_cython.py index bfdb692d764d..a1f09d0fef12 100644 --- a/numpy/core/tests/test_cython.py +++ b/numpy/core/tests/test_cython.py @@ -34,21 +34,19 @@ def install_temp(request, tmp_path): here = os.path.dirname(__file__) ext_dir = os.path.join(here, "examples") - tmp_path = tmp_path._str - cytest = os.path.join(tmp_path, "cytest") + cytest = str(tmp_path / "cytest") shutil.copytree(ext_dir, cytest) # build the examples and "install" them into a temporary directory - install_log = os.path.join(tmp_path, "tmp_install_log.txt") + install_log = str(tmp_path / "tmp_install_log.txt") subprocess.check_call( [ sys.executable, "setup.py", "build", "install", - "--prefix", - os.path.join(tmp_path, "installdir"), + "--prefix", str(tmp_path / "installdir"), "--single-version-externally-managed", "--record", install_log, diff --git a/numpy/core/tests/test_datetime.py b/numpy/core/tests/test_datetime.py index f725091c5067..f58cf307a0de 100644 --- a/numpy/core/tests/test_datetime.py +++ b/numpy/core/tests/test_datetime.py @@ -1654,8 +1654,9 @@ def test_datetime_as_string(self): '1959-10-13T12:34:56') assert_equal(np.datetime_as_string(np.datetime64(datetime, 'ms')), '1959-10-13T12:34:56.789') - assert_equal(np.datetime_as_string(np.datetime64(datetime, 'us')), - '1959-10-13T12:34:56.789012') + for us in ['us', 'μs', b'us']: # check non-ascii and bytes too + assert_equal(np.datetime_as_string(np.datetime64(datetime, us)), + '1959-10-13T12:34:56.789012') datetime = '1969-12-31T23:34:56.789012345678901234' diff --git a/numpy/core/tests/test_deprecations.py b/numpy/core/tests/test_deprecations.py index 17391e80cdc4..380b78f678db 100644 --- a/numpy/core/tests/test_deprecations.py +++ b/numpy/core/tests/test_deprecations.py @@ -81,6 +81,8 @@ def assert_deprecated(self, function, num=1, ignore_others=False, kwargs : dict Keyword arguments for `function` """ + __tracebackhide__ = True # Hide traceback for py.test + # reset the log self.log[:] = [] @@ -728,3 +730,44 @@ def test_not_deprecated(self): np.concatenate(([0.], [1.]), out=np.empty(2, dtype=np.int64), casting="same_kind") + +class TestDeprecateSubarrayDTypeDuringArrayCoercion(_DeprecationTestCase): + warning_cls = FutureWarning + message = "(creating|casting) an array (with|to) a subarray dtype" + + def test_deprecated_array(self): + # Arrays are more complex, since they "broadcast" on success: + arr = np.array([1, 2]) + + self.assert_deprecated(lambda: arr.astype("(2)i,")) + with pytest.warns(FutureWarning): + res = arr.astype("(2)i,") + + assert_array_equal(res, [[1, 2], [1, 2]]) + + self.assert_deprecated(lambda: np.array(arr, dtype="(2)i,")) + with pytest.warns(FutureWarning): + res = np.array(arr, dtype="(2)i,") + + assert_array_equal(res, [[1, 2], [1, 2]]) + + with pytest.warns(FutureWarning): + res = np.array([[(1,), (2,)], arr], dtype="(2)i,") + + assert_array_equal(res, [[[1, 1], [2, 2]], [[1, 2], [1, 2]]]) + + def test_deprecated_and_error(self): + # These error paths do not give a warning, but will succeed in the + # future. + arr = np.arange(5 * 2).reshape(5, 2) + def check(): + with pytest.raises(ValueError): + arr.astype("(2,2)f") + + self.assert_deprecated(check) + + def check(): + with pytest.raises(ValueError): + np.array(arr, dtype="(2,2)f") + + self.assert_deprecated(check) diff --git a/numpy/core/tests/test_dtype.py b/numpy/core/tests/test_dtype.py index 2e2b0dbe2a34..1b2b85cc1c29 100644 --- a/numpy/core/tests/test_dtype.py +++ b/numpy/core/tests/test_dtype.py @@ -6,6 +6,7 @@ import numpy as np from numpy.core._rational_tests import rational +from numpy.core._multiarray_tests import create_custom_field_dtype from numpy.testing import ( assert_, assert_equal, assert_array_equal, assert_raises, HAS_REFCOUNT) from numpy.compat import pickle @@ -313,6 +314,24 @@ def test_union_struct(self): 'formats':['i1', 'O'], 'offsets':[np.dtype('intp').itemsize, 0]}) + @pytest.mark.parametrize(["obj", "dtype", "expected"], + [([], ("(2)f4,"), np.empty((0, 2), dtype="f4")), + (3, "(3)f4,", [3, 3, 3]), + (np.float64(2), "(2)f4,", [2, 2]), + ([((0, 1), (1, 2)), ((2,),)], '(2,2)f4', None), + (["1", "2"], "(2)i,", None)]) + def test_subarray_list(self, obj, dtype, expected): + dtype = np.dtype(dtype) + res = np.array(obj, dtype=dtype) + + if expected is None: + # iterate the 1-d list to fill the array + expected = np.empty(len(obj), dtype=dtype) + for i in range(len(expected)): + expected[i] = obj[i] + + assert_array_equal(res, expected) + def test_comma_datetime(self): dt = np.dtype('M8[D],datetime64[Y],i8') assert_equal(dt, np.dtype([('f0', 'M8[D]'), @@ -766,6 +785,26 @@ def test1(self): ('yi', np.dtype((a, (3, 2))))]) assert_dtype_equal(c, d) + def test_list_recursion(self): + l = list() + l.append(('f', l)) + with pytest.raises(RecursionError): + np.dtype(l) + + def test_tuple_recursion(self): + d = np.int32 + for i in range(100000): + d = (d, (1,)) + with pytest.raises(RecursionError): + np.dtype(d) + + def test_dict_recursion(self): + d = dict(names=['self'], formats=[None], offsets=[0]) + d['formats'][0] = d + with pytest.raises(RecursionError): + np.dtype(d) + + class TestMetadata: def test_no_metadata(self): d = np.dtype(int) @@ -1338,3 +1377,35 @@ def test_pairs(self, pair): pair_type = np.dtype('{},{}'.format(*pair)) expected = np.dtype([('f0', pair[0]), ('f1', pair[1])]) assert_equal(pair_type, expected) + + +class TestUserDType: + @pytest.mark.leaks_references(reason="dynamically creates custom dtype.") + def test_custom_structured_dtype(self): + class mytype: + pass + + blueprint = np.dtype([("field", object)]) + dt = create_custom_field_dtype(blueprint, mytype, 0) + assert dt.type == mytype + # We cannot (currently) *create* this dtype with `np.dtype` because + # mytype does not inherit from `np.generic`. This seems like an + # unnecessary restriction, but one that has been around forever: + assert np.dtype(mytype) == np.dtype("O") + + def test_custom_structured_dtype_errors(self): + class mytype: + pass + + blueprint = np.dtype([("field", object)]) + + with pytest.raises(ValueError): + # Tests what happens if fields are unset during creation + # which is currently rejected due to the containing object + # (see PyArray_RegisterDataType). + create_custom_field_dtype(blueprint, mytype, 1) + + with pytest.raises(RuntimeError): + # Tests that a dtype must have its type field set up to np.dtype + # or in this case a builtin instance. + create_custom_field_dtype(blueprint, mytype, 2) diff --git a/numpy/core/tests/test_multiarray.py b/numpy/core/tests/test_multiarray.py index 6f8af175703c..291e8ba8eb15 100644 --- a/numpy/core/tests/test_multiarray.py +++ b/numpy/core/tests/test_multiarray.py @@ -22,6 +22,7 @@ import numpy as np import numpy.core._multiarray_tests as _multiarray_tests +from numpy.core._rational_tests import rational from numpy.testing import ( assert_, assert_raises, assert_warns, assert_equal, assert_almost_equal, assert_array_equal, assert_raises_regex, assert_array_almost_equal, @@ -7143,6 +7144,21 @@ def test_export_flags(self): _multiarray_tests.get_buffer_info, np.arange(5)[::2], ('SIMPLE',)) + @pytest.mark.parametrize(["obj", "error"], [ + pytest.param(np.array([1, 2], dtype=rational), ValueError, id="array"), + pytest.param(rational(1, 2), TypeError, id="scalar")]) + def test_export_and_pickle_user_dtype(self, obj, error): + # User dtypes should export successfully when FORMAT was not requested. + with pytest.raises(error): + _multiarray_tests.get_buffer_info(obj, ("STRIDED", "FORMAT")) + + _multiarray_tests.get_buffer_info(obj, ("STRIDED",)) + + # This is currently also necessary to implement pickling: + pickle_obj = pickle.dumps(obj) + res = pickle.loads(pickle_obj) + assert_array_equal(res, obj) + def test_padding(self): for j in range(8): x = np.array([(1,), (2,)], dtype={'f0': (int, j)}) @@ -7178,9 +7194,10 @@ def test_padded_struct_array(self): x3 = np.arange(dt3.itemsize, dtype=np.int8).view(dt3) self._check_roundtrip(x3) - def test_relaxed_strides(self): - # Test that relaxed strides are converted to non-relaxed - c = np.ones((1, 10, 10), dtype='i8') + @pytest.mark.valgrind_error(reason="leaks buffer info cache temporarily.") + def test_relaxed_strides(self, c=np.ones((1, 10, 10), dtype='i8')): + # Note: c defined as parameter so that it is persistent and leak + # checks will notice gh-16934 (buffer info cache leak). # Check for NPY_RELAXED_STRIDES_CHECKING: if np.ones((10, 1), order="C").flags.f_contiguous: @@ -7205,6 +7222,23 @@ def test_relaxed_strides(self): arr, ['C_CONTIGUOUS']) assert_(strides[-1] == 8) + @pytest.mark.valgrind_error(reason="leaks buffer info cache temporarily.") + @pytest.mark.skipif(not np.ones((10, 1), order="C").flags.f_contiguous, + reason="Test is unnecessary (but fails) without relaxed strides.") + def test_relaxed_strides_buffer_info_leak(self, arr=np.ones((1, 10))): + """Test that alternating export of C- and F-order buffers from + an array which is both C- and F-order when relaxed strides is + active works. + This test defines array in the signature to ensure leaking more + references every time the test is run (catching the leak with + pytest-leaks). + """ + for i in range(10): + _, s = _multiarray_tests.get_buffer_info(arr, ['F_CONTIGUOUS']) + assert s == (8, 8) + _, s = _multiarray_tests.get_buffer_info(arr, ['C_CONTIGUOUS']) + assert s == (80, 8) + def test_out_of_order_fields(self): dt = np.dtype(dict( formats=['i8")])], + [np.dtype("i8,i8"), np.dtype("i8,>i8")], + ]) + def test_valid_void_promotion(self, dtype1, dtype2): + assert np.promote_types(dtype1, dtype2) is dtype1 def test_can_cast(self): assert_(np.can_cast(np.int32, np.int64)) diff --git a/numpy/core/tests/test_regression.py b/numpy/core/tests/test_regression.py index 2e731d4faa4d..831e48e8b529 100644 --- a/numpy/core/tests/test_regression.py +++ b/numpy/core/tests/test_regression.py @@ -1506,10 +1506,10 @@ def test_type(t): test_type(t) def test_buffer_hashlib(self): - from hashlib import md5 + from hashlib import sha256 x = np.array([1, 2, 3], dtype=np.dtype('> count for a in data_a]) + # right shift + shr = self.shr(vdata_a, count) + assert shr == data_shr_a + # right shift by an immediate constant + shri = self.shri(vdata_a, count) + assert shri == data_shr_a + + def test_arithmetic_subadd_saturated(self): + if self.sfx in ("u32", "s32", "u64", "s64"): + return + + data_a = self._data(self._int_max() - self.nlanes) + data_b = self._data(self._int_min(), reverse=True) + vdata_a, vdata_b = self.load(data_a), self.load(data_b) + + data_adds = self._int_clip([a + b for a, b in zip(data_a, data_b)]) + adds = self.adds(vdata_a, vdata_b) + assert adds == data_adds + + data_subs = self._int_clip([a - b for a, b in zip(data_a, data_b)]) + subs = self.subs(vdata_a, vdata_b) + assert subs == data_subs + +class _SIMD_FP(_Test_Utility): + """ + To test all float vector types at once + """ + def test_arithmetic_fused(self): + vdata_a, vdata_b, vdata_c = [self.load(self._data())]*3 + vdata_cx2 = self.add(vdata_c, vdata_c) + # multiply and add, a*b + c + data_fma = self.load([a * b + c for a, b, c in zip(vdata_a, vdata_b, vdata_c)]) + fma = self.muladd(vdata_a, vdata_b, vdata_c) + assert fma == data_fma + # multiply and subtract, a*b - c + fms = self.mulsub(vdata_a, vdata_b, vdata_c) + data_fms = self.sub(data_fma, vdata_cx2) + assert fms == data_fms + # negate multiply and add, -(a*b) + c + nfma = self.nmuladd(vdata_a, vdata_b, vdata_c) + data_nfma = self.sub(vdata_cx2, data_fma) + assert nfma == data_nfma + # negate multiply and subtract, -(a*b) - c + nfms = self.nmulsub(vdata_a, vdata_b, vdata_c) + data_nfms = self.mul(data_fma, self.setall(-1)) + assert nfms == data_nfms + +class _SIMD_ALL(_Test_Utility): + """ + To test all vector types at once + """ + def test_memory_load(self): + data = self._data() + # unaligned load + load_data = self.load(data) + assert load_data == data + # aligned load + loada_data = self.loada(data) + assert loada_data == data + # stream load + loads_data = self.loads(data) + assert loads_data == data + # load lower part + loadl = self.loadl(data) + loadl_half = list(loadl)[:self.nlanes//2] + data_half = data[:self.nlanes//2] + assert loadl_half == data_half + assert loadl != data # detect overflow + + def test_memory_store(self): + data = self._data() + vdata = self.load(data) + # unaligned store + store = [0] * self.nlanes + self.store(store, vdata) + assert store == data + # aligned store + store_a = [0] * self.nlanes + self.storea(store_a, vdata) + assert store_a == data + # stream store + store_s = [0] * self.nlanes + self.stores(store_s, vdata) + assert store_s == data + # store lower part + store_l = [0] * self.nlanes + self.storel(store_l, vdata) + assert store_l[:self.nlanes//2] == data[:self.nlanes//2] + assert store_l != vdata # detect overflow + # store higher part + store_h = [0] * self.nlanes + self.storeh(store_h, vdata) + assert store_h[:self.nlanes//2] == data[self.nlanes//2:] + assert store_h != vdata # detect overflow + + def test_memory_partial_load(self): + if self.sfx in ("u8", "s8", "u16", "s16"): + return + + data = self._data() + lanes = list(range(1, self.nlanes + 1)) + lanes += [self.nlanes**2, self.nlanes**4] # test out of range + for n in lanes: + load_till = self.load_till(data, n, 15) + data_till = data[:n] + [15] * (self.nlanes-n) + assert load_till == data_till + load_tillz = self.load_tillz(data, n) + data_tillz = data[:n] + [0] * (self.nlanes-n) + assert load_tillz == data_tillz + + def test_memory_partial_store(self): + if self.sfx in ("u8", "s8", "u16", "s16"): + return + + data = self._data() + data_rev = self._data(reverse=True) + vdata = self.load(data) + lanes = list(range(1, self.nlanes + 1)) + lanes += [self.nlanes**2, self.nlanes**4] + for n in lanes: + data_till = data_rev.copy() + data_till[:n] = data[:n] + store_till = self._data(reverse=True) + self.store_till(store_till, n, vdata) + assert store_till == data_till + + def test_memory_noncont_load(self): + if self.sfx in ("u8", "s8", "u16", "s16"): + return + + for stride in range(1, 64): + data = self._data(count=stride*self.nlanes) + data_stride = data[::stride] + loadn = self.loadn(data, stride) + assert loadn == data_stride + + for stride in range(-64, 0): + data = self._data(stride, -stride*self.nlanes) + data_stride = self.load(data[::stride]) # cast unsigned + loadn = self.loadn(data, stride) + assert loadn == data_stride + + def test_memory_noncont_partial_load(self): + if self.sfx in ("u8", "s8", "u16", "s16"): + return + + lanes = list(range(1, self.nlanes + 1)) + lanes += [self.nlanes**2, self.nlanes**4] + for stride in range(1, 64): + data = self._data(count=stride*self.nlanes) + data_stride = data[::stride] + for n in lanes: + data_stride_till = data_stride[:n] + [15] * (self.nlanes-n) + loadn_till = self.loadn_till(data, stride, n, 15) + assert loadn_till == data_stride_till + data_stride_tillz = data_stride[:n] + [0] * (self.nlanes-n) + loadn_tillz = self.loadn_tillz(data, stride, n) + assert loadn_tillz == data_stride_tillz + + for stride in range(-64, 0): + data = self._data(stride, -stride*self.nlanes) + data_stride = list(self.load(data[::stride])) # cast unsigned + for n in lanes: + data_stride_till = data_stride[:n] + [15] * (self.nlanes-n) + loadn_till = self.loadn_till(data, stride, n, 15) + assert loadn_till == data_stride_till + data_stride_tillz = data_stride[:n] + [0] * (self.nlanes-n) + loadn_tillz = self.loadn_tillz(data, stride, n) + assert loadn_tillz == data_stride_tillz + + def test_memory_noncont_store(self): + if self.sfx in ("u8", "s8", "u16", "s16"): + return + + vdata = self.load(self._data()) + for stride in range(1, 64): + data = [15] * stride * self.nlanes + data[::stride] = vdata + storen = [15] * stride * self.nlanes + storen += [127]*64 + self.storen(storen, stride, vdata) + assert storen[:-64] == data + assert storen[-64:] == [127]*64 # detect overflow + + for stride in range(-64, 0): + data = [15] * -stride * self.nlanes + data[::stride] = vdata + storen = [127]*64 + storen += [15] * -stride * self.nlanes + self.storen(storen, stride, vdata) + assert storen[64:] == data + assert storen[:64] == [127]*64 # detect overflow + + def test_memory_noncont_partial_store(self): + if self.sfx in ("u8", "s8", "u16", "s16"): + return + + data = self._data() + vdata = self.load(data) + lanes = list(range(1, self.nlanes + 1)) + lanes += [self.nlanes**2, self.nlanes**4] + for stride in range(1, 64): + for n in lanes: + data_till = [15] * stride * self.nlanes + data_till[::stride] = data[:n] + [15] * (self.nlanes-n) + storen_till = [15] * stride * self.nlanes + storen_till += [127]*64 + self.storen_till(storen_till, stride, n, vdata) + assert storen_till[:-64] == data_till + assert storen_till[-64:] == [127]*64 # detect overflow + + for stride in range(-64, 0): + for n in lanes: + data_till = [15] * -stride * self.nlanes + data_till[::stride] = data[:n] + [15] * (self.nlanes-n) + storen_till = [127]*64 + storen_till += [15] * -stride * self.nlanes + self.storen_till(storen_till, stride, n, vdata) + assert storen_till[64:] == data_till + assert storen_till[:64] == [127]*64 # detect overflow + + def test_misc(self): + broadcast_zero = self.zero() + assert broadcast_zero == [0] * self.nlanes + for i in range(1, 10): + broadcasti = self.setall(i) + assert broadcasti == [i] * self.nlanes + + data_a, data_b = self._data(), self._data(reverse=True) + vdata_a, vdata_b = self.load(data_a), self.load(data_b) + + # py level of npyv_set_* don't support ignoring the extra specified lanes or + # fill non-specified lanes with zero. + vset = self.set(*data_a) + assert vset == data_a + # py level of npyv_setf_* don't support ignoring the extra specified lanes or + # fill non-specified lanes with the specified scalar. + vsetf = self.setf(10, *data_a) + assert vsetf == data_a + + # We're testing the sainty of _simd's type-vector, + # reinterpret* intrinsics itself are tested via compiler + # during the build of _simd module + sfxes = ["u8", "s8", "u16", "s16", "u32", "s32", "u64", "s64", "f32"] + if self.npyv.simd_f64: + sfxes.append("f64") + for sfx in sfxes: + vec_name = getattr(self, "reinterpret_" + sfx)(vdata_a).__name__ + assert vec_name == "npyv_" + sfx + + # select & mask operations + select_a = self.select(self.cmpeq(self.zero(), self.zero()), vdata_a, vdata_b) + assert select_a == data_a + select_b = self.select(self.cmpneq(self.zero(), self.zero()), vdata_a, vdata_b) + assert select_b == data_b + + # cleanup intrinsic is only used with AVX for + # zeroing registers to avoid the AVX-SSE transition penalty, + # so nothing to test here + self.npyv.cleanup() + + def test_reorder(self): + data_a, data_b = self._data(), self._data(reverse=True) + vdata_a, vdata_b = self.load(data_a), self.load(data_b) + # lower half part + data_a_lo = data_a[:self.nlanes//2] + data_b_lo = data_b[:self.nlanes//2] + # higher half part + data_a_hi = data_a[self.nlanes//2:] + data_b_hi = data_b[self.nlanes//2:] + # combine two lower parts + combinel = self.combinel(vdata_a, vdata_b) + assert combinel == data_a_lo + data_b_lo + # combine two higher parts + combineh = self.combineh(vdata_a, vdata_b) + assert combineh == data_a_hi + data_b_hi + # combine x2 + combine = self.combine(vdata_a, vdata_b) + assert combine == (data_a_lo + data_b_lo, data_a_hi + data_b_hi) + # zip(interleave) + data_zipl = [v for p in zip(data_a_lo, data_b_lo) for v in p] + data_ziph = [v for p in zip(data_a_hi, data_b_hi) for v in p] + vzip = self.zip(vdata_a, vdata_b) + assert vzip == (data_zipl, data_ziph) + + def test_operators_comparison(self): + if self._is_fp(): + data_a = self._data() + else: + data_a = self._data(self._int_max() - self.nlanes) + data_b = self._data(self._int_min(), reverse=True) + vdata_a, vdata_b = self.load(data_a), self.load(data_b) + + mask_true = self._true_mask() + def to_bool(vector): + return [lane == mask_true for lane in vector] + # equal + data_eq = [a == b for a, b in zip(data_a, data_b)] + cmpeq = to_bool(self.cmpeq(vdata_a, vdata_b)) + assert cmpeq == data_eq + # not equal + data_neq = [a != b for a, b in zip(data_a, data_b)] + cmpneq = to_bool(self.cmpneq(vdata_a, vdata_b)) + assert cmpneq == data_neq + # greater than + data_gt = [a > b for a, b in zip(data_a, data_b)] + cmpgt = to_bool(self.cmpgt(vdata_a, vdata_b)) + assert cmpgt == data_gt + # greater than and equal + data_ge = [a >= b for a, b in zip(data_a, data_b)] + cmpge = to_bool(self.cmpge(vdata_a, vdata_b)) + assert cmpge == data_ge + # less than + data_lt = [a < b for a, b in zip(data_a, data_b)] + cmplt = to_bool(self.cmplt(vdata_a, vdata_b)) + assert cmplt == data_lt + # less than and equal + data_le = [a <= b for a, b in zip(data_a, data_b)] + cmple = to_bool(self.cmple(vdata_a, vdata_b)) + assert cmple == data_le + + def test_operators_logical(self): + if self._is_fp(): + data_a = self._data() + else: + data_a = self._data(self._int_max() - self.nlanes) + data_b = self._data(self._int_min(), reverse=True) + vdata_a, vdata_b = self.load(data_a), self.load(data_b) + + if self._is_fp(): + data_cast_a = self._to_unsigned(vdata_a) + data_cast_b = self._to_unsigned(vdata_b) + cast, cast_data = self._to_unsigned, self._to_unsigned + else: + data_cast_a, data_cast_b = data_a, data_b + cast, cast_data = lambda a: a, self.load + + data_xor = cast_data([a ^ b for a, b in zip(data_cast_a, data_cast_b)]) + vxor = cast(self.xor(vdata_a, vdata_b)) + assert vxor == data_xor + + data_or = cast_data([a | b for a, b in zip(data_cast_a, data_cast_b)]) + vor = cast(getattr(self, "or")(vdata_a, vdata_b)) + assert vor == data_or + + data_and = cast_data([a & b for a, b in zip(data_cast_a, data_cast_b)]) + vand = cast(getattr(self, "and")(vdata_a, vdata_b)) + assert vand == data_and + + data_not = cast_data([~a for a in data_cast_a]) + vnot = cast(getattr(self, "not")(vdata_a)) + assert vnot == data_not + + def test_conversion_boolean(self): + bsfx = "b" + self.sfx[1:] + to_boolean = getattr(self.npyv, "cvt_%s_%s" % (bsfx, self.sfx)) + from_boolean = getattr(self.npyv, "cvt_%s_%s" % (self.sfx, bsfx)) + + false_vb = to_boolean(self.setall(0)) + true_vb = self.cmpeq(self.setall(0), self.setall(0)) + assert false_vb != true_vb + + false_vsfx = from_boolean(false_vb) + true_vsfx = from_boolean(true_vb) + assert false_vsfx != true_vsfx + + def test_arithmetic_subadd(self): + if self._is_fp(): + data_a = self._data() + else: + data_a = self._data(self._int_max() - self.nlanes) + data_b = self._data(self._int_min(), reverse=True) + vdata_a, vdata_b = self.load(data_a), self.load(data_b) + + # non-saturated + data_add = self.load([a + b for a, b in zip(data_a, data_b)]) # load to cast + add = self.add(vdata_a, vdata_b) + assert add == data_add + data_sub = self.load([a - b for a, b in zip(data_a, data_b)]) + sub = self.sub(vdata_a, vdata_b) + assert sub == data_sub + + def test_arithmetic_mul(self): + if self.sfx in ("u64", "s64"): + return + + if self._is_fp(): + data_a = self._data() + else: + data_a = self._data(self._int_max() - self.nlanes) + data_b = self._data(self._int_min(), reverse=True) + vdata_a, vdata_b = self.load(data_a), self.load(data_b) + + data_mul = self.load([a * b for a, b in zip(data_a, data_b)]) + mul = self.mul(vdata_a, vdata_b) + assert mul == data_mul + + def test_arithmetic_div(self): + if not self._is_fp(): + return + + data_a, data_b = self._data(), self._data(reverse=True) + vdata_a, vdata_b = self.load(data_a), self.load(data_b) + + # load to truncate f64 to precision of f32 + data_div = self.load([a / b for a, b in zip(data_a, data_b)]) + div = self.div(vdata_a, vdata_b) + assert div == data_div + + def test_arithmetic_reduce_sum(self): + if not self._is_fp(): + return + # reduce sum + data = self._data() + vdata = self.load(data) + + data_sum = sum(data) + vsum = self.sum(vdata) + assert vsum == data_sum + +int_sfx = ("u8", "s8", "u16", "s16", "u32", "s32", "u64", "s64") +fp_sfx = ("f32", "f64") +all_sfx = int_sfx + fp_sfx +tests_registry = { + int_sfx : _SIMD_INT, + fp_sfx : _SIMD_FP, + all_sfx : _SIMD_ALL +} +for target_name, npyv in targets.items(): + simd_width = npyv.simd if npyv else '' + pretty_name = target_name.split('__') # multi-target separator + if len(pretty_name) > 1: + # multi-target + pretty_name = f"({' '.join(pretty_name)})" + else: + pretty_name = pretty_name[0] + + skip = "" + skip_sfx = dict() + if not npyv: + skip = f"target '{pretty_name}' isn't supported by current machine" + elif not npyv.simd: + skip = f"target '{pretty_name}' isn't supported by NPYV" + elif not npyv.simd_f64: + skip_sfx["f64"] = f"target '{pretty_name}' doesn't support double-precision" + + for sfxes, cls in tests_registry.items(): + for sfx in sfxes: + skip_m = skip_sfx.get(sfx, skip) + inhr = (cls,) + attr = dict(npyv=targets[target_name], sfx=sfx) + tcls = type(f"Test{cls.__name__}_{simd_width}_{target_name}_{sfx}", inhr, attr) + if skip_m: + pytest.mark.skip(reason=skip_m)(tcls) + globals()[tcls.__name__] = tcls diff --git a/numpy/core/tests/test_simd_module.py b/numpy/core/tests/test_simd_module.py new file mode 100644 index 000000000000..3d710884ab09 --- /dev/null +++ b/numpy/core/tests/test_simd_module.py @@ -0,0 +1,97 @@ +import pytest +from numpy.core._simd import targets +""" +This testing unit only for checking the sanity of common functionality, +therefore all we need is just to take one submodule that represents any +of enabled SIMD extensions to run the test on it and the second submodule +required to run only one check related to the possibility of mixing +the data types among each submodule. +""" +npyvs = [npyv_mod for npyv_mod in targets.values() if npyv_mod and npyv_mod.simd] +npyv, npyv2 = (npyvs + [None, None])[:2] + +unsigned_sfx = ["u8", "u16", "u32", "u64"] +signed_sfx = ["s8", "s16", "s32", "s64"] +fp_sfx = ["f32"] +if npyv and npyv.simd_f64: + fp_sfx.append("f64") + +int_sfx = unsigned_sfx + signed_sfx +all_sfx = unsigned_sfx + int_sfx + +@pytest.mark.skipif(not npyv, reason="could not find any SIMD extension with NPYV support") +class Test_SIMD_MODULE: + + @pytest.mark.parametrize('sfx', all_sfx) + def test_num_lanes(self, sfx): + nlanes = getattr(npyv, "nlanes_" + sfx) + vector = getattr(npyv, "setall_" + sfx)(1) + assert len(vector) == nlanes + + @pytest.mark.parametrize('sfx', all_sfx) + def test_type_name(self, sfx): + vector = getattr(npyv, "setall_" + sfx)(1) + assert vector.__name__ == "npyv_" + sfx + + def test_raises(self): + a, b = [npyv.setall_u32(1)]*2 + for sfx in all_sfx: + vcb = lambda intrin: getattr(npyv, f"{intrin}_{sfx}") + pytest.raises(TypeError, vcb("add"), a) + pytest.raises(TypeError, vcb("add"), a, b, a) + pytest.raises(TypeError, vcb("setall")) + pytest.raises(TypeError, vcb("setall"), [1]) + pytest.raises(TypeError, vcb("load"), 1) + pytest.raises(ValueError, vcb("load"), [1]) + pytest.raises(ValueError, vcb("store"), [1], getattr(npyv, f"reinterpret_{sfx}_u32")(a)) + + @pytest.mark.skipif(not npyv2, reason=( + "could not find a second SIMD extension with NPYV support" + )) + def test_nomix(self): + # mix among submodules isn't allowed + a = npyv.setall_u32(1) + a2 = npyv2.setall_u32(1) + pytest.raises(TypeError, npyv.add_u32, a2, a2) + pytest.raises(TypeError, npyv2.add_u32, a, a) + + @pytest.mark.parametrize('sfx', unsigned_sfx) + def test_unsigned_overflow(self, sfx): + nlanes = getattr(npyv, "nlanes_" + sfx) + maxu = (1 << int(sfx[1:])) - 1 + maxu_72 = (1 << 72) - 1 + lane = getattr(npyv, "setall_" + sfx)(maxu_72)[0] + assert lane == maxu + lanes = getattr(npyv, "load_" + sfx)([maxu_72] * nlanes) + assert lanes == [maxu] * nlanes + lane = getattr(npyv, "setall_" + sfx)(-1)[0] + assert lane == maxu + lanes = getattr(npyv, "load_" + sfx)([-1] * nlanes) + assert lanes == [maxu] * nlanes + + @pytest.mark.parametrize('sfx', signed_sfx) + def test_signed_overflow(self, sfx): + nlanes = getattr(npyv, "nlanes_" + sfx) + maxs_72 = (1 << 71) - 1 + lane = getattr(npyv, "setall_" + sfx)(maxs_72)[0] + assert lane == -1 + lanes = getattr(npyv, "load_" + sfx)([maxs_72] * nlanes) + assert lanes == [-1] * nlanes + mins_72 = -1 << 71 + lane = getattr(npyv, "setall_" + sfx)(mins_72)[0] + assert lane == 0 + lanes = getattr(npyv, "load_" + sfx)([mins_72] * nlanes) + assert lanes == [0] * nlanes + + def test_truncate_f32(self): + f32 = npyv.setall_f32(0.1)[0] + assert f32 != 0.1 + assert round(f32, 1) == 0.1 + + def test_compare(self): + data_range = range(0, npyv.nlanes_u32) + vdata = npyv.load_u32(data_range) + assert vdata == list(data_range) + assert vdata == tuple(data_range) + for i in data_range: + assert vdata[i] == data_range[i] diff --git a/numpy/core/tests/test_ufunc.py b/numpy/core/tests/test_ufunc.py index 9eaa1a977715..0e9760853def 100644 --- a/numpy/core/tests/test_ufunc.py +++ b/numpy/core/tests/test_ufunc.py @@ -178,6 +178,10 @@ def __getattr__(self, attr): assert_array_equal(res_num.astype("O"), res_obj) +def _pickleable_module_global(): + pass + + class TestUfunc: def test_pickle(self): for proto in range(2, pickle.HIGHEST_PROTOCOL + 1): @@ -195,6 +199,15 @@ def test_pickle_withstring(self): b"(S'numpy.core.umath'\np1\nS'cos'\np2\ntp3\nRp4\n.") assert_(pickle.loads(astring) is np.cos) + def test_pickle_name_is_qualname(self): + # This tests that a simplification of our ufunc pickle code will + # lead to allowing qualnames as names. Future ufuncs should + # possible add a specific qualname, or a hook into pickling instead + # (dask+numba may benefit). + _pickleable_module_global.ufunc = umt._pickleable_module_global_ufunc + obj = pickle.loads(pickle.dumps(_pickleable_module_global.ufunc)) + assert obj is umt._pickleable_module_global_ufunc + def test_reduceat_shifting_sum(self): L = 6 x = np.arange(L) diff --git a/numpy/distutils/ccompiler_opt.py b/numpy/distutils/ccompiler_opt.py index 72ea0c3888c5..3eba6e32af91 100644 --- a/numpy/distutils/ccompiler_opt.py +++ b/numpy/distutils/ccompiler_opt.py @@ -2372,19 +2372,18 @@ def report(self, full=False): else: dispatch_rows.append(("Generated", '')) for tar in self.feature_sorted(target_sources): - tar_as_seq = [tar] if isinstance(tar, str) else tar sources = target_sources[tar] - name = tar if isinstance(tar, str) else '(%s)' % ' '.join(tar) + pretty_name = tar if isinstance(tar, str) else '(%s)' % ' '.join(tar) flags = ' '.join(self.feature_flags(tar)) implies = ' '.join(self.feature_sorted(self.feature_implies(tar))) detect = ' '.join(self.feature_detect(tar)) extra_checks = [] - for name in tar_as_seq: + for name in ((tar,) if isinstance(tar, str) else tar): extra_checks += self.feature_extra_checks(name) extra_checks = (' '.join(extra_checks) if extra_checks else "none") dispatch_rows.append(('', '')) - dispatch_rows.append((name, implies)) + dispatch_rows.append((pretty_name, implies)) dispatch_rows.append(("Flags", flags)) dispatch_rows.append(("Extra checks", extra_checks)) dispatch_rows.append(("Detect", detect)) diff --git a/numpy/distutils/command/autodist.py b/numpy/distutils/command/autodist.py index 8f6436004a0a..b72d0cab1a7d 100644 --- a/numpy/distutils/command/autodist.py +++ b/numpy/distutils/command/autodist.py @@ -46,15 +46,16 @@ def check_restrict(cmd): return '' -def check_compiler_gcc4(cmd): - """Return True if the C compiler is GCC 4.x.""" +def check_compiler_gcc(cmd): + """Check if the compiler is GCC.""" + cmd._check_compiler() body = textwrap.dedent(""" int main() { - #if (! defined __GNUC__) || (__GNUC__ < 4) - #error gcc >= 4 required + #if (! defined __GNUC__) + #error gcc required #endif return 0; } @@ -62,6 +63,30 @@ def check_compiler_gcc4(cmd): return cmd.try_compile(body, None, None) +def check_gcc_version_at_least(cmd, major, minor=0, patchlevel=0): + """ + Check that the gcc version is at least the specified version.""" + + cmd._check_compiler() + version = '.'.join([str(major), str(minor), str(patchlevel)]) + body = textwrap.dedent(""" + int + main() + { + #if (! defined __GNUC__) || (__GNUC__ < %(major)d) || \\ + (__GNUC_MINOR__ < %(minor)d) || \\ + (__GNUC_PATCHLEVEL__ < %(patchlevel)d) + #error gcc >= %(version)s required + #endif + return 0; + } + """) + kw = {'version': version, 'major': major, 'minor': minor, + 'patchlevel': patchlevel} + + return cmd.try_compile(body % kw, None, None) + + def check_gcc_function_attribute(cmd, attribute, name): """Return True if the given function attribute is supported.""" cmd._check_compiler() diff --git a/numpy/distutils/command/build.py b/numpy/distutils/command/build.py index 60ba4c917169..a4fda537d5dc 100644 --- a/numpy/distutils/command/build.py +++ b/numpy/distutils/command/build.py @@ -22,6 +22,8 @@ class build(old_build): "specify a list of dispatched CPU optimizations"), ('disable-optimization', None, "disable CPU optimized code(dispatch,simd,fast...)"), + ('simd-test=', None, + "specify a list of CPU optimizations to be tested against NumPy SIMD interface"), ] help_options = old_build.help_options + [ @@ -36,6 +38,16 @@ def initialize_options(self): self.cpu_baseline = "min" self.cpu_dispatch = "max -xop -fma4" # drop AMD legacy features by default self.disable_optimization = False + """ + the '_simd' module is a very large. Adding more dispatched features + will increase binary size and compile time. By default we minimize + the targeted features to those most commonly used by the NumPy SIMD interface(NPYV), + NOTE: any specified features will be ignored if they're: + - part of the baseline(--cpu-baseline) + - not part of dispatch-able features(--cpu-dispatch) + - not supported by compiler or platform + """ + self.simd_test = "BASELINE SSE2 SSE42 XOP FMA4 (FMA3 AVX2) AVX512F AVX512_SKX VSX VSX2 VSX3 NEON ASIMD" def finalize_options(self): build_scripts = self.build_scripts diff --git a/numpy/distutils/command/build_clib.py b/numpy/distutils/command/build_clib.py index 87345adbc2a0..a0db6f31f7e5 100644 --- a/numpy/distutils/command/build_clib.py +++ b/numpy/distutils/command/build_clib.py @@ -259,57 +259,56 @@ def build_a_library(self, build_info, lib_name, libraries): if requiref90: self.mkpath(module_build_dir) - dispatch_objects = [] - if not self.disable_optimization: - dispatch_sources = [ - c_sources.pop(c_sources.index(src)) - for src in c_sources[:] if src.endswith(".dispatch.c") - ] - if dispatch_sources: - if not self.inplace: - build_src = self.get_finalized_command("build_src").build_src - else: - build_src = None - dispatch_objects = self.compiler_opt.try_dispatch( - dispatch_sources, - output_dir=self.build_temp, - src_dir=build_src, - macros=macros, - include_dirs=include_dirs, - debug=self.debug, - extra_postargs=extra_postargs - ) - extra_args_baseopt = extra_postargs + self.compiler_opt.cpu_baseline_flags() - else: - extra_args_baseopt = extra_postargs - macros.append(("NPY_DISABLE_OPTIMIZATION", 1)) - if compiler.compiler_type == 'msvc': # this hack works around the msvc compiler attributes # problem, msvc uses its own convention :( c_sources += cxx_sources cxx_sources = [] + # filtering C dispatch-table sources when optimization is not disabled, + # otherwise treated as normal sources. + copt_c_sources = [] + copt_baseline_flags = [] + copt_macros = [] + if not self.disable_optimization: + copt_build_src = None if self.inplace else self.get_finalized_command("build_src").build_src + copt_c_sources = [ + c_sources.pop(c_sources.index(src)) + for src in c_sources[:] if src.endswith(".dispatch.c") + ] + copt_baseline_flags = self.compiler_opt.cpu_baseline_flags() + else: + copt_macros.append(("NPY_DISABLE_OPTIMIZATION", 1)) + objects = [] + if copt_c_sources: + log.info("compiling C dispatch-able sources") + objects += self.compiler_opt.try_dispatch(copt_c_sources, + output_dir=self.build_temp, + src_dir=copt_build_src, + macros=macros + copt_macros, + include_dirs=include_dirs, + debug=self.debug, + extra_postargs=extra_postargs) + if c_sources: log.info("compiling C sources") - objects = compiler.compile(c_sources, - output_dir=self.build_temp, - macros=macros, - include_dirs=include_dirs, - debug=self.debug, - extra_postargs=extra_args_baseopt) - objects.extend(dispatch_objects) + objects += compiler.compile(c_sources, + output_dir=self.build_temp, + macros=macros + copt_macros, + include_dirs=include_dirs, + debug=self.debug, + extra_postargs=extra_postargs + copt_baseline_flags) if cxx_sources: log.info("compiling C++ sources") cxx_compiler = compiler.cxx_compiler() cxx_objects = cxx_compiler.compile(cxx_sources, output_dir=self.build_temp, - macros=macros, + macros=macros + copt_macros, include_dirs=include_dirs, debug=self.debug, - extra_postargs=extra_postargs) + extra_postargs=extra_postargs + copt_baseline_flags) objects.extend(cxx_objects) if f_sources or fmodule_sources: diff --git a/numpy/distutils/command/build_ext.py b/numpy/distutils/command/build_ext.py index c7502d3e6daa..ca6f8bcd24d6 100644 --- a/numpy/distutils/command/build_ext.py +++ b/numpy/distutils/command/build_ext.py @@ -19,8 +19,7 @@ has_cxx_sources, has_f_sources, is_sequence ) from numpy.distutils.command.config_compiler import show_fortran_compilers -from numpy.distutils.ccompiler_opt import new_ccompiler_opt - +from numpy.distutils.ccompiler_opt import new_ccompiler_opt, CCompilerOpt class build_ext (old_build_ext): @@ -39,6 +38,8 @@ class build_ext (old_build_ext): "specify a list of dispatched CPU optimizations"), ('disable-optimization', None, "disable CPU optimized code(dispatch,simd,fast...)"), + ('simd-test=', None, + "specify a list of CPU optimizations to be tested against NumPy SIMD interface"), ] help_options = old_build_ext.help_options + [ @@ -56,6 +57,7 @@ def initialize_options(self): self.cpu_baseline = None self.cpu_dispatch = None self.disable_optimization = None + self.simd_test = None def finalize_options(self): if self.parallel: @@ -87,7 +89,9 @@ def finalize_options(self): ('cpu_baseline', 'cpu_baseline'), ('cpu_dispatch', 'cpu_dispatch'), ('disable_optimization', 'disable_optimization'), + ('simd_test', 'simd_test') ) + CCompilerOpt.conf_target_groups["simd_test"] = self.simd_test def run(self): if not self.extensions: @@ -406,52 +410,49 @@ def build_extension(self, ext): include_dirs = ext.include_dirs + get_numpy_include_dirs() - dispatch_objects = [] + # filtering C dispatch-table sources when optimization is not disabled, + # otherwise treated as normal sources. + copt_c_sources = [] + copt_baseline_flags = [] + copt_macros = [] if not self.disable_optimization: - dispatch_sources = [ + copt_build_src = None if self.inplace else self.get_finalized_command("build_src").build_src + copt_c_sources = [ c_sources.pop(c_sources.index(src)) for src in c_sources[:] if src.endswith(".dispatch.c") ] - if dispatch_sources: - if not self.inplace: - build_src = self.get_finalized_command("build_src").build_src - else: - build_src = None - dispatch_objects = self.compiler_opt.try_dispatch( - dispatch_sources, - output_dir=output_dir, - src_dir=build_src, - macros=macros, - include_dirs=include_dirs, - debug=self.debug, - extra_postargs=extra_args, - **kws - ) - extra_args_baseopt = extra_args + self.compiler_opt.cpu_baseline_flags() + copt_baseline_flags = self.compiler_opt.cpu_baseline_flags() else: - extra_args_baseopt = extra_args - macros.append(("NPY_DISABLE_OPTIMIZATION", 1)) + copt_macros.append(("NPY_DISABLE_OPTIMIZATION", 1)) c_objects = [] + if copt_c_sources: + log.info("compiling C dispatch-able sources") + c_objects += self.compiler_opt.try_dispatch(copt_c_sources, + output_dir=output_dir, + src_dir=copt_build_src, + macros=macros + copt_macros, + include_dirs=include_dirs, + debug=self.debug, + extra_postargs=extra_args, + **kws) if c_sources: log.info("compiling C sources") - c_objects = self.compiler.compile(c_sources, - output_dir=output_dir, - macros=macros, - include_dirs=include_dirs, - debug=self.debug, - extra_postargs=extra_args_baseopt, - **kws) - c_objects.extend(dispatch_objects) - + c_objects += self.compiler.compile(c_sources, + output_dir=output_dir, + macros=macros + copt_macros, + include_dirs=include_dirs, + debug=self.debug, + extra_postargs=extra_args + copt_baseline_flags, + **kws) if cxx_sources: log.info("compiling C++ sources") c_objects += cxx_compiler.compile(cxx_sources, output_dir=output_dir, - macros=macros, + macros=macros + copt_macros, include_dirs=include_dirs, debug=self.debug, - extra_postargs=extra_args, + extra_postargs=extra_args + copt_baseline_flags, **kws) extra_postargs = [] diff --git a/numpy/distutils/command/config.py b/numpy/distutils/command/config.py index e54a5444995b..60881f4a3a4a 100644 --- a/numpy/distutils/command/config.py +++ b/numpy/distutils/command/config.py @@ -20,9 +20,10 @@ from numpy.distutils.command.autodist import (check_gcc_function_attribute, check_gcc_function_attribute_with_intrinsics, check_gcc_variable_attribute, + check_gcc_version_at_least, check_inline, check_restrict, - check_compiler_gcc4) + check_compiler_gcc) LANG_EXT['f77'] = '.f' LANG_EXT['f90'] = '.f90' @@ -416,9 +417,9 @@ def check_restrict(self): otherwise.""" return check_restrict(self) - def check_compiler_gcc4(self): - """Return True if the C compiler is gcc >= 4.""" - return check_compiler_gcc4(self) + def check_compiler_gcc(self): + """Return True if the C compiler is gcc""" + return check_compiler_gcc(self) def check_gcc_function_attribute(self, attribute, name): return check_gcc_function_attribute(self, attribute, name) @@ -431,6 +432,11 @@ def check_gcc_function_attribute_with_intrinsics(self, attribute, name, def check_gcc_variable_attribute(self, attribute): return check_gcc_variable_attribute(self, attribute) + def check_gcc_version_at_least(self, major, minor=0, patchlevel=0): + """Return True if the GCC version is greater than or equal to the + specified version.""" + return check_gcc_version_at_least(self, major, minor, patchlevel) + def get_output(self, body, headers=None, include_dirs=None, libraries=None, library_dirs=None, lang="c", use_tee=None): diff --git a/numpy/distutils/fcompiler/__init__.py b/numpy/distutils/fcompiler/__init__.py index a1c52412d0bf..1f340a412913 100644 --- a/numpy/distutils/fcompiler/__init__.py +++ b/numpy/distutils/fcompiler/__init__.py @@ -743,7 +743,7 @@ def wrap_unlinkable_objects(self, objects, output_dir, extra_dll_dir): ('win32', ('gnu', 'intelv', 'absoft', 'compaqv', 'intelev', 'gnu95', 'g95', 'intelvem', 'intelem', 'flang')), ('cygwin.*', ('gnu', 'intelv', 'absoft', 'compaqv', 'intelev', 'gnu95', 'g95')), - ('linux.*', ('gnu95', 'intel', 'lahey', 'pg', 'absoft', 'nag', 'vast', 'compaq', + ('linux.*', ('gnu95', 'intel', 'lahey', 'pg', 'nv', 'absoft', 'nag', 'vast', 'compaq', 'intele', 'intelem', 'gnu', 'g95', 'pathf95', 'nagfor')), ('darwin.*', ('gnu95', 'nag', 'absoft', 'ibm', 'intel', 'gnu', 'g95', 'pg')), ('sunos.*', ('sun', 'gnu', 'gnu95', 'g95')), diff --git a/numpy/distutils/fcompiler/nv.py b/numpy/distutils/fcompiler/nv.py new file mode 100644 index 000000000000..8e9f1683558a --- /dev/null +++ b/numpy/distutils/fcompiler/nv.py @@ -0,0 +1,55 @@ +import sys + +from numpy.distutils.fcompiler import FCompiler + +compilers = ['NVHPCFCompiler'] + +class NVHPCFCompiler(FCompiler): + """ NVIDIA High Performance Computing (HPC) SDK Fortran Compiler + + https://developer.nvidia.com/hpc-sdk + + Since august 2020 the NVIDIA HPC SDK includes the compilers formerly known as The Portland Group compilers, + https://www.pgroup.com/index.htm. + See also `numpy.distutils.fcompiler.pg`. + """ + + compiler_type = 'nv' + description = 'NVIDIA HPC SDK' + version_pattern = r'\s*(nvfortran|(pg(f77|f90|fortran)) \(aka nvfortran\)) (?P[\d.-]+).*' + + executables = { + 'version_cmd': ["", "-V"], + 'compiler_f77': ["nvfortran"], + 'compiler_fix': ["nvfortran", "-Mfixed"], + 'compiler_f90': ["nvfortran"], + 'linker_so': [""], + 'archiver': ["ar", "-cr"], + 'ranlib': ["ranlib"] + } + pic_flags = ['-fpic'] + + module_dir_switch = '-module ' + module_include_switch = '-I' + + def get_flags(self): + opt = ['-Minform=inform', '-Mnosecond_underscore'] + return self.pic_flags + opt + + def get_flags_opt(self): + return ['-fast'] + + def get_flags_debug(self): + return ['-g'] + + def get_flags_linker_so(self): + return ["-shared", '-fpic'] + + def runtime_library_dir_option(self, dir): + return '-R%s' % dir + +if __name__ == '__main__': + from distutils import log + log.set_verbosity(2) + from numpy.distutils import customized_fcompiler + print(customized_fcompiler(compiler='nv').get_version()) diff --git a/numpy/distutils/misc_util.py b/numpy/distutils/misc_util.py index aa649a23f390..778723bfeeda 100644 --- a/numpy/distutils/misc_util.py +++ b/numpy/distutils/misc_util.py @@ -2356,6 +2356,7 @@ def show(): Examples -------- + >>> import numpy as np >>> np.show_config() blas_opt_info: language = c diff --git a/numpy/distutils/setup.py b/numpy/distutils/setup.py index 798c3686f46d..522756fc9db3 100644 --- a/numpy/distutils/setup.py +++ b/numpy/distutils/setup.py @@ -8,6 +8,7 @@ def configuration(parent_package='',top_path=None): config.add_data_files('site.cfg') config.add_data_files('mingw/gfortran_vs2003_hack.c') config.add_data_dir('checks') + config.add_data_files('*.pyi') config.make_config_py() return config diff --git a/numpy/distutils/system_info.py b/numpy/distutils/system_info.py index 19f7482f2a86..c3bd6347c172 100644 --- a/numpy/distutils/system_info.py +++ b/numpy/distutils/system_info.py @@ -415,6 +415,89 @@ def get_standard_file(fname): return filenames +def _parse_env_order(base_order, env): + """ Parse an environment variable `env` by splitting with "," and only returning elements from `base_order` + + This method will sequence the environment variable and check for their invidual elements in `base_order`. + + The items in the environment variable may be negated via '^item' or '!itema,itemb'. + It must start with ^/! to negate all options. + + Raises + ------ + ValueError: for mixed negated and non-negated orders or multiple negated orders + + Parameters + ---------- + base_order : list of str + the base list of orders + env : str + the environment variable to be parsed, if none is found, `base_order` is returned + + Returns + ------- + allow_order : list of str + allowed orders in lower-case + unknown_order : list of str + for values not overlapping with `base_order` + """ + order_str = os.environ.get(env, None) + + # ensure all base-orders are lower-case (for easier comparison) + base_order = [order.lower() for order in base_order] + if order_str is None: + return base_order, [] + + neg = order_str.startswith('^') or order_str.startswith('!') + # Check format + order_str_l = list(order_str) + sum_neg = order_str_l.count('^') + order_str_l.count('!') + if neg: + if sum_neg > 1: + raise ValueError(f"Environment variable '{env}' may only contain a single (prefixed) negation: {order_str}") + # remove prefix + order_str = order_str[1:] + elif sum_neg > 0: + raise ValueError(f"Environment variable '{env}' may not mix negated an non-negated items: {order_str}") + + # Split and lower case + orders = order_str.lower().split(',') + + # to inform callee about non-overlapping elements + unknown_order = [] + + # if negated, we have to remove from the order + if neg: + allow_order = base_order.copy() + + for order in orders: + if not order: + continue + + if order not in base_order: + unknown_order.append(order) + continue + + if order in allow_order: + allow_order.remove(order) + + else: + allow_order = [] + + for order in orders: + if not order: + continue + + if order not in base_order: + unknown_order.append(order) + continue + + if order not in allow_order: + allow_order.append(order) + + return allow_order, unknown_order + + def get_info(name, notfound_action=0): """ notfound_action: @@ -1766,24 +1849,11 @@ def _calc_info(self, name): return getattr(self, '_calc_info_{}'.format(name))() def calc_info(self): - user_order = os.environ.get(self.order_env_var_name, None) - if user_order is None: - lapack_order = self.lapack_order - else: - # the user has requested the order of the - # check they are all in the available list, a COMMA SEPARATED list - user_order = user_order.lower().split(',') - non_existing = [] - lapack_order = [] - for order in user_order: - if order in self.lapack_order: - lapack_order.append(order) - elif len(order) > 0: - non_existing.append(order) - if len(non_existing) > 0: - raise ValueError("lapack_opt_info user defined " - "LAPACK order has unacceptable " - "values: {}".format(non_existing)) + lapack_order, unknown_order = _parse_env_order(self.lapack_order, self.order_env_var_name) + if len(unknown_order) > 0: + raise ValueError("lapack_opt_info user defined " + "LAPACK order has unacceptable " + "values: {}".format(unknown_order)) for lapack in lapack_order: if self._calc_info(lapack): @@ -1911,22 +1981,9 @@ def _calc_info(self, name): return getattr(self, '_calc_info_{}'.format(name))() def calc_info(self): - user_order = os.environ.get(self.order_env_var_name, None) - if user_order is None: - blas_order = self.blas_order - else: - # the user has requested the order of the - # check they are all in the available list - user_order = user_order.lower().split(',') - non_existing = [] - blas_order = [] - for order in user_order: - if order in self.blas_order: - blas_order.append(order) - elif len(order) > 0: - non_existing.append(order) - if len(non_existing) > 0: - raise ValueError("blas_opt_info user defined BLAS order has unacceptable values: {}".format(non_existing)) + blas_order, unknown_order = _parse_env_order(self.blas_order, self.order_env_var_name) + if len(unknown_order) > 0: + raise ValueError("blas_opt_info user defined BLAS order has unacceptable values: {}".format(unknown_order)) for blas in blas_order: if self._calc_info(blas): diff --git a/numpy/distutils/tests/test_system_info.py b/numpy/distutils/tests/test_system_info.py index 0768ffdde55a..ec15126f7f7b 100644 --- a/numpy/distutils/tests/test_system_info.py +++ b/numpy/distutils/tests/test_system_info.py @@ -284,4 +284,37 @@ def test_overrides(self): assert info.get_lib_dirs() == lib_dirs finally: os.chdir(previousDir) - + + +def test_distutils_parse_env_order(monkeypatch): + from numpy.distutils.system_info import _parse_env_order + env = 'NPY_TESTS_DISTUTILS_PARSE_ENV_ORDER' + + base_order = list('abcdef') + + monkeypatch.setenv(env, 'b,i,e,f') + order, unknown = _parse_env_order(base_order, env) + assert len(order) == 3 + assert order == list('bef') + assert len(unknown) == 1 + + # For when LAPACK/BLAS optimization is disabled + monkeypatch.setenv(env, '') + order, unknown = _parse_env_order(base_order, env) + assert len(order) == 0 + assert len(unknown) == 0 + + for prefix in '^!': + monkeypatch.setenv(env, f'{prefix}b,i,e') + order, unknown = _parse_env_order(base_order, env) + assert len(order) == 4 + assert order == list('acdf') + assert len(unknown) == 1 + + with pytest.raises(ValueError): + monkeypatch.setenv(env, 'b,^e,i') + _parse_env_order(base_order, env) + + with pytest.raises(ValueError): + monkeypatch.setenv(env, '!b,^e,i') + _parse_env_order(base_order, env) diff --git a/numpy/f2py/cfuncs.py b/numpy/f2py/cfuncs.py index 94867b3093ec..26b43e7e6964 100644 --- a/numpy/f2py/cfuncs.py +++ b/numpy/f2py/cfuncs.py @@ -629,7 +629,9 @@ """ needs['string_from_pyobj'] = ['string', 'STRINGMALLOC', 'STRINGCOPYN'] cfuncs['string_from_pyobj'] = """\ -static int string_from_pyobj(string *str,int *len,const string inistr,PyObject *obj,const char *errmess) { +static int +string_from_pyobj(string *str,int *len,const string inistr,PyObject *obj,const char *errmess) +{ PyArrayObject *arr = NULL; PyObject *tmp = NULL; #ifdef DEBUGCFUNCS @@ -684,127 +686,165 @@ Py_XDECREF(tmp); { PyObject* err = PyErr_Occurred(); - if (err==NULL) err = #modulename#_error; - PyErr_SetString(err,errmess); + if (err == NULL) { + err = #modulename#_error; + } + PyErr_SetString(err, errmess); } return 0; } """ + + needs['char_from_pyobj'] = ['int_from_pyobj'] cfuncs['char_from_pyobj'] = """\ -static int char_from_pyobj(char* v,PyObject *obj,const char *errmess) { - int i=0; - if (int_from_pyobj(&i,obj,errmess)) { +static int +char_from_pyobj(char* v, PyObject *obj, const char *errmess) { + int i = 0; + if (int_from_pyobj(&i, obj, errmess)) { *v = (char)i; return 1; } return 0; } """ + + needs['signed_char_from_pyobj'] = ['int_from_pyobj', 'signed_char'] cfuncs['signed_char_from_pyobj'] = """\ -static int signed_char_from_pyobj(signed_char* v,PyObject *obj,const char *errmess) { - int i=0; - if (int_from_pyobj(&i,obj,errmess)) { +static int +signed_char_from_pyobj(signed_char* v, PyObject *obj, const char *errmess) { + int i = 0; + if (int_from_pyobj(&i, obj, errmess)) { *v = (signed_char)i; return 1; } return 0; } """ + + needs['short_from_pyobj'] = ['int_from_pyobj'] cfuncs['short_from_pyobj'] = """\ -static int short_from_pyobj(short* v,PyObject *obj,const char *errmess) { - int i=0; - if (int_from_pyobj(&i,obj,errmess)) { +static int +short_from_pyobj(short* v, PyObject *obj, const char *errmess) { + int i = 0; + if (int_from_pyobj(&i, obj, errmess)) { *v = (short)i; return 1; } return 0; } """ + + cfuncs['int_from_pyobj'] = """\ -static int int_from_pyobj(int* v,PyObject *obj,const char *errmess) { +static int +int_from_pyobj(int* v, PyObject *obj, const char *errmess) +{ PyObject* tmp = NULL; - if (PyInt_Check(obj)) { - *v = (int)PyInt_AS_LONG(obj); - return 1; + + if (PyLong_Check(obj)) { + *v = Npy__PyLong_AsInt(obj); + return !(*v == -1 && PyErr_Occurred()); } + tmp = PyNumber_Long(obj); if (tmp) { - *v = PyInt_AS_LONG(tmp); + *v = Npy__PyLong_AsInt(tmp); Py_DECREF(tmp); - return 1; + return !(*v == -1 && PyErr_Occurred()); } + if (PyComplex_Check(obj)) tmp = PyObject_GetAttrString(obj,\"real\"); else if (PyBytes_Check(obj) || PyUnicode_Check(obj)) /*pass*/; else if (PySequence_Check(obj)) - tmp = PySequence_GetItem(obj,0); + tmp = PySequence_GetItem(obj, 0); if (tmp) { PyErr_Clear(); - if (int_from_pyobj(v,tmp,errmess)) {Py_DECREF(tmp); return 1;} + if (int_from_pyobj(v, tmp, errmess)) { + Py_DECREF(tmp); + return 1; + } Py_DECREF(tmp); } { PyObject* err = PyErr_Occurred(); - if (err==NULL) err = #modulename#_error; - PyErr_SetString(err,errmess); + if (err == NULL) { + err = #modulename#_error; + } + PyErr_SetString(err, errmess); } return 0; } """ + + cfuncs['long_from_pyobj'] = """\ -static int long_from_pyobj(long* v,PyObject *obj,const char *errmess) { +static int +long_from_pyobj(long* v, PyObject *obj, const char *errmess) { PyObject* tmp = NULL; - if (PyInt_Check(obj)) { - *v = PyInt_AS_LONG(obj); - return 1; + + if (PyLong_Check(obj)) { + *v = PyLong_AsLong(obj); + return !(*v == -1 && PyErr_Occurred()); } + tmp = PyNumber_Long(obj); if (tmp) { - *v = PyInt_AS_LONG(tmp); + *v = PyLong_AsLong(tmp); Py_DECREF(tmp); - return 1; + return !(*v == -1 && PyErr_Occurred()); } + if (PyComplex_Check(obj)) tmp = PyObject_GetAttrString(obj,\"real\"); else if (PyBytes_Check(obj) || PyUnicode_Check(obj)) /*pass*/; else if (PySequence_Check(obj)) tmp = PySequence_GetItem(obj,0); + if (tmp) { PyErr_Clear(); - if (long_from_pyobj(v,tmp,errmess)) {Py_DECREF(tmp); return 1;} + if (long_from_pyobj(v, tmp, errmess)) { + Py_DECREF(tmp); + return 1; + } Py_DECREF(tmp); } { PyObject* err = PyErr_Occurred(); - if (err==NULL) err = #modulename#_error; - PyErr_SetString(err,errmess); + if (err == NULL) { + err = #modulename#_error; + } + PyErr_SetString(err, errmess); } return 0; } """ + + needs['long_long_from_pyobj'] = ['long_long'] cfuncs['long_long_from_pyobj'] = """\ -static int long_long_from_pyobj(long_long* v,PyObject *obj,const char *errmess) { +static int +long_long_from_pyobj(long_long* v, PyObject *obj, const char *errmess) +{ PyObject* tmp = NULL; + if (PyLong_Check(obj)) { *v = PyLong_AsLongLong(obj); - return (!PyErr_Occurred()); - } - if (PyInt_Check(obj)) { - *v = (long_long)PyInt_AS_LONG(obj); - return 1; + return !(*v == -1 && PyErr_Occurred()); } + tmp = PyNumber_Long(obj); if (tmp) { *v = PyLong_AsLongLong(tmp); Py_DECREF(tmp); - return (!PyErr_Occurred()); + return !(*v == -1 && PyErr_Occurred()); } + if (PyComplex_Check(obj)) tmp = PyObject_GetAttrString(obj,\"real\"); else if (PyBytes_Check(obj) || PyUnicode_Check(obj)) @@ -813,58 +853,64 @@ tmp = PySequence_GetItem(obj,0); if (tmp) { PyErr_Clear(); - if (long_long_from_pyobj(v,tmp,errmess)) {Py_DECREF(tmp); return 1;} + if (long_long_from_pyobj(v, tmp, errmess)) { + Py_DECREF(tmp); + return 1; + } Py_DECREF(tmp); } { PyObject* err = PyErr_Occurred(); - if (err==NULL) err = #modulename#_error; + if (err == NULL) { + err = #modulename#_error; + } PyErr_SetString(err,errmess); } return 0; } """ + + needs['long_double_from_pyobj'] = ['double_from_pyobj', 'long_double'] cfuncs['long_double_from_pyobj'] = """\ -static int long_double_from_pyobj(long_double* v,PyObject *obj,const char *errmess) { +static int +long_double_from_pyobj(long_double* v, PyObject *obj, const char *errmess) +{ double d=0; if (PyArray_CheckScalar(obj)){ if PyArray_IsScalar(obj, LongDouble) { PyArray_ScalarAsCtype(obj, v); return 1; } - else if (PyArray_Check(obj) && PyArray_TYPE(obj)==NPY_LONGDOUBLE) { + else if (PyArray_Check(obj) && PyArray_TYPE(obj) == NPY_LONGDOUBLE) { (*v) = *((npy_longdouble *)PyArray_DATA(obj)); return 1; } } - if (double_from_pyobj(&d,obj,errmess)) { + if (double_from_pyobj(&d, obj, errmess)) { *v = (long_double)d; return 1; } return 0; } """ + + cfuncs['double_from_pyobj'] = """\ -static int double_from_pyobj(double* v,PyObject *obj,const char *errmess) { +static int +double_from_pyobj(double* v, PyObject *obj, const char *errmess) +{ PyObject* tmp = NULL; if (PyFloat_Check(obj)) { -#ifdef __sgi *v = PyFloat_AsDouble(obj); -#else - *v = PyFloat_AS_DOUBLE(obj); -#endif - return 1; + return !(*v == -1.0 && PyErr_Occurred()); } + tmp = PyNumber_Float(obj); if (tmp) { -#ifdef __sgi *v = PyFloat_AsDouble(tmp); -#else - *v = PyFloat_AS_DOUBLE(tmp); -#endif Py_DECREF(tmp); - return 1; + return !(*v == -1.0 && PyErr_Occurred()); } if (PyComplex_Check(obj)) tmp = PyObject_GetAttrString(obj,\"real\"); @@ -885,9 +931,13 @@ return 0; } """ + + needs['float_from_pyobj'] = ['double_from_pyobj'] cfuncs['float_from_pyobj'] = """\ -static int float_from_pyobj(float* v,PyObject *obj,const char *errmess) { +static int +float_from_pyobj(float* v, PyObject *obj, const char *errmess) +{ double d=0.0; if (double_from_pyobj(&d,obj,errmess)) { *v = (float)d; @@ -896,11 +946,15 @@ return 0; } """ + + needs['complex_long_double_from_pyobj'] = ['complex_long_double', 'long_double', 'complex_double_from_pyobj'] cfuncs['complex_long_double_from_pyobj'] = """\ -static int complex_long_double_from_pyobj(complex_long_double* v,PyObject *obj,const char *errmess) { - complex_double cd={0.0,0.0}; +static int +complex_long_double_from_pyobj(complex_long_double* v, PyObject *obj, const char *errmess) +{ + complex_double cd = {0.0,0.0}; if (PyArray_CheckScalar(obj)){ if PyArray_IsScalar(obj, CLongDouble) { PyArray_ScalarAsCtype(obj, v); @@ -920,13 +974,17 @@ return 0; } """ + + needs['complex_double_from_pyobj'] = ['complex_double'] cfuncs['complex_double_from_pyobj'] = """\ -static int complex_double_from_pyobj(complex_double* v,PyObject *obj,const char *errmess) { +static int +complex_double_from_pyobj(complex_double* v, PyObject *obj, const char *errmess) { Py_complex c; if (PyComplex_Check(obj)) { - c=PyComplex_AsCComplex(obj); - (*v).r=c.real, (*v).i=c.imag; + c = PyComplex_AsCComplex(obj); + (*v).r = c.real; + (*v).i = c.imag; return 1; } if (PyArray_IsScalar(obj, ComplexFloating)) { @@ -955,28 +1013,22 @@ else { arr = PyArray_FromScalar(obj, PyArray_DescrFromType(NPY_CDOUBLE)); } - if (arr==NULL) return 0; + if (arr == NULL) { + return 0; + } (*v).r = ((npy_cdouble *)PyArray_DATA(arr))->real; (*v).i = ((npy_cdouble *)PyArray_DATA(arr))->imag; return 1; } /* Python does not provide PyNumber_Complex function :-( */ - (*v).i=0.0; + (*v).i = 0.0; if (PyFloat_Check(obj)) { -#ifdef __sgi (*v).r = PyFloat_AsDouble(obj); -#else - (*v).r = PyFloat_AS_DOUBLE(obj); -#endif - return 1; - } - if (PyInt_Check(obj)) { - (*v).r = (double)PyInt_AS_LONG(obj); - return 1; + return !((*v).r == -1.0 && PyErr_Occurred()); } if (PyLong_Check(obj)) { (*v).r = PyLong_AsDouble(obj); - return (!PyErr_Occurred()); + return !((*v).r == -1.0 && PyErr_Occurred()); } if (PySequence_Check(obj) && !(PyBytes_Check(obj) || PyUnicode_Check(obj))) { PyObject *tmp = PySequence_GetItem(obj,0); @@ -997,10 +1049,14 @@ return 0; } """ + + needs['complex_float_from_pyobj'] = [ 'complex_float', 'complex_double_from_pyobj'] cfuncs['complex_float_from_pyobj'] = """\ -static int complex_float_from_pyobj(complex_float* v,PyObject *obj,const char *errmess) { +static int +complex_float_from_pyobj(complex_float* v,PyObject *obj,const char *errmess) +{ complex_double cd={0.0,0.0}; if (complex_double_from_pyobj(&cd,obj,errmess)) { (*v).r = (float)cd.r; @@ -1010,6 +1066,8 @@ return 0; } """ + + needs['try_pyarr_from_char'] = ['pyobj_from_char1', 'TRYPYARRAYTEMPLATE'] cfuncs[ 'try_pyarr_from_char'] = 'static int try_pyarr_from_char(PyObject* obj,char* v) {\n TRYPYARRAYTEMPLATE(char,\'c\');\n}\n' @@ -1047,14 +1105,18 @@ cfuncs[ 'try_pyarr_from_complex_double'] = 'static int try_pyarr_from_complex_double(PyObject* obj,complex_double* v) {\n TRYCOMPLEXPYARRAYTEMPLATE(double,\'D\');\n}\n' -needs['create_cb_arglist'] = ['CFUNCSMESS', 'PRINTPYOBJERR', 'MINMAX'] +needs['create_cb_arglist'] = ['CFUNCSMESS', 'PRINTPYOBJERR', 'MINMAX'] # create the list of arguments to be used when calling back to python cfuncs['create_cb_arglist'] = """\ -static int create_cb_arglist(PyObject* fun,PyTupleObject* xa,const int maxnofargs,const int nofoptargs,int *nofargs,PyTupleObject **args,const char *errmess) { +static int +create_cb_arglist(PyObject* fun, PyTupleObject* xa , const int maxnofargs, + const int nofoptargs, int *nofargs, PyTupleObject **args, + const char *errmess) +{ PyObject *tmp = NULL; PyObject *tmp_fun = NULL; - int tot,opt,ext,siz,i,di=0; + Py_ssize_t tot, opt, ext, siz, i, di = 0; CFUNCSMESS(\"create_cb_arglist\\n\"); tot=opt=ext=siz=0; /* Get the total number of arguments */ @@ -1103,10 +1165,15 @@ Py_INCREF(tmp_fun); } } -if (tmp_fun==NULL) { -fprintf(stderr,\"Call-back argument must be function|instance|instance.__call__|f2py-function but got %s.\\n\",(fun==NULL?\"NULL\":Py_TYPE(fun)->tp_name)); -goto capi_fail; -} + + if (tmp_fun == NULL) { + fprintf(stderr, + \"Call-back argument must be function|instance|instance.__call__|f2py-function \" + \"but got %s.\\n\", + ((fun == NULL) ? \"NULL\" : Py_TYPE(fun)->tp_name)); + goto capi_fail; + } + if (PyObject_HasAttrString(tmp_fun,\"__code__\")) { if (PyObject_HasAttrString(tmp = PyObject_GetAttrString(tmp_fun,\"__code__\"),\"co_argcount\")) { PyObject *tmp_argcount = PyObject_GetAttrString(tmp,\"co_argcount\"); @@ -1114,7 +1181,7 @@ if (tmp_argcount == NULL) { goto capi_fail; } - tot = PyInt_AsLong(tmp_argcount) - di; + tot = PyLong_AsSsize_t(tmp_argcount) - di; Py_DECREF(tmp_argcount); } } @@ -1130,13 +1197,23 @@ /* Calculate the size of call-backs argument list */ siz = MIN(maxnofargs+ext,tot); *nofargs = MAX(0,siz-ext); + #ifdef DEBUGCFUNCS - fprintf(stderr,\"debug-capi:create_cb_arglist:maxnofargs(-nofoptargs),tot,opt,ext,siz,nofargs=%d(-%d),%d,%d,%d,%d,%d\\n\",maxnofargs,nofoptargs,tot,opt,ext,siz,*nofargs); + fprintf(stderr, + \"debug-capi:create_cb_arglist:maxnofargs(-nofoptargs),\" + \"tot,opt,ext,siz,nofargs = %d(-%d), %zd, %zd, %zd, %zd, %d\\n\", + maxnofargs, nofoptargs, tot, opt, ext, siz, *nofargs); #endif - if (siz>> getlincoef('2*x + 1', {'x'}) + (2, 1, 'x') + >>> getlincoef('3*x + x*2 + 2 + 1', {'x'}) + (5, 3, 'x') + >>> getlincoef('0', {'x'}) + (0, 0, None) + >>> getlincoef('0*x', {'x'}) + (0, 0, 'x') + >>> getlincoef('x*x', {'x'}) + (None, None, None) + + This can be tricked by sufficiently complex expressions + + >>> getlincoef('(x - 0.5)*(x - 1.5)*(x - 1)*x + 2*x + 3', {'x'}) + (2.0, 3.0, 'x') + """ try: c = int(myeval(e, {}, {})) return 0, c, None @@ -2156,6 +2177,15 @@ def getlincoef(e, xset): # e = a*x+b ; x in xset m1 = re_1.match(ee) c2 = myeval(ee, {}, {}) if (a * 0.5 + b == c and a * 1.5 + b == c2): + # gh-8062: return integers instead of floats if possible. + try: + a = int(a) + except: + pass + try: + b = int(b) + except: + pass return a, b, x except Exception: pass @@ -2166,6 +2196,37 @@ def getlincoef(e, xset): # e = a*x+b ; x in xset def getarrlen(dl, args, star='*'): + """ + Parameters + ---------- + dl : sequence of two str objects + dimensions of the array + args : Iterable[str] + symbols used in the expression + star : Any + unused + + Returns + ------- + expr : str + Some numeric expression as a string + arg : Optional[str] + If understood, the argument from `args` present in `expr` + expr2 : Optional[str] + If understood, an expression fragment that should be used as + ``"(%s%s".format(something, expr2)``. + + Examples + -------- + >>> getarrlen(['10*x + 20', '40*x'], {'x'}) + ('30 * x - 19', 'x', '+19)/(30)') + >>> getarrlen(['1', '10*x + 20'], {'x'}) + ('10 * x + 20', 'x', '-20)/(10)') + >>> getarrlen(['10*x + 20', '1'], {'x'}) + ('-10 * x - 18', 'x', '+18)/(-10)') + >>> getarrlen(['20', '1'], {'x'}) + ('-18', None, None) + """ edl = [] try: edl.append(myeval(dl[0], {}, {})) diff --git a/numpy/f2py/f2py2e.py b/numpy/f2py/f2py2e.py index 71a049e410f6..be2c345d1a99 100755 --- a/numpy/f2py/f2py2e.py +++ b/numpy/f2py/f2py2e.py @@ -604,7 +604,7 @@ def run_compile(): if modulename: break - extra_objects, sources = filter_files('', '[.](o|a|so)', sources) + extra_objects, sources = filter_files('', '[.](o|a|so|dylib)', sources) include_dirs, sources = filter_files('-I', '', sources, remove_prefix=1) library_dirs, sources = filter_files('-L', '', sources, remove_prefix=1) libraries, sources = filter_files('-l', '', sources, remove_prefix=1) @@ -650,7 +650,9 @@ def run_compile(): sys.argv.extend(['build', '--build-temp', build_dir, '--build-base', build_dir, - '--build-platlib', '.']) + '--build-platlib', '.', + # disable CCompilerOpt + '--disable-optimization']) if fc_flags: sys.argv.extend(['config_fc'] + fc_flags) if flib_flags: diff --git a/numpy/f2py/setup.py b/numpy/f2py/setup.py index 80b47e527aab..0a35db477494 100644 --- a/numpy/f2py/setup.py +++ b/numpy/f2py/setup.py @@ -30,6 +30,7 @@ def configuration(parent_package='', top_path=None): config.add_data_files( 'src/fortranobject.c', 'src/fortranobject.h') + config.add_data_files('*.pyi') return config diff --git a/numpy/f2py/src/fortranobject.c b/numpy/f2py/src/fortranobject.c index aa46c57d0b92..3275f90ad2cb 100644 --- a/numpy/f2py/src/fortranobject.c +++ b/numpy/f2py/src/fortranobject.c @@ -213,6 +213,8 @@ format_def(char *buf, Py_ssize_t size, FortranDataDef def) return -1; } memcpy(p, notalloc, sizeof(notalloc)); + p += sizeof(notalloc); + size -= sizeof(notalloc); } return p - buf; @@ -255,7 +257,7 @@ fortran_doc(FortranDataDef def) } else { PyArray_Descr *d = PyArray_DescrFromType(def.type); - n = PyOS_snprintf(p, size, "'%c'-", d->type); + n = PyOS_snprintf(p, size, "%s : '%c'-", def.name, d->type); Py_DECREF(d); if (n < 0 || n >= size) { goto fail; @@ -264,7 +266,7 @@ fortran_doc(FortranDataDef def) size -= n; if (def.data == NULL) { - n = format_def(p, size, def) == -1; + n = format_def(p, size, def); if (n < 0) { goto fail; } @@ -288,6 +290,7 @@ fortran_doc(FortranDataDef def) p += n; size -= n; } + } if (size <= 1) { goto fail; diff --git a/numpy/f2py/tests/src/module_data/mod.mod b/numpy/f2py/tests/src/module_data/mod.mod new file mode 100644 index 000000000000..8670a97e911c Binary files /dev/null and b/numpy/f2py/tests/src/module_data/mod.mod differ diff --git a/numpy/f2py/tests/src/module_data/module_data_docstring.f90 b/numpy/f2py/tests/src/module_data/module_data_docstring.f90 new file mode 100644 index 000000000000..4505e0cbc31e --- /dev/null +++ b/numpy/f2py/tests/src/module_data/module_data_docstring.f90 @@ -0,0 +1,12 @@ +module mod + integer :: i + integer :: x(4) + real, dimension(2,3) :: a + real, allocatable, dimension(:,:) :: b +contains + subroutine foo + integer :: k + k = 1 + a(1,2) = a(1,2)+3 + end subroutine foo +end module mod diff --git a/numpy/f2py/tests/test_block_docstring.py b/numpy/f2py/tests/test_block_docstring.py index e431f5ba6c97..7d725165b2fb 100644 --- a/numpy/f2py/tests/test_block_docstring.py +++ b/numpy/f2py/tests/test_block_docstring.py @@ -19,5 +19,5 @@ class TestBlockDocString(util.F2PyTest): @pytest.mark.xfail(IS_PYPY, reason="PyPy cannot modify tp_doc after PyType_Ready") def test_block_docstring(self): - expected = "'i'-array(2,3)\n" + expected = "bar : 'i'-array(2,3)\n" assert_equal(self.module.block.__doc__, expected) diff --git a/numpy/f2py/tests/test_crackfortran.py b/numpy/f2py/tests/test_crackfortran.py index 735804024c9e..96adffc727b5 100644 --- a/numpy/f2py/tests/test_crackfortran.py +++ b/numpy/f2py/tests/test_crackfortran.py @@ -86,3 +86,25 @@ def test_defaultPublic(self, tmp_path): assert 'public' not in mod['vars']['a']['attrspec'] assert 'private' not in mod['vars']['seta']['attrspec'] assert 'public' in mod['vars']['seta']['attrspec'] + + +class TestArrayDimCalculation(util.F2PyTest): + # Issue gh-8062. Calculations that occur in the dimensions of fortran + # array declarations should be interpreted by f2py as integers not floats. + # Prior to fix, test fails as generated fortran wrapper does not compile. + code = """ + function test(n, a) + integer, intent(in) :: n + real(8), intent(out) :: a(0:2*n/2) + integer :: test + a(:) = n + test = 1 + endfunction + """ + + def test_issue_8062(self): + for n in (5, 11): + _, a = self.module.test(n) + assert(a.shape == (n+1,)) + assert_array_equal(a, n) + diff --git a/numpy/f2py/tests/test_module_doc.py b/numpy/f2py/tests/test_module_doc.py new file mode 100644 index 000000000000..4b9555cee1fc --- /dev/null +++ b/numpy/f2py/tests/test_module_doc.py @@ -0,0 +1,30 @@ +import os +import sys +import pytest +import textwrap + +from . import util +from numpy.testing import assert_equal, IS_PYPY + + +def _path(*a): + return os.path.join(*((os.path.dirname(__file__),) + a)) + + +class TestModuleDocString(util.F2PyTest): + sources = [_path('src', 'module_data', 'module_data_docstring.f90')] + + @pytest.mark.skipif(sys.platform=='win32', + reason='Fails with MinGW64 Gfortran (Issue #9673)') + @pytest.mark.xfail(IS_PYPY, + reason="PyPy cannot modify tp_doc after PyType_Ready") + def test_module_docstring(self): + assert_equal(self.module.mod.__doc__, + textwrap.dedent('''\ + i : 'i'-scalar + x : 'i'-array(4) + a : 'f'-array(2,3) + b : 'f'-array(-1,-1), not allocated\x00 + foo()\n + Wrapper for ``foo``.\n\n''') + ) diff --git a/numpy/f2py/tests/util.py b/numpy/f2py/tests/util.py index c5b06697d330..d5fa76fedf27 100644 --- a/numpy/f2py/tests/util.py +++ b/numpy/f2py/tests/util.py @@ -19,8 +19,6 @@ from numpy.testing import temppath from importlib import import_module -from hashlib import md5 - # # Maintaining a temporary module directory # diff --git a/numpy/fft/setup.py b/numpy/fft/setup.py index 9ed824e4f8ab..477948a0986b 100644 --- a/numpy/fft/setup.py +++ b/numpy/fft/setup.py @@ -14,6 +14,7 @@ def configuration(parent_package='',top_path=None): define_macros=defs, ) + config.add_data_files('*.pyi') return config if __name__ == '__main__': diff --git a/numpy/lib/arraysetops.py b/numpy/lib/arraysetops.py index 6a2ad004cbf0..6c6c1ff809ce 100644 --- a/numpy/lib/arraysetops.py +++ b/numpy/lib/arraysetops.py @@ -1,28 +1,17 @@ """ Set operations for arrays based on sorting. -:Contains: - unique, - isin, - ediff1d, - intersect1d, - setxor1d, - in1d, - union1d, - setdiff1d - -:Notes: +Notes +----- For floating point arrays, inaccurate results may appear due to usual round-off and floating point comparison issues. Speed could be gained in some operations by an implementation of -sort(), that can provide directly the permutation vectors, avoiding -thus calls to argsort(). +`numpy.sort`, that can provide directly the permutation vectors, thus avoiding +calls to `numpy.argsort`. -To do: Optionally return indices analogously to unique for all functions. - -:Author: Robert Cimrman +Original author: Robert Cimrman """ import functools @@ -104,7 +93,7 @@ def ediff1d(ary, to_end=None, to_begin=None): else: to_begin = np.asanyarray(to_begin) if not np.can_cast(to_begin, dtype_req, casting="same_kind"): - raise TypeError("dtype of `to_end` must be compatible " + raise TypeError("dtype of `to_begin` must be compatible " "with input `ary` under the `same_kind` rule.") to_begin = to_begin.ravel() diff --git a/numpy/lib/format.py b/numpy/lib/format.py index afbd3784a9ad..5d951e262570 100644 --- a/numpy/lib/format.py +++ b/numpy/lib/format.py @@ -746,7 +746,7 @@ def read_array(fp, allow_pickle=False, pickle_kwargs=None): # Friendlier error message raise UnicodeError("Unpickling a python object failed: %r\n" "You may need to pass the encoding= option " - "to numpy.load" % (err,)) + "to numpy.load" % (err,)) from err else: if isfileobj(fp): # We can use the fast fromfile() function. diff --git a/numpy/lib/function_base.py b/numpy/lib/function_base.py index c43b2fb531f3..984f3086e5b7 100644 --- a/numpy/lib/function_base.py +++ b/numpy/lib/function_base.py @@ -1450,7 +1450,7 @@ def angle(z, deg=False): The counterclockwise angle from the positive real axis on the complex plane in the range ``(-pi, pi]``, with dtype as numpy.float64. - ..versionchanged:: 1.16.0 + .. versionchanged:: 1.16.0 This function works on subclasses of ndarray like `ma.array`. See Also @@ -2268,13 +2268,13 @@ def _vectorize_call_with_signature(self, func, args): def _cov_dispatcher(m, y=None, rowvar=None, bias=None, ddof=None, - fweights=None, aweights=None): + fweights=None, aweights=None, *, dtype=None): return (m, y, fweights, aweights) @array_function_dispatch(_cov_dispatcher) def cov(m, y=None, rowvar=True, bias=False, ddof=None, fweights=None, - aweights=None): + aweights=None, *, dtype=None): """ Estimate a covariance matrix, given data and weights. @@ -2325,6 +2325,11 @@ def cov(m, y=None, rowvar=True, bias=False, ddof=None, fweights=None, weights can be used to assign probabilities to observation vectors. .. versionadded:: 1.10 + dtype : data-type, optional + Data-type of the result. By default, the return data-type will have + at least `numpy.float64` precision. + + .. versionadded:: 1.20 Returns ------- @@ -2400,13 +2405,16 @@ def cov(m, y=None, rowvar=True, bias=False, ddof=None, fweights=None, if m.ndim > 2: raise ValueError("m has more than 2 dimensions") - if y is None: - dtype = np.result_type(m, np.float64) - else: + if y is not None: y = np.asarray(y) if y.ndim > 2: raise ValueError("y has more than 2 dimensions") - dtype = np.result_type(m, y, np.float64) + + if dtype is None: + if y is None: + dtype = np.result_type(m, np.float64) + else: + dtype = np.result_type(m, y, np.float64) X = array(m, ndmin=2, dtype=dtype) if not rowvar and X.shape[0] != 1: @@ -2486,12 +2494,14 @@ def cov(m, y=None, rowvar=True, bias=False, ddof=None, fweights=None, return c.squeeze() -def _corrcoef_dispatcher(x, y=None, rowvar=None, bias=None, ddof=None): +def _corrcoef_dispatcher(x, y=None, rowvar=None, bias=None, ddof=None, *, + dtype=None): return (x, y) @array_function_dispatch(_corrcoef_dispatcher) -def corrcoef(x, y=None, rowvar=True, bias=np._NoValue, ddof=np._NoValue): +def corrcoef(x, y=None, rowvar=True, bias=np._NoValue, ddof=np._NoValue, *, + dtype=None): """ Return Pearson product-moment correlation coefficients. @@ -2525,6 +2535,11 @@ def corrcoef(x, y=None, rowvar=True, bias=np._NoValue, ddof=np._NoValue): Has no effect, do not use. .. deprecated:: 1.10.0 + dtype : data-type, optional + Data-type of the result. By default, the return data-type will have + at least `numpy.float64` precision. + + .. versionadded:: 1.20 Returns ------- @@ -2616,7 +2631,7 @@ def corrcoef(x, y=None, rowvar=True, bias=np._NoValue, ddof=np._NoValue): # 2015-03-15, 1.10 warnings.warn('bias and ddof have no effect and are deprecated', DeprecationWarning, stacklevel=3) - c = cov(x, y, rowvar) + c = cov(x, y, rowvar, dtype=dtype) try: d = diag(c) except ValueError: diff --git a/numpy/lib/npyio.py b/numpy/lib/npyio.py index 90e16643c013..805e59bc1ff4 100644 --- a/numpy/lib/npyio.py +++ b/numpy/lib/npyio.py @@ -86,7 +86,7 @@ def __getattribute__(self, key): try: return object.__getattribute__(self, '_obj')[key] except KeyError: - raise AttributeError(key) + raise AttributeError(key) from None def __dir__(self): """ @@ -446,9 +446,9 @@ def load(file, mmap_mode=None, allow_pickle=False, fix_imports=True, "when allow_pickle=False") try: return pickle.load(fid, **pickle_kwargs) - except Exception: + except Exception as e: raise IOError( - "Failed to interpret file %s as a pickle" % repr(file)) + "Failed to interpret file %s as a pickle" % repr(file)) from e def _save_dispatcher(file, arr, allow_pickle=None, fix_imports=None): @@ -1435,10 +1435,10 @@ def first_write(self, v): for row in X: try: v = format % tuple(row) + newline - except TypeError: + except TypeError as e: raise TypeError("Mismatch between array dtype ('%s') and " "format specifier ('%s')" - % (str(X.dtype), format)) + % (str(X.dtype), format)) from e fh.write(v) if len(footer) > 0: diff --git a/numpy/lib/polynomial.py b/numpy/lib/polynomial.py index 7b89eeb70367..0fd9bbd79c6a 100644 --- a/numpy/lib/polynomial.py +++ b/numpy/lib/polynomial.py @@ -426,7 +426,7 @@ def polyder(p, m=1): >>> np.polyder(p, 3) poly1d([6]) >>> np.polyder(p, 4) - poly1d([0.]) + poly1d([0]) """ m = int(m) @@ -754,11 +754,11 @@ def polyval(p, x): >>> np.polyval([3,0,1], 5) # 3 * 5**2 + 0 * 5**1 + 1 76 >>> np.polyval([3,0,1], np.poly1d(5)) - poly1d([76.]) + poly1d([76]) >>> np.polyval(np.poly1d([3,0,1]), 5) 76 >>> np.polyval(np.poly1d([3,0,1]), np.poly1d(5)) - poly1d([76.]) + poly1d([76]) """ p = NX.asarray(p) @@ -1236,7 +1236,7 @@ def __init__(self, c_or_r, r=False, variable=None): raise ValueError("Polynomial must be 1d only.") c_or_r = trim_zeros(c_or_r, trim='f') if len(c_or_r) == 0: - c_or_r = NX.array([0.]) + c_or_r = NX.array([0], dtype=c_or_r.dtype) self._coeffs = c_or_r if variable is None: variable = 'x' diff --git a/numpy/lib/recfunctions.py b/numpy/lib/recfunctions.py index cfc5dc9cae99..fbfbca73d442 100644 --- a/numpy/lib/recfunctions.py +++ b/numpy/lib/recfunctions.py @@ -513,7 +513,7 @@ def drop_fields(base, drop_names, usemask=True, asrecarray=False): Nested fields are supported. - ..versionchanged: 1.18.0 + .. versionchanged:: 1.18.0 `drop_fields` returns an array with 0 fields if all fields are dropped, rather than returning ``None`` as it did previously. diff --git a/numpy/lib/setup.py b/numpy/lib/setup.py index b3f441f38d3b..7520b72d7ac0 100644 --- a/numpy/lib/setup.py +++ b/numpy/lib/setup.py @@ -4,6 +4,7 @@ def configuration(parent_package='',top_path=None): config = Configuration('lib', parent_package, top_path) config.add_subpackage('tests') config.add_data_dir('tests/data') + config.add_data_files('*.pyi') return config if __name__ == '__main__': diff --git a/numpy/lib/stride_tricks.py b/numpy/lib/stride_tricks.py index 502235bdfbcd..d8a8b325e3a4 100644 --- a/numpy/lib/stride_tricks.py +++ b/numpy/lib/stride_tricks.py @@ -6,9 +6,9 @@ """ import numpy as np -from numpy.core.overrides import array_function_dispatch +from numpy.core.overrides import array_function_dispatch, set_module -__all__ = ['broadcast_to', 'broadcast_arrays'] +__all__ = ['broadcast_to', 'broadcast_arrays', 'broadcast_shapes'] class DummyArray: @@ -165,6 +165,12 @@ def broadcast_to(array, shape, subok=False): If the array is not compatible with the new shape according to NumPy's broadcasting rules. + See Also + -------- + broadcast + broadcast_arrays + broadcast_shapes + Notes ----- .. versionadded:: 1.10.0 @@ -197,6 +203,49 @@ def _broadcast_shape(*args): return b.shape +@set_module('numpy') +def broadcast_shapes(*args): + """ + Broadcast the input shapes into a single shape. + + :ref:`Learn more about broadcasting here `. + + .. versionadded:: 1.20.0 + + Parameters + ---------- + `*args` : tuples of ints, or ints + The shapes to be broadcast against each other. + + Returns + ------- + tuple + Broadcasted shape. + + Raises + ------ + ValueError + If the shapes are not compatible and cannot be broadcast according + to NumPy's broadcasting rules. + + See Also + -------- + broadcast + broadcast_arrays + broadcast_to + + Examples + -------- + >>> np.broadcast_shapes((1, 2), (3, 1), (3, 2)) + (3, 2) + + >>> np.broadcast_shapes((6, 7), (5, 6, 1), (7,), (5, 1, 7)) + (5, 6, 7) + """ + arrays = [np.empty(x, dtype=[]) for x in args] + return _broadcast_shape(*arrays) + + def _broadcast_arrays_dispatcher(*args, subok=None): return args @@ -230,6 +279,12 @@ def broadcast_arrays(*args, subok=False): warning will be emitted. A future version will set the ``writable`` flag False so writing to it will raise an error. + See Also + -------- + broadcast + broadcast_to + broadcast_shapes + Examples -------- >>> x = np.array([[1,2,3]]) diff --git a/numpy/lib/tests/test_arraysetops.py b/numpy/lib/tests/test_arraysetops.py index 81ba789e30d6..847e6cb8a3e7 100644 --- a/numpy/lib/tests/test_arraysetops.py +++ b/numpy/lib/tests/test_arraysetops.py @@ -125,32 +125,36 @@ def test_ediff1d(self): assert_array_equal([7, 1], ediff1d(two_elem, to_begin=7)) assert_array_equal([5, 6, 1], ediff1d(two_elem, to_begin=[5, 6])) - @pytest.mark.parametrize("ary, prepend, append", [ + @pytest.mark.parametrize("ary, prepend, append, expected", [ # should fail because trying to cast # np.nan standard floating point value # into an integer array: (np.array([1, 2, 3], dtype=np.int64), None, - np.nan), + np.nan, + 'to_end'), # should fail because attempting # to downcast to int type: (np.array([1, 2, 3], dtype=np.int64), np.array([5, 7, 2], dtype=np.float32), - None), + None, + 'to_begin'), # should fail because attempting to cast # two special floating point values - # to integers (on both sides of ary): + # to integers (on both sides of ary), + # `to_begin` is in the error message as the impl checks this first: (np.array([1., 3., 9.], dtype=np.int8), np.nan, - np.nan), + np.nan, + 'to_begin'), ]) - def test_ediff1d_forbidden_type_casts(self, ary, prepend, append): + def test_ediff1d_forbidden_type_casts(self, ary, prepend, append, expected): # verify resolution of gh-11490 # specifically, raise an appropriate # Exception when attempting to append or # prepend with an incompatible type - msg = 'must be compatible' + msg = 'dtype of `{}` must be compatible'.format(expected) with assert_raises_regex(TypeError, msg): ediff1d(ary=ary, to_end=append, diff --git a/numpy/lib/tests/test_function_base.py b/numpy/lib/tests/test_function_base.py index 7bddb941c5c8..4c7c0480c106 100644 --- a/numpy/lib/tests/test_function_base.py +++ b/numpy/lib/tests/test_function_base.py @@ -2023,6 +2023,12 @@ def test_extreme(self): assert_array_almost_equal(c, np.array([[1., -1.], [-1., 1.]])) assert_(np.all(np.abs(c) <= 1.0)) + @pytest.mark.parametrize("test_type", [np.half, np.single, np.double, np.longdouble]) + def test_corrcoef_dtype(self, test_type): + cast_A = self.A.astype(test_type) + res = corrcoef(cast_A, dtype=test_type) + assert test_type == res.dtype + class TestCov: x1 = np.array([[0, 2], [1, 1], [2, 0]]).T @@ -2123,6 +2129,12 @@ def test_unit_fweights_and_aweights(self): aweights=self.unit_weights), self.res1) + @pytest.mark.parametrize("test_type", [np.half, np.single, np.double, np.longdouble]) + def test_cov_dtype(self, test_type): + cast_x1 = self.x1.astype(test_type) + res = cov(cast_x1, dtype=test_type) + assert test_type == res.dtype + class Test_I0: diff --git a/numpy/lib/tests/test_polynomial.py b/numpy/lib/tests/test_polynomial.py index ab6691b4384f..6c3e4fa02212 100644 --- a/numpy/lib/tests/test_polynomial.py +++ b/numpy/lib/tests/test_polynomial.py @@ -227,6 +227,20 @@ def test_poly_int_overflow(self): v = np.arange(1, 21) assert_almost_equal(np.poly(v), np.poly(np.diag(v))) + def test_zero_poly_dtype(self): + """ + Regression test for gh-16354. + """ + z = np.array([0, 0, 0]) + p = np.poly1d(z.astype(np.int64)) + assert_equal(p.coeffs.dtype, np.int64) + + p = np.poly1d(z.astype(np.float32)) + assert_equal(p.coeffs.dtype, np.float32) + + p = np.poly1d(z.astype(np.complex64)) + assert_equal(p.coeffs.dtype, np.complex64) + def test_poly_eq(self): p = np.poly1d([1, 2, 3]) p2 = np.poly1d([1, 2, 4]) diff --git a/numpy/lib/tests/test_stride_tricks.py b/numpy/lib/tests/test_stride_tricks.py index 9d95eb9d0321..10d7a19abec0 100644 --- a/numpy/lib/tests/test_stride_tricks.py +++ b/numpy/lib/tests/test_stride_tricks.py @@ -5,7 +5,8 @@ assert_raises_regex, assert_warns, ) from numpy.lib.stride_tricks import ( - as_strided, broadcast_arrays, _broadcast_shape, broadcast_to + as_strided, broadcast_arrays, _broadcast_shape, broadcast_to, + broadcast_shapes, ) def assert_shapes_correct(input_shapes, expected_shape): @@ -274,7 +275,9 @@ def test_broadcast_to_raises(): def test_broadcast_shape(): - # broadcast_shape is already exercized indirectly by broadcast_arrays + # tests internal _broadcast_shape + # _broadcast_shape is already exercised indirectly by broadcast_arrays + # _broadcast_shape is also exercised by the public broadcast_shapes function assert_equal(_broadcast_shape(), ()) assert_equal(_broadcast_shape([1, 2]), (2,)) assert_equal(_broadcast_shape(np.ones((1, 1))), (1, 1)) @@ -288,6 +291,64 @@ def test_broadcast_shape(): assert_raises(ValueError, lambda: _broadcast_shape(*bad_args)) +def test_broadcast_shapes_succeeds(): + # tests public broadcast_shapes + data = [ + [[], ()], + [[()], ()], + [[(7,)], (7,)], + [[(1, 2), (2,)], (1, 2)], + [[(1, 1)], (1, 1)], + [[(1, 1), (3, 4)], (3, 4)], + [[(6, 7), (5, 6, 1), (7,), (5, 1, 7)], (5, 6, 7)], + [[(5, 6, 1)], (5, 6, 1)], + [[(1, 3), (3, 1)], (3, 3)], + [[(1, 0), (0, 0)], (0, 0)], + [[(0, 1), (0, 0)], (0, 0)], + [[(1, 0), (0, 1)], (0, 0)], + [[(1, 1), (0, 0)], (0, 0)], + [[(1, 1), (1, 0)], (1, 0)], + [[(1, 1), (0, 1)], (0, 1)], + [[(), (0,)], (0,)], + [[(0,), (0, 0)], (0, 0)], + [[(0,), (0, 1)], (0, 0)], + [[(1,), (0, 0)], (0, 0)], + [[(), (0, 0)], (0, 0)], + [[(1, 1), (0,)], (1, 0)], + [[(1,), (0, 1)], (0, 1)], + [[(1,), (1, 0)], (1, 0)], + [[(), (1, 0)], (1, 0)], + [[(), (0, 1)], (0, 1)], + [[(1,), (3,)], (3,)], + [[2, (3, 2)], (3, 2)], + ] + for input_shapes, target_shape in data: + assert_equal(broadcast_shapes(*input_shapes), target_shape) + + assert_equal(broadcast_shapes(*([(1, 2)] * 32)), (1, 2)) + assert_equal(broadcast_shapes(*([(1, 2)] * 100)), (1, 2)) + + # regression tests for gh-5862 + assert_equal(broadcast_shapes(*([(2,)] * 32)), (2,)) + + +def test_broadcast_shapes_raises(): + # tests public broadcast_shapes + data = [ + [(3,), (4,)], + [(2, 3), (2,)], + [(3,), (3,), (4,)], + [(1, 3, 4), (2, 3, 3)], + [(1, 2), (3,1), (3,2), (10, 5)], + [2, (2, 3)], + ] + for input_shapes in data: + assert_raises(ValueError, lambda: broadcast_shapes(*input_shapes)) + + bad_args = [(2,)] * 32 + [(3,)] * 32 + assert_raises(ValueError, lambda: broadcast_shapes(*bad_args)) + + def test_as_strided(): a = np.array([None]) a_view = as_strided(a) diff --git a/numpy/lib/tests/test_utils.py b/numpy/lib/tests/test_utils.py index 261cfef5d680..33951b92ade9 100644 --- a/numpy/lib/tests/test_utils.py +++ b/numpy/lib/tests/test_utils.py @@ -140,3 +140,22 @@ def test_strided(self): def test_assert_raises_regex_context_manager(): with assert_raises_regex(ValueError, 'no deprecation warning'): raise ValueError('no deprecation warning') + + +def test_info_method_heading(): + # info(class) should only print "Methods:" heading if methods exist + + class NoPublicMethods: + pass + + class WithPublicMethods: + def first_method(): + pass + + def _has_method_heading(cls): + out = StringIO() + utils.info(cls, output=out) + return 'Methods:' in out.getvalue() + + assert _has_method_heading(WithPublicMethods) + assert not _has_method_heading(NoPublicMethods) diff --git a/numpy/lib/utils.py b/numpy/lib/utils.py index d511c2a400e9..5447608bf5bb 100644 --- a/numpy/lib/utils.py +++ b/numpy/lib/utils.py @@ -587,11 +587,11 @@ def info(object=None, maxwidth=76, output=sys.stdout, toplevel='numpy'): print(inspect.getdoc(object), file=output) methods = pydoc.allmethods(object) - if methods != []: + + public_methods = [meth for meth in methods if meth[0] != '_'] + if public_methods: print("\n\nMethods:\n", file=output) - for meth in methods: - if meth[0] == '_': - continue + for meth in public_methods: thisobj = getattr(object, meth, None) if thisobj is not None: methstr, other = pydoc.splitdoc( diff --git a/numpy/linalg/lapack_lite/make_lite.py b/numpy/linalg/lapack_lite/make_lite.py index 23921acf4bd5..cf15b2541361 100755 --- a/numpy/linalg/lapack_lite/make_lite.py +++ b/numpy/linalg/lapack_lite/make_lite.py @@ -81,7 +81,7 @@ def dependencies(self): return self._dependencies def __repr__(self): - return "FortranRoutine({!r}, filename={!r})".format(self.name, self.filename) + return f'FortranRoutine({self.name!r}, filename={self.filename!r})' class UnknownFortranRoutine(FortranRoutine): """Wrapper for a Fortran routine for which the corresponding file @@ -193,7 +193,7 @@ def allRoutinesByType(self, typename): def printRoutineNames(desc, routines): print(desc) for r in routines: - print('\t%s' % r.name) + print(f'\t{r.name}') def getLapackRoutines(wrapped_routines, ignores, lapack_dir): blas_src_dir = os.path.join(lapack_dir, 'BLAS', 'SRC') @@ -243,7 +243,7 @@ def dumpRoutineNames(library, output_dir): with open(filename, 'w') as fo: for r in routines: deps = r.dependencies() - fo.write('%s: %s\n' % (r.name, ' '.join(deps))) + fo.write(f"{r.name}: {' '.join(deps)}\n") def concatenateRoutines(routines, output_file): with open(output_file, 'w') as output_fo: @@ -316,13 +316,13 @@ def create_name_header(output_dir): # Rename BLAS/LAPACK symbols for name in sorted(symbols): - f.write("#define %s_ BLAS_FUNC(%s)\n" % (name, name)) + f.write(f'#define {name}_ BLAS_FUNC({name})\n') # Rename also symbols that f2c exports itself f.write("\n" "/* Symbols exported by f2c.c */\n") for name in sorted(f2c_symbols): - f.write("#define %s numpy_lapack_lite_%s\n" % (name, name)) + f.write(f'#define {name} numpy_lapack_lite_{name}\n') def main(): if len(sys.argv) != 3: @@ -348,9 +348,9 @@ def main(): dumpRoutineNames(library, output_dir) for typename in types: - fortran_file = os.path.join(output_dir, 'f2c_%s.f' % typename) + fortran_file = os.path.join(output_dir, f'f2c_{typename}.f') c_file = fortran_file[:-2] + '.c' - print('creating %s ...' % c_file) + print(f'creating {c_file} ...') routines = library.allRoutinesByType(typename) concatenateRoutines(routines, fortran_file) @@ -358,11 +358,11 @@ def main(): patch_file = os.path.basename(fortran_file) + '.patch' if os.path.exists(patch_file): subprocess.check_call(['patch', '-u', fortran_file, patch_file]) - print("Patched {}".format(fortran_file)) + print(f'Patched {fortran_file}') try: runF2C(fortran_file, output_dir) except F2CError: - print('f2c failed on %s' % fortran_file) + print(f'f2c failed on {fortran_file}') break scrubF2CSource(c_file) diff --git a/numpy/linalg/linalg.py b/numpy/linalg/linalg.py index 92f93d671a12..5970a00baca3 100644 --- a/numpy/linalg/linalg.py +++ b/numpy/linalg/linalg.py @@ -901,7 +901,7 @@ def qr(a, mode='reduced'): warnings.warn(msg, DeprecationWarning, stacklevel=3) mode = 'economic' else: - raise ValueError("Unrecognized mode '%s'" % mode) + raise ValueError(f"Unrecognized mode '{mode}'") a, wrap = _makearray(a) _assert_2d(a) @@ -2206,8 +2206,8 @@ def lstsq(a, b, rcond="warn"): Least-squares solution. If `b` is two-dimensional, the solutions are in the `K` columns of `x`. residuals : {(1,), (K,), (0,)} ndarray - Sums of residuals; squared Euclidean 2-norm for each column in - ``b - a*x``. + Sums of squared residuals: Squared Euclidean 2-norm for each column in + ``b - a @ x``. If the rank of `a` is < N or M <= N, this is an empty array. If `b` is 1-dimensional, this is a (1,) shape array. Otherwise the shape is (K,). @@ -2558,7 +2558,7 @@ def norm(x, ord=None, axis=None, keepdims=False): # special case for speedup s = (x.conj() * x).real return sqrt(add.reduce(s, axis=axis, keepdims=keepdims)) - # None of the str-type keywords for ord ('fro', 'nuc') + # None of the str-type keywords for ord ('fro', 'nuc') # are valid for vectors elif isinstance(ord, str): raise ValueError(f"Invalid norm order '{ord}' for vectors") diff --git a/numpy/linalg/setup.py b/numpy/linalg/setup.py index bb070ed9d892..5c9f2a4cb56c 100644 --- a/numpy/linalg/setup.py +++ b/numpy/linalg/setup.py @@ -80,6 +80,7 @@ def get_lapack_lite_sources(ext, build_dir): extra_info=lapack_info, libraries=['npymath'], ) + config.add_data_files('*.pyi') return config if __name__ == '__main__': diff --git a/numpy/linalg/tests/test_build.py b/numpy/linalg/tests/test_build.py index cbf3089bcf5f..4859226d99a4 100644 --- a/numpy/linalg/tests/test_build.py +++ b/numpy/linalg/tests/test_build.py @@ -16,13 +16,13 @@ def __init__(self): p = Popen(self.cmd, stdout=PIPE, stderr=PIPE) stdout, stderr = p.communicate() except OSError: - raise RuntimeError("command %s cannot be run" % self.cmd) + raise RuntimeError(f'command {self.cmd} cannot be run') def get_dependencies(self, lfile): p = Popen(self.cmd + [lfile], stdout=PIPE, stderr=PIPE) stdout, stderr = p.communicate() if not (p.returncode == 0): - raise RuntimeError("failed dependencies check for %s" % lfile) + raise RuntimeError(f'failed dependencies check for {lfile}') return stdout diff --git a/numpy/linalg/tests/test_linalg.py b/numpy/linalg/tests/test_linalg.py index 3f3bf9f70917..21fab58e1d42 100644 --- a/numpy/linalg/tests/test_linalg.py +++ b/numpy/linalg/tests/test_linalg.py @@ -85,7 +85,7 @@ def check(self, do): do(self.a, self.b, tags=self.tags) def __repr__(self): - return "" % (self.name,) + return f'' def apply_tag(tag, cases): @@ -349,7 +349,7 @@ def check_cases(self, require=set(), exclude=set()): try: case.check(self.do) except Exception: - msg = "In test case: %r\n\n" % case + msg = f'In test case: {case!r}\n\n' msg += traceback.format_exc() raise AssertionError(msg) @@ -1732,7 +1732,7 @@ def test_basic_property(self): b = np.matmul(c, c.transpose(t).conj()) assert_allclose(b, a, - err_msg="{} {}\n{}\n{}".format(shape, dtype, a, c), + err_msg=f'{shape} {dtype}\n{a}\n{c}', atol=500 * a.shape[0] * np.finfo(dtype).eps) def test_0_size(self): diff --git a/numpy/ma/bench.py b/numpy/ma/bench.py index 83cc6aea7fb2..e29d54365c33 100644 --- a/numpy/ma/bench.py +++ b/numpy/ma/bench.py @@ -58,7 +58,7 @@ def compare_functions_1v(func, nloop=500, xs=xs, nmxs=nmxs, xl=xl, nmxl=nmxl): funcname = func.__name__ print("-"*50) - print("%s on small arrays" % funcname) + print(f'{funcname} on small arrays') module, data = "numpy.ma", "nmxs" timer("%(module)s.%(funcname)s(%(data)s)" % locals(), v="%11s" % module, nloop=nloop) @@ -70,8 +70,8 @@ def compare_functions_1v(func, nloop=500, def compare_methods(methodname, args, vars='x', nloop=500, test=True, xs=xs, nmxs=nmxs, xl=xl, nmxl=nmxl): print("-"*50) - print("%s on small arrays" % methodname) - data, ver = "nm%ss" % vars, 'numpy.ma' + print(f'{methodname} on small arrays') + data, ver = f'nm{vars}l', 'numpy.ma' timer("%(data)s.%(methodname)s(%(args)s)" % locals(), v=ver, nloop=nloop) print("%s on large arrays" % methodname) @@ -86,11 +86,11 @@ def compare_functions_2v(func, nloop=500, test=True, yl=yl, nmyl=nmyl): funcname = func.__name__ print("-"*50) - print("%s on small arrays" % funcname) + print(f'{funcname} on small arrays') module, data = "numpy.ma", "nmxs,nmys" timer("%(module)s.%(funcname)s(%(data)s)" % locals(), v="%11s" % module, nloop=nloop) - print("%s on large arrays" % funcname) + print(f'{funcname} on large arrays') module, data = "numpy.ma", "nmxl,nmyl" timer("%(module)s.%(funcname)s(%(data)s)" % locals(), v="%11s" % module, nloop=nloop) return diff --git a/numpy/ma/core.py b/numpy/ma/core.py index b5371f51a6bc..313d9e0b9b16 100644 --- a/numpy/ma/core.py +++ b/numpy/ma/core.py @@ -443,9 +443,9 @@ def _check_fill_value(fill_value, ndtype): if isinstance(fill_value, (ndarray, np.void)): try: fill_value = np.array(fill_value, copy=False, dtype=ndtype) - except ValueError: + except ValueError as e: err_msg = "Unable to transform %s to dtype %s" - raise ValueError(err_msg % (fill_value, ndtype)) + raise ValueError(err_msg % (fill_value, ndtype)) from e else: fill_value = np.asarray(fill_value, dtype=object) fill_value = np.array(_recursive_set_fill_value(fill_value, ndtype), @@ -460,12 +460,12 @@ def _check_fill_value(fill_value, ndtype): # Also in case of converting string arrays. try: fill_value = np.array(fill_value, copy=False, dtype=ndtype) - except (OverflowError, ValueError): + except (OverflowError, ValueError) as e: # Raise TypeError instead of OverflowError or ValueError. # OverflowError is seldom used, and the real problem here is # that the passed fill_value is not compatible with the ndtype. err_msg = "Cannot convert fill_value %s to dtype %s" - raise TypeError(err_msg % (fill_value, ndtype)) + raise TypeError(err_msg % (fill_value, ndtype)) from e return np.array(fill_value) @@ -5389,7 +5389,7 @@ def round(self, decimals=0, out=None): See Also -------- - numpy.ndarray.around : corresponding function for ndarrays + numpy.ndarray.round : corresponding function for ndarrays numpy.around : equivalent function """ result = self._data.round(decimals=decimals, out=out).view(type(self)) diff --git a/numpy/ma/extras.py b/numpy/ma/extras.py index 8ede29da18af..1bf03e966326 100644 --- a/numpy/ma/extras.py +++ b/numpy/ma/extras.py @@ -901,11 +901,11 @@ def compress_rows(a): Suppress whole rows of a 2-D array that contain masked values. This is equivalent to ``np.ma.compress_rowcols(a, 0)``, see - `extras.compress_rowcols` for details. + `compress_rowcols` for details. See Also -------- - extras.compress_rowcols + compress_rowcols """ a = asarray(a) @@ -918,11 +918,11 @@ def compress_cols(a): Suppress whole columns of a 2-D array that contain masked values. This is equivalent to ``np.ma.compress_rowcols(a, 1)``, see - `extras.compress_rowcols` for details. + `compress_rowcols` for details. See Also -------- - extras.compress_rowcols + compress_rowcols """ a = asarray(a) @@ -1641,7 +1641,7 @@ def flatnotmasked_contiguous(a): slice_list : list A sorted sequence of `slice` objects (start index, end index). - ..versionchanged:: 1.15.0 + .. versionchanged:: 1.15.0 Now returns an empty list instead of None for a fully masked array See Also diff --git a/numpy/ma/mrecords.py b/numpy/ma/mrecords.py index cd93a9a14313..70087632e9f6 100644 --- a/numpy/ma/mrecords.py +++ b/numpy/ma/mrecords.py @@ -60,7 +60,7 @@ def _checknames(descr, names=None): elif isinstance(names, str): new_names = names.split(',') else: - raise NameError("illegal input names %s" % repr(names)) + raise NameError(f'illegal input names {names!r}') nnames = len(new_names) if nnames < ndescr: new_names += default_names[nnames:] @@ -198,8 +198,8 @@ def __getattribute__(self, attr): fielddict = ndarray.__getattribute__(self, 'dtype').fields try: res = fielddict[attr][:2] - except (TypeError, KeyError): - raise AttributeError("record array has no attribute %s" % attr) + except (TypeError, KeyError) as e: + raise AttributeError(f'record array has no attribute {attr}') from e # So far, so good _localdict = ndarray.__getattribute__(self, '__dict__') _data = ndarray.view(self, _localdict['_baseclass']) @@ -274,7 +274,7 @@ def __setattr__(self, attr, val): try: res = fielddict[attr][:2] except (TypeError, KeyError): - raise AttributeError("record array has no attribute %s" % attr) + raise AttributeError(f'record array has no attribute {attr}') if val is masked: _fill_value = _localdict['_fill_value'] @@ -337,13 +337,13 @@ def __str__(self): """ if self.size > 1: - mstr = ["(%s)" % ",".join([str(i) for i in s]) + mstr = [f"({','.join([str(i) for i in s])})" for s in zip(*[getattr(self, f) for f in self.dtype.names])] - return "[%s]" % ", ".join(mstr) + return f"[{', '.join(mstr)}]" else: - mstr = ["%s" % ",".join([str(i) for i in s]) + mstr = [f"{','.join([str(i) for i in s])}" for s in zip([getattr(self, f) for f in self.dtype.names])] - return "(%s)" % ", ".join(mstr) + return f"({', '.join(mstr)})" def __repr__(self): """ @@ -657,7 +657,7 @@ def openfile(fname): try: f = open(fname) except IOError: - raise IOError("No such file: '%s'" % fname) + raise IOError(f"No such file: '{fname}'") if f.readline()[:2] != "\\x": f.seek(0, 0) return f diff --git a/numpy/ma/setup.py b/numpy/ma/setup.py index d3f34c874607..018d38cdd500 100644 --- a/numpy/ma/setup.py +++ b/numpy/ma/setup.py @@ -3,6 +3,7 @@ def configuration(parent_package='',top_path=None): from numpy.distutils.misc_util import Configuration config = Configuration('ma', parent_package, top_path) config.add_subpackage('tests') + config.add_data_files('*.pyi') return config if __name__ == "__main__": diff --git a/numpy/ma/tests/test_core.py b/numpy/ma/tests/test_core.py index 0ed2971e6bd2..6a2ed8ca75ea 100644 --- a/numpy/ma/tests/test_core.py +++ b/numpy/ma/tests/test_core.py @@ -2748,7 +2748,7 @@ def test_inplace_addition_scalar_type(self): xm += t(1) assert_equal(xm, y + t(1)) - assert_equal(len(w), 0, "Failed on type=%s." % t) + assert_equal(len(w), 0, f'Failed on type={t}.') def test_inplace_addition_array_type(self): # Test of inplace additions @@ -2765,7 +2765,7 @@ def test_inplace_addition_array_type(self): assert_equal(xm, y + a) assert_equal(xm.mask, mask_or(m, a.mask)) - assert_equal(len(w), 0, "Failed on type=%s." % t) + assert_equal(len(w), 0, f'Failed on type={t}.') def test_inplace_subtraction_scalar_type(self): # Test of inplace subtractions @@ -2778,7 +2778,7 @@ def test_inplace_subtraction_scalar_type(self): xm -= t(1) assert_equal(xm, y - t(1)) - assert_equal(len(w), 0, "Failed on type=%s." % t) + assert_equal(len(w), 0, f'Failed on type={t}.') def test_inplace_subtraction_array_type(self): # Test of inplace subtractions @@ -2795,7 +2795,7 @@ def test_inplace_subtraction_array_type(self): assert_equal(xm, y - a) assert_equal(xm.mask, mask_or(m, a.mask)) - assert_equal(len(w), 0, "Failed on type=%s." % t) + assert_equal(len(w), 0, f'Failed on type={t}.') def test_inplace_multiplication_scalar_type(self): # Test of inplace multiplication @@ -2808,7 +2808,7 @@ def test_inplace_multiplication_scalar_type(self): xm *= t(2) assert_equal(xm, y * t(2)) - assert_equal(len(w), 0, "Failed on type=%s." % t) + assert_equal(len(w), 0, f'Failed on type={t}.') def test_inplace_multiplication_array_type(self): # Test of inplace multiplication @@ -2825,7 +2825,7 @@ def test_inplace_multiplication_array_type(self): assert_equal(xm, y * a) assert_equal(xm.mask, mask_or(m, a.mask)) - assert_equal(len(w), 0, "Failed on type=%s." % t) + assert_equal(len(w), 0, f'Failed on type={t}.') def test_inplace_floor_division_scalar_type(self): # Test of inplace division @@ -2861,7 +2861,7 @@ def test_inplace_floor_division_array_type(self): mask_or(mask_or(m, a.mask), (a == t(0))) ) - assert_equal(len(w), 0, "Failed on type=%s." % t) + assert_equal(len(w), 0, f'Failed on type={t}.') def test_inplace_division_scalar_type(self): # Test of inplace division @@ -2895,9 +2895,9 @@ def test_inplace_division_scalar_type(self): warnings.warn(str(e), stacklevel=1) if issubclass(t, np.integer): - assert_equal(len(sup.log), 2, "Failed on type=%s." % t) + assert_equal(len(sup.log), 2, f'Failed on type={t}.') else: - assert_equal(len(sup.log), 0, "Failed on type=%s." % t) + assert_equal(len(sup.log), 0, f'Failed on type={t}.') def test_inplace_division_array_type(self): # Test of inplace division @@ -2934,9 +2934,9 @@ def test_inplace_division_array_type(self): warnings.warn(str(e), stacklevel=1) if issubclass(t, np.integer): - assert_equal(len(sup.log), 2, "Failed on type=%s." % t) + assert_equal(len(sup.log), 2, f'Failed on type={t}.') else: - assert_equal(len(sup.log), 0, "Failed on type=%s." % t) + assert_equal(len(sup.log), 0, f'Failed on type={t}.') def test_inplace_pow_type(self): # Test keeping data w/ (inplace) power @@ -2954,7 +2954,7 @@ def test_inplace_pow_type(self): assert_equal(x.data, xx_r.data) assert_equal(x.mask, xx_r.mask) - assert_equal(len(w), 0, "Failed on type=%s." % t) + assert_equal(len(w), 0, f'Failed on type={t}.') class TestMaskedArrayMethods: diff --git a/numpy/ma/tests/test_old_ma.py b/numpy/ma/tests/test_old_ma.py index 96c7e3609795..ab003b94e584 100644 --- a/numpy/ma/tests/test_old_ma.py +++ b/numpy/ma/tests/test_old_ma.py @@ -27,7 +27,7 @@ def eq(v, w, msg=''): result = allclose(v, w) if not result: - print("Not eq:%s\n%s\n----%s" % (msg, str(v), str(w))) + print(f'Not eq:{msg}\n{v}\n----{w}') return result diff --git a/numpy/ma/tests/test_subclassing.py b/numpy/ma/tests/test_subclassing.py index caa746740534..f2623406d183 100644 --- a/numpy/ma/tests/test_subclassing.py +++ b/numpy/ma/tests/test_subclassing.py @@ -109,11 +109,11 @@ def __next__(self): class ComplicatedSubArray(SubArray): def __str__(self): - return 'myprefix {0} mypostfix'.format(self.view(SubArray)) + return f'myprefix {self.view(SubArray)} mypostfix' def __repr__(self): # Return a repr that does not start with 'name(' - return '<{0} {1}>'.format(self.__class__.__name__, self) + return f'<{self.__class__.__name__} {self}>' def _validate_input(self, value): if not isinstance(value, ComplicatedSubArray): @@ -317,8 +317,8 @@ def test_subclass_repr(self): assert_startswith(repr(mx), 'masked_array') xsub = SubArray(x) mxsub = masked_array(xsub, mask=[True, False, True, False, False]) - assert_startswith(repr(mxsub), - 'masked_{0}(data=[--, 1, --, 3, 4]'.format(SubArray.__name__)) + assert_startswith(repr(mxsub), + f'masked_{SubArray.__name__}(data=[--, 1, --, 3, 4]') def test_subclass_str(self): """test str with subclass that has overridden str, setitem""" diff --git a/numpy/ma/testutils.py b/numpy/ma/testutils.py index 51ab03948ebc..8d55e1763da1 100644 --- a/numpy/ma/testutils.py +++ b/numpy/ma/testutils.py @@ -86,7 +86,7 @@ def _assert_equal_on_sequences(actual, desired, err_msg=''): """ assert_equal(len(actual), len(desired), err_msg) for k in range(len(desired)): - assert_equal(actual[k], desired[k], 'item=%r\n%s' % (k, err_msg)) + assert_equal(actual[k], desired[k], f'item={k!r}\n{err_msg}') return @@ -117,8 +117,8 @@ def assert_equal(actual, desired, err_msg=''): assert_equal(len(actual), len(desired), err_msg) for k, i in desired.items(): if k not in actual: - raise AssertionError("%s not in %s" % (k, actual)) - assert_equal(actual[k], desired[k], 'key=%r\n%s' % (k, err_msg)) + raise AssertionError(f"{k} not in {actual}") + assert_equal(actual[k], desired[k], f'key={k!r}\n{err_msg}') return # Case #2: lists ..... if isinstance(desired, (list, tuple)) and isinstance(actual, (list, tuple)): @@ -156,12 +156,12 @@ def fail_if_equal(actual, desired, err_msg='',): for k, i in desired.items(): if k not in actual: raise AssertionError(repr(k)) - fail_if_equal(actual[k], desired[k], 'key=%r\n%s' % (k, err_msg)) + fail_if_equal(actual[k], desired[k], f'key={k!r}\n{err_msg}') return if isinstance(desired, (list, tuple)) and isinstance(actual, (list, tuple)): fail_if_equal(len(actual), len(desired), err_msg) for k in range(len(desired)): - fail_if_equal(actual[k], desired[k], 'item=%r\n%s' % (k, err_msg)) + fail_if_equal(actual[k], desired[k], f'item={k!r}\n{err_msg}') return if isinstance(actual, np.ndarray) or isinstance(desired, np.ndarray): return fail_if_array_equal(actual, desired, err_msg) diff --git a/numpy/ma/timer_comparison.py b/numpy/ma/timer_comparison.py index f5855efcf737..aff72f79b8d2 100644 --- a/numpy/ma/timer_comparison.py +++ b/numpy/ma/timer_comparison.py @@ -77,8 +77,7 @@ def assert_array_compare(self, comparison, x, y, err_msg='', header='', if not cond: msg = build_err_msg([x, y], err_msg - + '\n(shapes %s, %s mismatch)' % (x.shape, - y.shape), + + f'\n(shapes {x.shape}, {y.shape} mismatch)', header=header, names=('x', 'y')) assert cond, msg @@ -434,4 +433,4 @@ def test_A(self): cur = np.sort(cur) print("#%i" % i + 50*'.') print(eval("ModuleTester.test_%i.__doc__" % i)) - print("core_current : %.3f - %.3f" % (cur[0], cur[1])) + print(f'core_current : {cur[0]:.3f} - {cur[1]:.3f}') diff --git a/numpy/matrixlib/setup.py b/numpy/matrixlib/setup.py index 19b3bb2de7c6..4fed75de1cbc 100644 --- a/numpy/matrixlib/setup.py +++ b/numpy/matrixlib/setup.py @@ -3,6 +3,7 @@ def configuration(parent_package='', top_path=None): from numpy.distutils.misc_util import Configuration config = Configuration('matrixlib', parent_package, top_path) config.add_subpackage('tests') + config.add_data_files('*.pyi') return config if __name__ == "__main__": diff --git a/numpy/polynomial/_polybase.py b/numpy/polynomial/_polybase.py index f4a67a2227f9..59c380f10e3a 100644 --- a/numpy/polynomial/_polybase.py +++ b/numpy/polynomial/_polybase.py @@ -547,8 +547,8 @@ def __divmod__(self, other): othercoef = self._get_coefficients(other) try: quo, rem = self._div(self.coef, othercoef) - except ZeroDivisionError as e: - raise e + except ZeroDivisionError: + raise except Exception: return NotImplemented quo = self.__class__(quo, self.domain, self.window) @@ -605,8 +605,8 @@ def __rmod__(self, other): def __rdivmod__(self, other): try: quo, rem = self._div(other, self.coef) - except ZeroDivisionError as e: - raise e + except ZeroDivisionError: + raise except Exception: return NotImplemented quo = self.__class__(quo, self.domain, self.window) diff --git a/numpy/polynomial/setup.py b/numpy/polynomial/setup.py index 6414645186fa..b58e867a133f 100644 --- a/numpy/polynomial/setup.py +++ b/numpy/polynomial/setup.py @@ -2,6 +2,7 @@ def configuration(parent_package='',top_path=None): from numpy.distutils.misc_util import Configuration config = Configuration('polynomial', parent_package, top_path) config.add_subpackage('tests') + config.add_data_files('*.pyi') return config if __name__ == '__main__': diff --git a/numpy/random/_examples/cython/setup.py b/numpy/random/_examples/cython/setup.py index 42425c2c199f..83f06fde8581 100644 --- a/numpy/random/_examples/cython/setup.py +++ b/numpy/random/_examples/cython/setup.py @@ -19,7 +19,7 @@ lib_path = join(np.get_include(), '..', '..', 'random', 'lib') extending = Extension("extending", - sources=[join(path, 'extending.pyx')], + sources=[join('.', 'extending.pyx')], include_dirs=[ np.get_include(), join(path, '..', '..') @@ -27,7 +27,7 @@ define_macros=defs, ) distributions = Extension("extending_distributions", - sources=[join(path, 'extending_distributions.pyx')], + sources=[join('.', 'extending_distributions.pyx')], include_dirs=[inc_path], library_dirs=[lib_path], libraries=['npyrandom'], diff --git a/numpy/random/_examples/numba/extending.py b/numpy/random/_examples/numba/extending.py index 0d240596b67c..da4d9394461a 100644 --- a/numpy/random/_examples/numba/extending.py +++ b/numpy/random/_examples/numba/extending.py @@ -44,9 +44,9 @@ def numpycall(): assert r1.shape == r2.shape t1 = timeit(numbacall, number=1000) -print('{:.2f} secs for {} PCG64 (Numba/PCG64) gaussian randoms'.format(t1, n)) ++print(f'{t1:.2f} secs for {n} PCG64 (Numba/PCG64) gaussian randoms') t2 = timeit(numpycall, number=1000) -print('{:.2f} secs for {} PCG64 (NumPy/PCG64) gaussian randoms'.format(t2, n)) +print(f'{t2:.2f} secs for {n} PCG64 (NumPy/PCG64) gaussian randoms') # example 2 diff --git a/numpy/random/_generator.pyx b/numpy/random/_generator.pyx index e40dcefe34b7..bff023d7ffb6 100644 --- a/numpy/random/_generator.pyx +++ b/numpy/random/_generator.pyx @@ -4521,7 +4521,7 @@ def default_rng(seed=None): unpredictable entropy will be pulled from the OS. If an ``int`` or ``array_like[ints]`` is passed, then it will be passed to `SeedSequence` to derive the initial `BitGenerator` state. One may also - pass in a`SeedSequence` instance + pass in a `SeedSequence` instance. Additionally, when passed a `BitGenerator`, it will be wrapped by `Generator`. If passed a `Generator`, it will be returned unaltered. diff --git a/numpy/random/setup.py b/numpy/random/setup.py index 88ddb12687ea..bfd08e4691aa 100644 --- a/numpy/random/setup.py +++ b/numpy/random/setup.py @@ -137,6 +137,7 @@ def generate_libraries(ext, build_dir): define_macros=defs + LEGACY_DEFS, ) config.add_data_files(*depends) + config.add_data_files('*.pyi') return config diff --git a/numpy/random/tests/test_direct.py b/numpy/random/tests/test_direct.py index dad12c8a80ec..d602b36b4169 100644 --- a/numpy/random/tests/test_direct.py +++ b/numpy/random/tests/test_direct.py @@ -230,13 +230,13 @@ def test_uniform_float(self): def test_repr(self): rs = Generator(self.bit_generator(*self.data1['seed'])) assert 'Generator' in repr(rs) - assert '{:#x}'.format(id(rs)).upper().replace('X', 'x') in repr(rs) + assert f'{id(rs):#x}'.upper().replace('X', 'x') in repr(rs) def test_str(self): rs = Generator(self.bit_generator(*self.data1['seed'])) assert 'Generator' in str(rs) assert str(self.bit_generator.__name__) in str(rs) - assert '{:#x}'.format(id(rs)).upper().replace('X', 'x') not in str(rs) + assert f'{id(rs):#x}'.upper().replace('X', 'x') not in str(rs) def test_pickle(self): import pickle diff --git a/numpy/random/tests/test_generator_mt19937.py b/numpy/random/tests/test_generator_mt19937.py index 6be7d852b60d..b69cd38d4a4a 100644 --- a/numpy/random/tests/test_generator_mt19937.py +++ b/numpy/random/tests/test_generator_mt19937.py @@ -18,20 +18,20 @@ { "seed": 0, "steps": 10, - "initial": {"key_md5": "64eaf265d2203179fb5ffb73380cd589", "pos": 9}, - "jumped": {"key_md5": "8cb7b061136efceef5217a9ce2cc9a5a", "pos": 598}, + "initial": {"key_sha256": "bb1636883c2707b51c5b7fc26c6927af4430f2e0785a8c7bc886337f919f9edf", "pos": 9}, + "jumped": {"key_sha256": "ff682ac12bb140f2d72fba8d3506cf4e46817a0db27aae1683867629031d8d55", "pos": 598}, }, { "seed":384908324, "steps":312, - "initial": {"key_md5": "e99708a47b82ff51a2c7b0625b81afb5", "pos": 311}, - "jumped": {"key_md5": "2ecdbfc47a895b253e6e19ccb2e74b90", "pos": 276}, + "initial": {"key_sha256": "16b791a1e04886ccbbb4d448d6ff791267dc458ae599475d08d5cced29d11614", "pos": 311}, + "jumped": {"key_sha256": "a0110a2cf23b56be0feaed8f787a7fc84bef0cb5623003d75b26bdfa1c18002c", "pos": 276}, }, { "seed": [839438204, 980239840, 859048019, 821], "steps": 511, - "initial": {"key_md5": "9fcd6280df9199785e17e93162ce283c", "pos": 510}, - "jumped": {"key_md5": "433b85229f2ed853cde06cd872818305", "pos": 475}, + "initial": {"key_sha256": "d306cf01314d51bd37892d874308200951a35265ede54d200f1e065004c3e9ea", "pos": 510}, + "jumped": {"key_sha256": "0e00ab449f01a5195a83b4aee0dfbc2ce8d46466a640b92e33977d2e42f777f8", "pos": 475}, }, ] @@ -483,18 +483,18 @@ def test_scalar_array_equiv(self, endpoint): assert_array_equal(scalar, array) def test_repeatability(self, endpoint): - # We use a md5 hash of generated sequences of 1000 samples + # We use a sha256 hash of generated sequences of 1000 samples # in the range [0, 6) for all but bool, where the range # is [0, 2). Hashes are for little endian numbers. - tgt = {'bool': 'b3300e66d2bb59e493d255d47c3a6cbe', - 'int16': '39624ead49ad67e37545744024d2648b', - 'int32': '5c4810373f979336c6c0c999996e47a1', - 'int64': 'ab126c15edff26f55c50d2b7e37391ac', - 'int8': 'ba71ccaffeeeb9eeb1860f8075020b9c', - 'uint16': '39624ead49ad67e37545744024d2648b', - 'uint32': '5c4810373f979336c6c0c999996e47a1', - 'uint64': 'ab126c15edff26f55c50d2b7e37391ac', - 'uint8': 'ba71ccaffeeeb9eeb1860f8075020b9c'} + tgt = {'bool': '053594a9b82d656f967c54869bc6970aa0358cf94ad469c81478459c6a90eee3', + 'int16': '54de9072b6ee9ff7f20b58329556a46a447a8a29d67db51201bf88baa6e4e5d4', + 'int32': 'd3a0d5efb04542b25ac712e50d21f39ac30f312a5052e9bbb1ad3baa791ac84b', + 'int64': '14e224389ac4580bfbdccb5697d6190b496f91227cf67df60989de3d546389b1', + 'int8': '0e203226ff3fbbd1580f15da4621e5f7164d0d8d6b51696dd42d004ece2cbec1', + 'uint16': '54de9072b6ee9ff7f20b58329556a46a447a8a29d67db51201bf88baa6e4e5d4', + 'uint32': 'd3a0d5efb04542b25ac712e50d21f39ac30f312a5052e9bbb1ad3baa791ac84b', + 'uint64': '14e224389ac4580bfbdccb5697d6190b496f91227cf67df60989de3d546389b1', + 'uint8': '0e203226ff3fbbd1580f15da4621e5f7164d0d8d6b51696dd42d004ece2cbec1'} for dt in self.itype[1:]: random = Generator(MT19937(1234)) @@ -507,14 +507,14 @@ def test_repeatability(self, endpoint): val = random.integers(0, 6 - endpoint, size=1000, endpoint=endpoint, dtype=dt).byteswap() - res = hashlib.md5(val).hexdigest() + res = hashlib.sha256(val).hexdigest() assert_(tgt[np.dtype(dt).name] == res) # bools do not depend on endianness random = Generator(MT19937(1234)) val = random.integers(0, 2 - endpoint, size=1000, endpoint=endpoint, dtype=bool).view(np.int8) - res = hashlib.md5(val).hexdigest() + res = hashlib.sha256(val).hexdigest() assert_(tgt[np.dtype(bool).name] == res) def test_repeatability_broadcasting(self, endpoint): @@ -905,12 +905,12 @@ def test_choice_return_type(self): assert actual.dtype == np.int64 def test_choice_large_sample(self): - choice_hash = 'd44962a0b1e92f4a3373c23222244e21' + choice_hash = '4266599d12bfcfb815213303432341c06b4349f5455890446578877bb322e222' random = Generator(MT19937(self.seed)) actual = random.choice(10000, 5000, replace=False) if sys.byteorder != 'little': actual = actual.byteswap() - res = hashlib.md5(actual.view(np.int8)).hexdigest() + res = hashlib.sha256(actual.view(np.int8)).hexdigest() assert_(choice_hash == res) def test_bytes(self): @@ -2424,7 +2424,7 @@ def test_three_arg_funcs(self): @pytest.mark.parametrize("config", JUMP_TEST_DATA) def test_jumped(config): # Each config contains the initial seed, a number of raw steps - # the md5 hashes of the initial and the final states' keys and + # the sha256 hashes of the initial and the final states' keys and # the position of of the initial and the final state. # These were produced using the original C implementation. seed = config["seed"] @@ -2436,17 +2436,17 @@ def test_jumped(config): key = mt19937.state["state"]["key"] if sys.byteorder == 'big': key = key.byteswap() - md5 = hashlib.md5(key) + sha256 = hashlib.sha256(key) assert mt19937.state["state"]["pos"] == config["initial"]["pos"] - assert md5.hexdigest() == config["initial"]["key_md5"] + assert sha256.hexdigest() == config["initial"]["key_sha256"] jumped = mt19937.jumped() key = jumped.state["state"]["key"] if sys.byteorder == 'big': key = key.byteswap() - md5 = hashlib.md5(key) + sha256 = hashlib.sha256(key) assert jumped.state["state"]["pos"] == config["jumped"]["pos"] - assert md5.hexdigest() == config["jumped"]["key_md5"] + assert sha256.hexdigest() == config["jumped"]["key_sha256"] def test_broadcast_size_error(): diff --git a/numpy/random/tests/test_generator_mt19937_regressions.py b/numpy/random/tests/test_generator_mt19937_regressions.py index 456c932d4cb2..2ef6b0631920 100644 --- a/numpy/random/tests/test_generator_mt19937_regressions.py +++ b/numpy/random/tests/test_generator_mt19937_regressions.py @@ -33,11 +33,11 @@ def test_logseries_convergence(self): # numbers with this large sample # theoretical large N result is 0.49706795 freq = np.sum(rvsn == 1) / float(N) - msg = "Frequency was %f, should be > 0.45" % freq + msg = f'Frequency was {freq:f}, should be > 0.45' assert_(freq > 0.45, msg) # theoretical large N result is 0.19882718 freq = np.sum(rvsn == 2) / float(N) - msg = "Frequency was %f, should be < 0.23" % freq + msg = f'Frequency was {freq:f}, should be < 0.23' assert_(freq < 0.23, msg) def test_shuffle_mixed_dimension(self): diff --git a/numpy/random/tests/test_random.py b/numpy/random/tests/test_random.py index 276b5bc81c5e..c13fc39e3339 100644 --- a/numpy/random/tests/test_random.py +++ b/numpy/random/tests/test_random.py @@ -211,18 +211,18 @@ def test_in_bounds_fuzz(self): def test_repeatability(self): import hashlib - # We use a md5 hash of generated sequences of 1000 samples + # We use a sha256 hash of generated sequences of 1000 samples # in the range [0, 6) for all but bool, where the range # is [0, 2). Hashes are for little endian numbers. - tgt = {'bool': '7dd3170d7aa461d201a65f8bcf3944b0', - 'int16': '1b7741b80964bb190c50d541dca1cac1', - 'int32': '4dc9fcc2b395577ebb51793e58ed1a05', - 'int64': '17db902806f448331b5a758d7d2ee672', - 'int8': '27dd30c4e08a797063dffac2490b0be6', - 'uint16': '1b7741b80964bb190c50d541dca1cac1', - 'uint32': '4dc9fcc2b395577ebb51793e58ed1a05', - 'uint64': '17db902806f448331b5a758d7d2ee672', - 'uint8': '27dd30c4e08a797063dffac2490b0be6'} + tgt = {'bool': '509aea74d792fb931784c4b0135392c65aec64beee12b0cc167548a2c3d31e71', + 'int16': '7b07f1a920e46f6d0fe02314155a2330bcfd7635e708da50e536c5ebb631a7d4', + 'int32': 'e577bfed6c935de944424667e3da285012e741892dcb7051a8f1ce68ab05c92f', + 'int64': '0fbead0b06759df2cfb55e43148822d4a1ff953c7eb19a5b08445a63bb64fa9e', + 'int8': '001aac3a5acb935a9b186cbe14a1ca064b8bb2dd0b045d48abeacf74d0203404', + 'uint16': '7b07f1a920e46f6d0fe02314155a2330bcfd7635e708da50e536c5ebb631a7d4', + 'uint32': 'e577bfed6c935de944424667e3da285012e741892dcb7051a8f1ce68ab05c92f', + 'uint64': '0fbead0b06759df2cfb55e43148822d4a1ff953c7eb19a5b08445a63bb64fa9e', + 'uint8': '001aac3a5acb935a9b186cbe14a1ca064b8bb2dd0b045d48abeacf74d0203404'} for dt in self.itype[1:]: np.random.seed(1234) @@ -233,13 +233,13 @@ def test_repeatability(self): else: val = self.rfunc(0, 6, size=1000, dtype=dt).byteswap() - res = hashlib.md5(val.view(np.int8)).hexdigest() + res = hashlib.sha256(val.view(np.int8)).hexdigest() assert_(tgt[np.dtype(dt).name] == res) # bools do not depend on endianness np.random.seed(1234) val = self.rfunc(0, 2, size=1000, dtype=bool).view(np.int8) - res = hashlib.md5(val).hexdigest() + res = hashlib.sha256(val).hexdigest() assert_(tgt[np.dtype(bool).name] == res) def test_int64_uint64_corner_case(self): diff --git a/numpy/random/tests/test_randomstate.py b/numpy/random/tests/test_randomstate.py index 23dbbed6a057..b70a043472cc 100644 --- a/numpy/random/tests/test_randomstate.py +++ b/numpy/random/tests/test_randomstate.py @@ -26,24 +26,24 @@ if np.iinfo(int).max < 2**32: # Windows and some 32-bit platforms, e.g., ARM - INT_FUNC_HASHES = {'binomial': '670e1c04223ffdbab27e08fbbad7bdba', - 'logseries': '6bd0183d2f8030c61b0d6e11aaa60caf', - 'geometric': '6e9df886f3e1e15a643168568d5280c0', - 'hypergeometric': '7964aa611b046aecd33063b90f4dec06', - 'multinomial': '68a0b049c16411ed0aa4aff3572431e4', - 'negative_binomial': 'dc265219eec62b4338d39f849cd36d09', - 'poisson': '7b4dce8e43552fc82701c2fa8e94dc6e', - 'zipf': 'fcd2a2095f34578723ac45e43aca48c5', + INT_FUNC_HASHES = {'binomial': '2fbead005fc63942decb5326d36a1f32fe2c9d32c904ee61e46866b88447c263', + 'logseries': '23ead5dcde35d4cfd4ef2c105e4c3d43304b45dc1b1444b7823b9ee4fa144ebb', + 'geometric': '0d764db64f5c3bad48c8c33551c13b4d07a1e7b470f77629bef6c985cac76fcf', + 'hypergeometric': '7b59bf2f1691626c5815cdcd9a49e1dd68697251d4521575219e4d2a1b8b2c67', + 'multinomial': 'd754fa5b92943a38ec07630de92362dd2e02c43577fc147417dc5b9db94ccdd3', + 'negative_binomial': '8eb216f7cb2a63cf55605422845caaff002fddc64a7dc8b2d45acd477a49e824', + 'poisson': '70c891d76104013ebd6f6bcf30d403a9074b886ff62e4e6b8eb605bf1a4673b7', + 'zipf': '01f074f97517cd5d21747148ac6ca4074dde7fcb7acbaec0a936606fecacd93f', } else: - INT_FUNC_HASHES = {'binomial': 'b5f8dcd74f172836536deb3547257b14', - 'geometric': '8814571f45c87c59699d62ccd3d6c350', - 'hypergeometric': 'bc64ae5976eac452115a16dad2dcf642', - 'logseries': '84be924b37485a27c4a98797bc88a7a4', - 'multinomial': 'ec3c7f9cf9664044bb0c6fb106934200', - 'negative_binomial': '210533b2234943591364d0117a552969', - 'poisson': '0536a8850c79da0c78defd742dccc3e0', - 'zipf': 'f2841f504dd2525cd67cdcad7561e532', + INT_FUNC_HASHES = {'binomial': '8626dd9d052cb608e93d8868de0a7b347258b199493871a1dc56e2a26cacb112', + 'geometric': '8edd53d272e49c4fc8fbbe6c7d08d563d62e482921f3131d0a0e068af30f0db9', + 'hypergeometric': '83496cc4281c77b786c9b7ad88b74d42e01603a55c60577ebab81c3ba8d45657', + 'logseries': '65878a38747c176bc00e930ebafebb69d4e1e16cd3a704e264ea8f5e24f548db', + 'multinomial': '7a984ae6dca26fd25374479e118b22f55db0aedccd5a0f2584ceada33db98605', + 'negative_binomial': 'd636d968e6a24ae92ab52fe11c46ac45b0897e98714426764e820a7d77602a61', + 'poisson': '956552176f77e7c9cb20d0118fc9cf690be488d790ed4b4c4747b965e61b0bb4', + 'zipf': 'f84ba7feffda41e606e20b28dfc0f1ea9964a74574513d4a4cbc98433a8bfa45', } @@ -319,18 +319,18 @@ def test_in_bounds_fuzz(self): assert_(vals.min() >= 0) def test_repeatability(self): - # We use a md5 hash of generated sequences of 1000 samples + # We use a sha256 hash of generated sequences of 1000 samples # in the range [0, 6) for all but bool, where the range # is [0, 2). Hashes are for little endian numbers. - tgt = {'bool': '7dd3170d7aa461d201a65f8bcf3944b0', - 'int16': '1b7741b80964bb190c50d541dca1cac1', - 'int32': '4dc9fcc2b395577ebb51793e58ed1a05', - 'int64': '17db902806f448331b5a758d7d2ee672', - 'int8': '27dd30c4e08a797063dffac2490b0be6', - 'uint16': '1b7741b80964bb190c50d541dca1cac1', - 'uint32': '4dc9fcc2b395577ebb51793e58ed1a05', - 'uint64': '17db902806f448331b5a758d7d2ee672', - 'uint8': '27dd30c4e08a797063dffac2490b0be6'} + tgt = {'bool': '509aea74d792fb931784c4b0135392c65aec64beee12b0cc167548a2c3d31e71', + 'int16': '7b07f1a920e46f6d0fe02314155a2330bcfd7635e708da50e536c5ebb631a7d4', + 'int32': 'e577bfed6c935de944424667e3da285012e741892dcb7051a8f1ce68ab05c92f', + 'int64': '0fbead0b06759df2cfb55e43148822d4a1ff953c7eb19a5b08445a63bb64fa9e', + 'int8': '001aac3a5acb935a9b186cbe14a1ca064b8bb2dd0b045d48abeacf74d0203404', + 'uint16': '7b07f1a920e46f6d0fe02314155a2330bcfd7635e708da50e536c5ebb631a7d4', + 'uint32': 'e577bfed6c935de944424667e3da285012e741892dcb7051a8f1ce68ab05c92f', + 'uint64': '0fbead0b06759df2cfb55e43148822d4a1ff953c7eb19a5b08445a63bb64fa9e', + 'uint8': '001aac3a5acb935a9b186cbe14a1ca064b8bb2dd0b045d48abeacf74d0203404'} for dt in self.itype[1:]: random.seed(1234) @@ -341,13 +341,13 @@ def test_repeatability(self): else: val = self.rfunc(0, 6, size=1000, dtype=dt).byteswap() - res = hashlib.md5(val.view(np.int8)).hexdigest() + res = hashlib.sha256(val.view(np.int8)).hexdigest() assert_(tgt[np.dtype(dt).name] == res) # bools do not depend on endianness random.seed(1234) val = self.rfunc(0, 2, size=1000, dtype=bool).view(np.int8) - res = hashlib.md5(val).hexdigest() + res = hashlib.sha256(val).hexdigest() assert_(tgt[np.dtype(bool).name] == res) @pytest.mark.skipif(np.iinfo('l').max < 2**32, @@ -1974,7 +1974,7 @@ def test_three_arg_funcs(self): # Ensure returned array dtype is correct for platform def test_integer_dtype(int_func): random.seed(123456789) - fname, args, md5 = int_func + fname, args, sha256 = int_func f = getattr(random, fname) actual = f(*args, size=2) assert_(actual.dtype == np.dtype('l')) @@ -1982,13 +1982,13 @@ def test_integer_dtype(int_func): def test_integer_repeat(int_func): random.seed(123456789) - fname, args, md5 = int_func + fname, args, sha256 = int_func f = getattr(random, fname) val = f(*args, size=1000000) if sys.byteorder != 'little': val = val.byteswap() - res = hashlib.md5(val.view(np.int8)).hexdigest() - assert_(res == md5) + res = hashlib.sha256(val.view(np.int8)).hexdigest() + assert_(res == sha256) def test_broadcast_size_error(): diff --git a/numpy/random/tests/test_randomstate_regression.py b/numpy/random/tests/test_randomstate_regression.py index 4eb82fc4ca93..0bf361e5eb46 100644 --- a/numpy/random/tests/test_randomstate_regression.py +++ b/numpy/random/tests/test_randomstate_regression.py @@ -44,11 +44,11 @@ def test_logseries_convergence(self): # numbers with this large sample # theoretical large N result is 0.49706795 freq = np.sum(rvsn == 1) / float(N) - msg = "Frequency was %f, should be > 0.45" % freq + msg = f'Frequency was {freq:f}, should be > 0.45' assert_(freq > 0.45, msg) # theoretical large N result is 0.19882718 freq = np.sum(rvsn == 2) / float(N) - msg = "Frequency was %f, should be < 0.23" % freq + msg = f'Frequency was {freq:f}, should be < 0.23' assert_(freq < 0.23, msg) def test_shuffle_mixed_dimension(self): diff --git a/numpy/random/tests/test_regression.py b/numpy/random/tests/test_regression.py index 278622287173..54d5a3efbdba 100644 --- a/numpy/random/tests/test_regression.py +++ b/numpy/random/tests/test_regression.py @@ -40,11 +40,11 @@ def test_logseries_convergence(self): # numbers with this large sample # theoretical large N result is 0.49706795 freq = np.sum(rvsn == 1) / float(N) - msg = "Frequency was %f, should be > 0.45" % freq + msg = f'Frequency was {freq:f}, should be > 0.45' assert_(freq > 0.45, msg) # theoretical large N result is 0.19882718 freq = np.sum(rvsn == 2) / float(N) - msg = "Frequency was %f, should be < 0.23" % freq + msg = f'Frequency was {freq:f}, should be < 0.23' assert_(freq < 0.23, msg) def test_shuffle_mixed_dimension(self): diff --git a/numpy/random/tests/test_smoke.py b/numpy/random/tests/test_smoke.py index ebfc6825ebb9..909bfaa8dab6 100644 --- a/numpy/random/tests/test_smoke.py +++ b/numpy/random/tests/test_smoke.py @@ -129,7 +129,7 @@ def test_advance(self): assert_(not comp_state(state, self.rg.bit_generator.state)) else: bitgen_name = self.rg.bit_generator.__class__.__name__ - pytest.skip('Advance is not supported by {0}'.format(bitgen_name)) + pytest.skip(f'Advance is not supported by {bitgen_name}') def test_jump(self): state = self.rg.bit_generator.state @@ -145,8 +145,8 @@ def test_jump(self): else: bitgen_name = self.rg.bit_generator.__class__.__name__ if bitgen_name not in ('SFC64',): - raise AttributeError('no "jumped" in %s' % bitgen_name) - pytest.skip('Jump is not supported by {0}'.format(bitgen_name)) + raise AttributeError(f'no "jumped" in {bitgen_name}') + pytest.skip(f'Jump is not supported by {bitgen_name}') def test_uniform(self): r = self.rg.uniform(-1.0, 0.0, size=10) @@ -447,8 +447,7 @@ def test_pickle(self): def test_seed_array(self): if self.seed_vector_bits is None: bitgen_name = self.bit_generator.__name__ - pytest.skip('Vector seeding is not supported by ' - '{0}'.format(bitgen_name)) + pytest.skip(f'Vector seeding is not supported by {bitgen_name}') if self.seed_vector_bits == 32: dtype = np.uint32 diff --git a/numpy/testing/_private/decorators.py b/numpy/testing/_private/decorators.py index b4b6259a0497..4c87d1a4919c 100644 --- a/numpy/testing/_private/decorators.py +++ b/numpy/testing/_private/decorators.py @@ -136,7 +136,7 @@ def get_msg(func,msg=None): else: out = msg - return "Skipping test: %s: %s" % (func.__name__, out) + return f'Skipping test: {func.__name__}: {out}' # We need to define *two* skippers because Python doesn't allow both # return with value and yield inside the same function. diff --git a/numpy/testing/_private/noseclasses.py b/numpy/testing/_private/noseclasses.py index 69e19e9594d4..48fa4dc1f5af 100644 --- a/numpy/testing/_private/noseclasses.py +++ b/numpy/testing/_private/noseclasses.py @@ -76,7 +76,7 @@ def _find(self, tests, obj, name, module, source_lines, globs, seen): # Look for tests in a module's contained objects. if ismodule(obj) and self._recurse: for valname, val in obj.__dict__.items(): - valname1 = '%s.%s' % (name, valname) + valname1 = f'{name}.{valname}' if ( (isroutine(val) or isclass(val)) and self._from_module(module, val)): @@ -96,7 +96,7 @@ def _find(self, tests, obj, name, module, source_lines, globs, seen): if ((isfunction(val) or isclass(val) or ismethod(val) or isinstance(val, property)) and self._from_module(module, val)): - valname = '%s.%s' % (name, valname) + valname = f'{name}.{valname}' self._find(tests, val, valname, module, source_lines, globs, seen) diff --git a/numpy/testing/_private/nosetester.py b/numpy/testing/_private/nosetester.py index 57691a448f73..bccec8236912 100644 --- a/numpy/testing/_private/nosetester.py +++ b/numpy/testing/_private/nosetester.py @@ -233,20 +233,20 @@ def _show_system_info(self): nose = import_nose() import numpy - print("NumPy version %s" % numpy.__version__) + print(f'NumPy version {numpy.__version__}') relaxed_strides = numpy.ones((10, 1), order="C").flags.f_contiguous print("NumPy relaxed strides checking option:", relaxed_strides) npdir = os.path.dirname(numpy.__file__) - print("NumPy is installed in %s" % npdir) + print(f'NumPy is installed in {npdir}') if 'scipy' in self.package_name: import scipy - print("SciPy version %s" % scipy.__version__) + print(f'SciPy version {scipy.__version__}') spdir = os.path.dirname(scipy.__file__) - print("SciPy is installed in %s" % spdir) + print(f'SciPy is installed in {spdir}') pyversion = sys.version.replace('\n', '') - print("Python version %s" % pyversion) + print(f'Python version {pyversion}') print("nose version %d.%d.%d" % nose.__versioninfo__) def _get_custom_doctester(self): @@ -278,7 +278,7 @@ def prepare_test_args(self, label='fast', verbose=1, extra_argv=None, argv = self._test_argv(label, verbose, extra_argv) # our way of doing coverage if coverage: - argv += ['--cover-package=%s' % self.package_name, '--with-coverage', + argv += [f'--cover-package={self.package_name}', '--with-coverage', '--cover-tests', '--cover-erase'] if timer: @@ -403,9 +403,9 @@ def test(self, label='fast', verbose=1, extra_argv=None, label, verbose, extra_argv, doctests, coverage, timer) if doctests: - print("Running unit tests and doctests for %s" % self.package_name) + print(f'Running unit tests and doctests for {self.package_name}') else: - print("Running unit tests for %s" % self.package_name) + print(f'Running unit tests for {self.package_name}') self._show_system_info() @@ -520,7 +520,7 @@ def bench(self, label='fast', verbose=1, extra_argv=None): """ - print("Running benchmarks for %s" % self.package_name) + print(f'Running benchmarks for {self.package_name}') self._show_system_info() argv = self._test_argv(label, verbose, extra_argv) diff --git a/numpy/testing/_private/parameterized.py b/numpy/testing/_private/parameterized.py index 3bd8ede91fad..ac7db6c4041f 100644 --- a/numpy/testing/_private/parameterized.py +++ b/numpy/testing/_private/parameterized.py @@ -205,7 +205,7 @@ def default_doc_func(func, num, p): all_args_with_values = parameterized_argument_value_pairs(func, p) # Assumes that the function passed is a bound method. - descs = ["%s=%s" %(n, short_repr(v)) for n, v in all_args_with_values] + descs = [f'{n}={short_repr(v)}' for n, v in all_args_with_values] # The documentation might be a multiline string, so split it # and just work with the first string, ignoring the period diff --git a/numpy/testing/_private/utils.py b/numpy/testing/_private/utils.py index 4fc22dcebaec..fb33bdcbd7f7 100644 --- a/numpy/testing/_private/utils.py +++ b/numpy/testing/_private/utils.py @@ -113,14 +113,14 @@ def gisnan(x): def gisfinite(x): - """like isfinite, but always raise an error if type not supported instead of - returning a TypeError object. + """like isfinite, but always raise an error if type not supported instead + of returning a TypeError object. Notes ----- - isfinite and other ufunc sometimes return a NotImplementedType object instead - of raising any exception. This function is a wrapper to make sure an - exception is always raised. + isfinite and other ufunc sometimes return a NotImplementedType object + instead of raising any exception. This function is a wrapper to make sure + an exception is always raised. This should be removed once this problem is solved at the Ufunc level.""" from numpy.core import isfinite, errstate @@ -160,12 +160,13 @@ def GetPerformanceAttributes(object, counter, instance=None, # you should copy this function, but keep the counter open, and call # CollectQueryData() each time you need to know. # See http://msdn.microsoft.com/library/en-us/dnperfmo/html/perfmonpt2.asp (dead link) - # My older explanation for this was that the "AddCounter" process forced - # the CPU to 100%, but the above makes more sense :) + # My older explanation for this was that the "AddCounter" process + # forced the CPU to 100%, but the above makes more sense :) import win32pdh if format is None: format = win32pdh.PDH_FMT_LONG - path = win32pdh.MakeCounterPath( (machine, object, instance, None, inum, counter)) + path = win32pdh.MakeCounterPath( (machine, object, instance, None, + inum, counter)) hq = win32pdh.OpenQuery() try: hc = win32pdh.AddCounter(hq, path) @@ -186,7 +187,7 @@ def memusage(processName="python", instance=0): win32pdh.PDH_FMT_LONG, None) elif sys.platform[:5] == 'linux': - def memusage(_proc_pid_stat='/proc/%s/stat' % (os.getpid())): + def memusage(_proc_pid_stat=f'/proc/{os.getpid()}/stat'): """ Return virtual memory size in bytes of the running python. @@ -207,8 +208,7 @@ def memusage(): if sys.platform[:5] == 'linux': - def jiffies(_proc_pid_stat='/proc/%s/stat' % (os.getpid()), - _load_time=[]): + def jiffies(_proc_pid_stat=f'/proc/{os.getpid()}/stat', _load_time=[]): """ Return number of jiffies elapsed. @@ -263,11 +263,11 @@ def build_err_msg(arrays, err_msg, header='Items are not equal:', try: r = r_func(a) except Exception as exc: - r = '[repr failed for <{}>: {}]'.format(type(a).__name__, exc) + r = f'[repr failed for <{type(a).__name__}>: {exc}]' if r.count('\n') > 3: r = '\n'.join(r.splitlines()[:3]) r += '...' - msg.append(' %s: %s' % (names[i], r)) + msg.append(f' {names[i]}: {r}') return '\n'.join(msg) @@ -329,12 +329,14 @@ def assert_equal(actual, desired, err_msg='', verbose=True): for k, i in desired.items(): if k not in actual: raise AssertionError(repr(k)) - assert_equal(actual[k], desired[k], 'key=%r\n%s' % (k, err_msg), verbose) + assert_equal(actual[k], desired[k], f'key={k!r}\n{err_msg}', + verbose) return if isinstance(desired, (list, tuple)) and isinstance(actual, (list, tuple)): assert_equal(len(actual), len(desired), err_msg, verbose) for k in range(len(desired)): - assert_equal(actual[k], desired[k], 'item=%r\n%s' % (k, err_msg), verbose) + assert_equal(actual[k], desired[k], f'item={k!r}\n{err_msg}', + verbose) return from numpy.core import ndarray, isscalar, signbit from numpy.lib import iscomplexobj, real, imag @@ -694,9 +696,8 @@ def assert_approx_equal(actual,desired,significant=7,err_msg='',verbose=True): raise AssertionError(msg) -def assert_array_compare(comparison, x, y, err_msg='', verbose=True, - header='', precision=6, equal_nan=True, - equal_inf=True): +def assert_array_compare(comparison, x, y, err_msg='', verbose=True, header='', + precision=6, equal_nan=True, equal_inf=True): __tracebackhide__ = True # Hide traceback for py.test from numpy.core import array, array2string, isnan, inf, bool_, errstate, all, max, object_ @@ -754,8 +755,7 @@ def func_assert_same_pos(x, y, func=isnan, hasval='nan'): if not cond: msg = build_err_msg([x, y], err_msg - + '\n(shapes %s, %s mismatch)' % (x.shape, - y.shape), + + f'\n(shapes {x.shape}, {y.shape} mismatch)', verbose=verbose, header=header, names=('x', 'y'), precision=precision) raise AssertionError(msg) @@ -843,7 +843,7 @@ def func_assert_same_pos(x, y, func=isnan, hasval='nan'): except ValueError: import traceback efmt = traceback.format_exc() - header = 'error during assertion:\n\n%s\n\n%s' % (efmt, header) + header = f'error during assertion:\n\n{efmt}\n\n{header}' msg = build_err_msg([x, y], err_msg, verbose=verbose, header=header, names=('x', 'y'), precision=precision) @@ -1170,7 +1170,8 @@ def assert_string_equal(actual, desired): if desired == actual: return - diff = list(difflib.Differ().compare(actual.splitlines(True), desired.splitlines(True))) + diff = list(difflib.Differ().compare(actual.splitlines(True), + desired.splitlines(True))) diff_list = [] while diff: d1 = diff.pop(0) @@ -1198,7 +1199,7 @@ def assert_string_equal(actual, desired): raise AssertionError(repr(d1)) if not diff_list: return - msg = 'Differences in strings:\n%s' % (''.join(diff_list)).rstrip() + msg = f"Differences in strings:\n{''.join(diff_list).rstrip()}" if actual != desired: raise AssertionError(msg) @@ -1434,9 +1435,7 @@ def measure(code_str, times=1, label=None): frame = sys._getframe(1) locs, globs = frame.f_locals, frame.f_globals - code = compile(code_str, - 'Test name: %s ' % label, - 'exec') + code = compile(code_str, f'Test name: {label} ', 'exec') i = 0 elapsed = jiffies() while i < times: @@ -1525,7 +1524,7 @@ def compare(x, y): equal_nan=equal_nan) actual, desired = np.asanyarray(actual), np.asanyarray(desired) - header = 'Not equal to tolerance rtol=%g, atol=%g' % (rtol, atol) + header = f'Not equal to tolerance rtol={rtol:g}, atol={atol:g}' assert_array_compare(compare, actual, desired, err_msg=str(err_msg), verbose=verbose, header=header, equal_nan=equal_nan) @@ -1724,8 +1723,8 @@ def _integer_repr(x, vdt, comp): def integer_repr(x): - """Return the signed-magnitude interpretation of the binary representation of - x.""" + """Return the signed-magnitude interpretation of the binary representation + of x.""" import numpy as np if x.dtype == np.float16: return _integer_repr(x, np.int16, np.int16(-2**15)) @@ -1734,7 +1733,7 @@ def integer_repr(x): elif x.dtype == np.float64: return _integer_repr(x, np.int64, np.int64(-2**63)) else: - raise ValueError("Unsupported dtype %s" % x.dtype) + raise ValueError(f'Unsupported dtype {x.dtype}') @contextlib.contextmanager @@ -1744,7 +1743,7 @@ def _assert_warns_context(warning_class, name=None): l = sup.record(warning_class) yield if not len(l) > 0: - name_str = " when calling %s" % name if name is not None else "" + name_str = f' when calling {name}' if name is not None else '' raise AssertionError("No warning raised" + name_str) @@ -1809,8 +1808,8 @@ def _assert_no_warnings_context(name=None): warnings.simplefilter('always') yield if len(l) > 0: - name_str = " when calling %s" % name if name is not None else "" - raise AssertionError("Got warnings%s: %s" % (name_str, l)) + name_str = f' when calling {name}' if name is not None else '' + raise AssertionError(f'Got warnings{name_str}: {l}') def assert_no_warnings(*args, **kwargs): @@ -2322,8 +2321,8 @@ def _assert_no_gc_cycles_context(name=None): break else: raise RuntimeError( - "Unable to fully collect garbage - perhaps a __del__ method is " - "creating more reference cycles?") + "Unable to fully collect garbage - perhaps a __del__ method " + "is creating more reference cycles?") gc.set_debug(gc.DEBUG_SAVEALL) yield @@ -2337,7 +2336,7 @@ def _assert_no_gc_cycles_context(name=None): gc.enable() if n_objects_in_cycles: - name_str = " when calling %s" % name if name is not None else "" + name_str = f' when calling {name}' if name is not None else '' raise AssertionError( "Reference cycles were found{}: {} objects were collected, " "of which {} are shown below:{}" @@ -2441,12 +2440,10 @@ def check_free_memory(free_bytes): try: mem_free = _parse_size(env_value) except ValueError as exc: - raise ValueError('Invalid environment variable {}: {!s}'.format( - env_var, exc)) + raise ValueError(f'Invalid environment variable {env_var}: {exc}') - msg = ('{0} GB memory required, but environment variable ' - 'NPY_AVAILABLE_MEM={1} set'.format( - free_bytes/1e9, env_value)) + msg = (f'{free_bytes/1e9} GB memory required, but environment variable ' + f'NPY_AVAILABLE_MEM={env_value} set') else: mem_free = _get_mem_available() @@ -2456,8 +2453,7 @@ def check_free_memory(free_bytes): "the test.") mem_free = -1 else: - msg = '{0} GB memory required, but {1} GB available'.format( - free_bytes/1e9, mem_free/1e9) + msg = f'{free_bytes/1e9} GB memory required, but {mem_free/1e9} GB available' return msg if mem_free < free_bytes else None @@ -2474,7 +2470,7 @@ def _parse_size(size_str): m = size_re.match(size_str.lower()) if not m or m.group(2) not in suffixes: - raise ValueError("value {!r} not a valid size".format(size_str)) + raise ValueError(f'value {size_str!r} not a valid size') return int(float(m.group(1)) * suffixes[m.group(2)]) diff --git a/numpy/testing/setup.py b/numpy/testing/setup.py index 13191f13fe0c..7652a94a2660 100755 --- a/numpy/testing/setup.py +++ b/numpy/testing/setup.py @@ -6,6 +6,7 @@ def configuration(parent_package='',top_path=None): config.add_subpackage('_private') config.add_subpackage('tests') + config.add_data_files('*.pyi') return config if __name__ == '__main__': diff --git a/numpy/tests/test_public_api.py b/numpy/tests/test_public_api.py index 21b8b838f5be..a7bd0f115d15 100644 --- a/numpy/tests/test_public_api.py +++ b/numpy/tests/test_public_api.py @@ -240,6 +240,7 @@ def test_NPY_NO_EXPORT(): "distutils.fcompiler.none", "distutils.fcompiler.pathf95", "distutils.fcompiler.pg", + "distutils.fcompiler.nv", "distutils.fcompiler.sun", "distutils.fcompiler.vast", "distutils.from_template", @@ -348,7 +349,7 @@ def test_all_modules_are_expected(): modnames.append(modname) if modnames: - raise AssertionError("Found unexpected modules: {}".format(modnames)) + raise AssertionError(f'Found unexpected modules: {modnames}') # Stuff that clearly shouldn't be in the API and is detected by the next test diff --git a/numpy/typing/__init__.py b/numpy/typing/__init__.py index 86fd5e7870e2..cbde75462332 100644 --- a/numpy/typing/__init__.py +++ b/numpy/typing/__init__.py @@ -14,10 +14,10 @@ addition, the following type aliases are available for users. - ``typing.ArrayLike``: objects that can be converted to arrays -- ``typing.DtypeLike``: objects that can be converted to dtypes +- ``typing.DTypeLike``: objects that can be converted to dtypes Roughly speaking, ``typing.ArrayLike`` is "objects that can be used as -inputs to ``np.array``" and ``typing.DtypeLike`` is "objects that can +inputs to ``np.array``" and ``typing.DTypeLike`` is "objects that can be used as inputs to ``np.dtype``". .. _typing-extensions: https://pypi.org/project/typing-extensions/ @@ -89,12 +89,103 @@ since its usage is discouraged. Please see : https://numpy.org/devdocs/reference/arrays.dtypes.html +NBitBase +~~~~~~~~ + +.. autoclass:: numpy.typing.NBitBase + """ + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + import sys + if sys.version_info >= (3, 8): + from typing import final + else: + from typing_extensions import final +else: + def final(f): return f + + +@final # Dissallow the creation of arbitrary `NBitBase` subclasses +class NBitBase: + """ + An object representing `numpy.number` precision during static type checking. + + Used exclusively for the purpose static type checking, `NBitBase` + represents the base of a hierachieral set of subclasses. + Each subsequent subclass is herein used for representing a lower level + of precision, *e.g.* ``64Bit > 32Bit > 16Bit``. + + Examples + -------- + Below is a typical usage example: `NBitBase` is herein used for annotating a + function that takes a float and integer of arbitrary precision as arguments + and returns a new float of whichever precision is largest + (*e.g.* ``np.float16 + np.int64 -> np.float64``). + + .. code-block:: python + + >>> from typing import TypeVar, TYPE_CHECKING + >>> import numpy as np + >>> import numpy.typing as npt + + >>> T = TypeVar("T", bound=npt.NBitBase) + + >>> def add(a: "np.floating[T]", b: "np.integer[T]") -> "np.floating[T]": + ... return a + b + + >>> a = np.float16() + >>> b = np.int64() + >>> out = add(a, b) + + >>> if TYPE_CHECKING: + ... reveal_locals() + ... # note: Revealed local types are: + ... # note: a: numpy.floating[numpy.typing._16Bit*] + ... # note: b: numpy.signedinteger[numpy.typing._64Bit*] + ... # note: out: numpy.floating[numpy.typing._64Bit*] + + """ + + def __init_subclass__(cls) -> None: + allowed_names = { + "NBitBase", "_256Bit", "_128Bit", "_96Bit", "_80Bit", + "_64Bit", "_32Bit", "_16Bit", "_8Bit", + } + if cls.__name__ not in allowed_names: + raise TypeError('cannot inherit from final class "NBitBase"') + super().__init_subclass__() + + +# Silence errors about subclassing a `@final`-decorated class +class _256Bit(NBitBase): ... # type: ignore[misc] +class _128Bit(_256Bit): ... # type: ignore[misc] +class _96Bit(_128Bit): ... # type: ignore[misc] +class _80Bit(_96Bit): ... # type: ignore[misc] +class _64Bit(_80Bit): ... # type: ignore[misc] +class _32Bit(_64Bit): ... # type: ignore[misc] +class _16Bit(_32Bit): ... # type: ignore[misc] +class _8Bit(_16Bit): ... # type: ignore[misc] + +# Clean up the namespace +del TYPE_CHECKING, final + +from ._scalars import ( + _CharLike, + _BoolLike, + _IntLike, + _FloatLike, + _ComplexLike, + _NumberLike, + _ScalarLike, + _VoidLike, +) from ._array_like import _SupportsArray, ArrayLike from ._shape import _Shape, _ShapeLike -from ._dtype_like import DtypeLike +from ._dtype_like import _SupportsDType, _VoidDTypeLike, DTypeLike from numpy._pytesttester import PytestTester test = PytestTester(__name__) del PytestTester - diff --git a/numpy/typing/_array_like.py b/numpy/typing/_array_like.py index 76c0c839c567..a1a604239390 100644 --- a/numpy/typing/_array_like.py +++ b/numpy/typing/_array_like.py @@ -2,7 +2,8 @@ from typing import Any, overload, Sequence, TYPE_CHECKING, Union from numpy import ndarray -from ._dtype_like import DtypeLike +from ._scalars import _ScalarLike +from ._dtype_like import DTypeLike if sys.version_info >= (3, 8): from typing import Protocol @@ -18,9 +19,9 @@ if TYPE_CHECKING or HAVE_PROTOCOL: class _SupportsArray(Protocol): @overload - def __array__(self, __dtype: DtypeLike = ...) -> ndarray: ... + def __array__(self, __dtype: DTypeLike = ...) -> ndarray: ... @overload - def __array__(self, dtype: DtypeLike = ...) -> ndarray: ... + def __array__(self, dtype: DTypeLike = ...) -> ndarray: ... else: _SupportsArray = Any @@ -31,4 +32,9 @@ def __array__(self, dtype: DtypeLike = ...) -> ndarray: ... # is resolved. See also the mypy issue: # # https://github.com/python/typing/issues/593 -ArrayLike = Union[bool, int, float, complex, _SupportsArray, Sequence] +ArrayLike = Union[ + _ScalarLike, + Sequence[_ScalarLike], + Sequence[Sequence[Any]], # TODO: Wait for support for recursive types + _SupportsArray, +] diff --git a/numpy/typing/_callable.py b/numpy/typing/_callable.py new file mode 100644 index 000000000000..91b7a4ec21c0 --- /dev/null +++ b/numpy/typing/_callable.py @@ -0,0 +1,336 @@ +""" +A module with various ``typing.Protocol`` subclasses that implement +the ``__call__`` magic method. + +See the `Mypy documentation`_ on protocols for more details. + +.. _`Mypy documentation`: https://mypy.readthedocs.io/en/stable/protocols.html#callback-protocols + +""" + +import sys +from typing import ( + Union, + TypeVar, + overload, + Any, + Tuple, + NoReturn, + TYPE_CHECKING, +) + +from numpy import ( + generic, + bool_, + timedelta64, + number, + integer, + unsignedinteger, + signedinteger, + int8, + floating, + float64, + complexfloating, + complex128, +) +from ._scalars import ( + _BoolLike, + _IntLike, + _FloatLike, + _ComplexLike, + _NumberLike, +) +from . import NBitBase + +if sys.version_info >= (3, 8): + from typing import Protocol + HAVE_PROTOCOL = True +else: + try: + from typing_extensions import Protocol + except ImportError: + HAVE_PROTOCOL = False + else: + HAVE_PROTOCOL = True + +if TYPE_CHECKING or HAVE_PROTOCOL: + _T = TypeVar("_T") + _2Tuple = Tuple[_T, _T] + + _NBit_co = TypeVar("_NBit_co", covariant=True, bound=NBitBase) + _NBit = TypeVar("_NBit", bound=NBitBase) + + _IntType = TypeVar("_IntType", bound=integer) + _FloatType = TypeVar("_FloatType", bound=floating) + _NumberType = TypeVar("_NumberType", bound=number) + _NumberType_co = TypeVar("_NumberType_co", covariant=True, bound=number) + _GenericType_co = TypeVar("_GenericType_co", covariant=True, bound=generic) + + class _BoolOp(Protocol[_GenericType_co]): + @overload + def __call__(self, __other: _BoolLike) -> _GenericType_co: ... + @overload # platform dependent + def __call__(self, __other: int) -> signedinteger[Any]: ... + @overload + def __call__(self, __other: float) -> float64: ... + @overload + def __call__(self, __other: complex) -> complex128: ... + @overload + def __call__(self, __other: _NumberType) -> _NumberType: ... + + class _BoolBitOp(Protocol[_GenericType_co]): + @overload + def __call__(self, __other: _BoolLike) -> _GenericType_co: ... + @overload # platform dependent + def __call__(self, __other: int) -> signedinteger[Any]: ... + @overload + def __call__(self, __other: _IntType) -> _IntType: ... + + class _BoolSub(Protocol): + # Note that `__other: bool_` is absent here + @overload + def __call__(self, __other: bool) -> NoReturn: ... + @overload # platform dependent + def __call__(self, __other: int) -> signedinteger[Any]: ... + @overload + def __call__(self, __other: float) -> float64: ... + @overload + def __call__(self, __other: complex) -> complex128: ... + @overload + def __call__(self, __other: _NumberType) -> _NumberType: ... + + class _BoolTrueDiv(Protocol): + @overload + def __call__(self, __other: Union[float, _IntLike, _BoolLike]) -> float64: ... + @overload + def __call__(self, __other: complex) -> complex128: ... + @overload + def __call__(self, __other: _NumberType) -> _NumberType: ... + + class _BoolMod(Protocol): + @overload + def __call__(self, __other: _BoolLike) -> int8: ... + @overload # platform dependent + def __call__(self, __other: int) -> signedinteger[Any]: ... + @overload + def __call__(self, __other: float) -> float64: ... + @overload + def __call__(self, __other: _IntType) -> _IntType: ... + @overload + def __call__(self, __other: _FloatType) -> _FloatType: ... + + class _BoolDivMod(Protocol): + @overload + def __call__(self, __other: _BoolLike) -> _2Tuple[int8]: ... + @overload # platform dependent + def __call__(self, __other: int) -> _2Tuple[signedinteger[Any]]: ... + @overload + def __call__(self, __other: float) -> _2Tuple[float64]: ... + @overload + def __call__(self, __other: _IntType) -> _2Tuple[_IntType]: ... + @overload + def __call__(self, __other: _FloatType) -> _2Tuple[_FloatType]: ... + + class _TD64Div(Protocol[_NumberType_co]): + @overload + def __call__(self, __other: timedelta64) -> _NumberType_co: ... + @overload + def __call__(self, __other: _FloatLike) -> timedelta64: ... + + class _IntTrueDiv(Protocol[_NBit_co]): + @overload + def __call__(self, __other: bool) -> floating[_NBit_co]: ... + @overload + def __call__(self, __other: int) -> floating[Any]: ... + @overload + def __call__(self, __other: float) -> float64: ... + @overload + def __call__(self, __other: complex) -> complex128: ... + @overload + def __call__(self, __other: integer[_NBit]) -> floating[Union[_NBit_co, _NBit]]: ... + + class _UnsignedIntOp(Protocol[_NBit_co]): + # NOTE: `uint64 + signedinteger -> float64` + @overload + def __call__(self, __other: bool) -> unsignedinteger[_NBit_co]: ... + @overload + def __call__( + self, __other: Union[int, signedinteger[Any]] + ) -> Union[signedinteger[Any], float64]: ... + @overload + def __call__(self, __other: float) -> float64: ... + @overload + def __call__(self, __other: complex) -> complex128: ... + @overload + def __call__( + self, __other: unsignedinteger[_NBit] + ) -> unsignedinteger[Union[_NBit_co, _NBit]]: ... + + class _UnsignedIntBitOp(Protocol[_NBit_co]): + @overload + def __call__(self, __other: bool) -> unsignedinteger[_NBit_co]: ... + @overload + def __call__(self, __other: int) -> signedinteger[Any]: ... + @overload + def __call__(self, __other: signedinteger[Any]) -> signedinteger[Any]: ... + @overload + def __call__( + self, __other: unsignedinteger[_NBit] + ) -> unsignedinteger[Union[_NBit_co, _NBit]]: ... + + class _UnsignedIntMod(Protocol[_NBit_co]): + @overload + def __call__(self, __other: bool) -> unsignedinteger[_NBit_co]: ... + @overload + def __call__( + self, __other: Union[int, signedinteger[Any]] + ) -> Union[signedinteger[Any], float64]: ... + @overload + def __call__(self, __other: float) -> float64: ... + @overload + def __call__( + self, __other: unsignedinteger[_NBit] + ) -> unsignedinteger[Union[_NBit_co, _NBit]]: ... + + class _UnsignedIntDivMod(Protocol[_NBit_co]): + @overload + def __call__(self, __other: bool) -> _2Tuple[signedinteger[_NBit_co]]: ... + @overload + def __call__( + self, __other: Union[int, signedinteger[Any]] + ) -> Union[_2Tuple[signedinteger[Any]], _2Tuple[float64]]: ... + @overload + def __call__(self, __other: float) -> _2Tuple[float64]: ... + @overload + def __call__( + self, __other: unsignedinteger[_NBit] + ) -> _2Tuple[unsignedinteger[Union[_NBit_co, _NBit]]]: ... + + class _SignedIntOp(Protocol[_NBit_co]): + @overload + def __call__(self, __other: bool) -> signedinteger[_NBit_co]: ... + @overload + def __call__(self, __other: int) -> signedinteger[Any]: ... + @overload + def __call__(self, __other: float) -> float64: ... + @overload + def __call__(self, __other: complex) -> complex128: ... + @overload + def __call__( + self, __other: signedinteger[_NBit] + ) -> signedinteger[Union[_NBit_co, _NBit]]: ... + + class _SignedIntBitOp(Protocol[_NBit_co]): + @overload + def __call__(self, __other: bool) -> signedinteger[_NBit_co]: ... + @overload + def __call__(self, __other: int) -> signedinteger[Any]: ... + @overload + def __call__( + self, __other: signedinteger[_NBit] + ) -> signedinteger[Union[_NBit_co, _NBit]]: ... + + class _SignedIntMod(Protocol[_NBit_co]): + @overload + def __call__(self, __other: bool) -> signedinteger[_NBit_co]: ... + @overload + def __call__(self, __other: int) -> signedinteger[Any]: ... + @overload + def __call__(self, __other: float) -> float64: ... + @overload + def __call__( + self, __other: signedinteger[_NBit] + ) -> signedinteger[Union[_NBit_co, _NBit]]: ... + + class _SignedIntDivMod(Protocol[_NBit_co]): + @overload + def __call__(self, __other: bool) -> _2Tuple[signedinteger[_NBit_co]]: ... + @overload + def __call__(self, __other: int) -> _2Tuple[signedinteger[Any]]: ... + @overload + def __call__(self, __other: float) -> _2Tuple[float64]: ... + @overload + def __call__( + self, __other: signedinteger[_NBit] + ) -> _2Tuple[signedinteger[Union[_NBit_co, _NBit]]]: ... + + class _FloatOp(Protocol[_NBit_co]): + @overload + def __call__(self, __other: bool) -> floating[_NBit_co]: ... + @overload + def __call__(self, __other: int) -> floating[Any]: ... + @overload + def __call__(self, __other: float) -> float64: ... + @overload + def __call__(self, __other: complex) -> complex128: ... + @overload + def __call__( + self, __other: Union[integer[_NBit], floating[_NBit]] + ) -> floating[Union[_NBit_co, _NBit]]: ... + + class _FloatMod(Protocol[_NBit_co]): + @overload + def __call__(self, __other: bool) -> floating[_NBit_co]: ... + @overload + def __call__(self, __other: int) -> floating[Any]: ... + @overload + def __call__(self, __other: float) -> float64: ... + @overload + def __call__( + self, __other: Union[integer[_NBit], floating[_NBit]] + ) -> floating[Union[_NBit_co, _NBit]]: ... + + class _FloatDivMod(Protocol[_NBit_co]): + @overload + def __call__(self, __other: bool) -> _2Tuple[floating[_NBit_co]]: ... + @overload + def __call__(self, __other: int) -> _2Tuple[floating[Any]]: ... + @overload + def __call__(self, __other: float) -> _2Tuple[float64]: ... + @overload + def __call__( + self, __other: Union[integer[_NBit], floating[_NBit]] + ) -> _2Tuple[floating[Union[_NBit_co, _NBit]]]: ... + + class _ComplexOp(Protocol[_NBit_co]): + @overload + def __call__(self, __other: bool) -> complexfloating[_NBit_co, _NBit_co]: ... + @overload + def __call__(self, __other: int) -> complexfloating[Any, Any]: ... + @overload + def __call__(self, __other: Union[float, complex]) -> complex128: ... + @overload + def __call__( + self, + __other: Union[ + integer[_NBit], + floating[_NBit], + complexfloating[_NBit, _NBit], + ] + ) -> complexfloating[Union[_NBit_co, _NBit], Union[_NBit_co, _NBit]]: ... + + class _NumberOp(Protocol): + def __call__(self, __other: _NumberLike) -> number: ... + +else: + _BoolOp = Any + _BoolBitOp = Any + _BoolSub = Any + _BoolTrueDiv = Any + _BoolMod = Any + _BoolDivMod = Any + _TD64Div = Any + _IntTrueDiv = Any + _UnsignedIntOp = Any + _UnsignedIntBitOp = Any + _UnsignedIntMod = Any + _UnsignedIntDivMod = Any + _SignedIntOp = Any + _SignedIntBitOp = Any + _SignedIntMod = Any + _SignedIntDivMod = Any + _FloatOp = Any + _FloatMod = Any + _FloatDivMod = Any + _ComplexOp = Any + _NumberOp = Any diff --git a/numpy/typing/_dtype_like.py b/numpy/typing/_dtype_like.py index 7c1946a3e8b5..1953bd5fcfcc 100644 --- a/numpy/typing/_dtype_like.py +++ b/numpy/typing/_dtype_like.py @@ -15,45 +15,36 @@ else: HAVE_PROTOCOL = True -_DtypeLikeNested = Any # TODO: wait for support for recursive types +_DTypeLikeNested = Any # TODO: wait for support for recursive types if TYPE_CHECKING or HAVE_PROTOCOL: # Mandatory keys - class _DtypeDictBase(TypedDict): + class _DTypeDictBase(TypedDict): names: Sequence[str] - formats: Sequence[_DtypeLikeNested] + formats: Sequence[_DTypeLikeNested] # Mandatory + optional keys - class _DtypeDict(_DtypeDictBase, total=False): + class _DTypeDict(_DTypeDictBase, total=False): offsets: Sequence[int] titles: Sequence[Any] # Only `str` elements are usable as indexing aliases, but all objects are legal itemsize: int aligned: bool # A protocol for anything with the dtype attribute - class _SupportsDtype(Protocol): - dtype: _DtypeLikeNested + class _SupportsDType(Protocol): + dtype: _DTypeLikeNested else: - _DtypeDict = Any - _SupportsDtype = Any + _DTypeDict = Any + _SupportsDType = Any -# Anything that can be coerced into numpy.dtype. -# Reference: https://docs.scipy.org/doc/numpy/reference/arrays.dtypes.html -DtypeLike = Union[ - dtype, - # default data type (float64) - None, - # array-scalar types and generic types - type, # TODO: enumerate these when we add type hints for numpy scalars - # anything with a dtype attribute - _SupportsDtype, - # character codes, type strings or comma-separated fields, e.g., 'float64' - str, + +# Would create a dtype[np.void] +_VoidDTypeLike = Union[ # (flexible_dtype, itemsize) - Tuple[_DtypeLikeNested, int], + Tuple[_DTypeLikeNested, int], # (fixed_dtype, shape) - Tuple[_DtypeLikeNested, _ShapeLike], + Tuple[_DTypeLikeNested, _ShapeLike], # [(field_name, field_dtype, field_shape), ...] # # The type here is quite broad because NumPy accepts quite a wide @@ -62,14 +53,29 @@ class _SupportsDtype(Protocol): List[Any], # {'names': ..., 'formats': ..., 'offsets': ..., 'titles': ..., # 'itemsize': ...} - _DtypeDict, + _DTypeDict, # (base_dtype, new_dtype) - Tuple[_DtypeLikeNested, _DtypeLikeNested], + Tuple[_DTypeLikeNested, _DTypeLikeNested], +] + +# Anything that can be coerced into numpy.dtype. +# Reference: https://docs.scipy.org/doc/numpy/reference/arrays.dtypes.html +DTypeLike = Union[ + dtype, + # default data type (float64) + None, + # array-scalar types and generic types + type, # TODO: enumerate these when we add type hints for numpy scalars + # anything with a dtype attribute + _SupportsDType, + # character codes, type strings or comma-separated fields, e.g., 'float64' + str, + _VoidDTypeLike, ] # NOTE: while it is possible to provide the dtype as a dict of # dtype-like objects (e.g. `{'field1': ..., 'field2': ..., ...}`), # this syntax is officially discourged and -# therefore not included in the Union defining `DtypeLike`. +# therefore not included in the Union defining `DTypeLike`. # # See https://github.com/numpy/numpy/issues/16891 for more details. diff --git a/numpy/typing/_scalars.py b/numpy/typing/_scalars.py new file mode 100644 index 000000000000..e4fc28b074ac --- /dev/null +++ b/numpy/typing/_scalars.py @@ -0,0 +1,26 @@ +from typing import Union, Tuple, Any + +import numpy as np + +# NOTE: `_StrLike` and `_BytesLike` are pointless, as `np.str_` and `np.bytes_` +# are already subclasses of their builtin counterpart + +_CharLike = Union[str, bytes] + +_BoolLike = Union[bool, np.bool_] +_IntLike = Union[int, np.integer] +_FloatLike = Union[_IntLike, float, np.floating] +_ComplexLike = Union[_FloatLike, complex, np.complexfloating] +_NumberLike = Union[int, float, complex, np.number, np.bool_] + +_ScalarLike = Union[ + int, + float, + complex, + str, + bytes, + np.generic, +] + +# `_VoidLike` is technically not a scalar, but it's close enough +_VoidLike = Union[Tuple[Any, ...], np.void] diff --git a/numpy/typing/setup.py b/numpy/typing/setup.py index c444e769fb6d..694a756dc5ab 100644 --- a/numpy/typing/setup.py +++ b/numpy/typing/setup.py @@ -3,6 +3,7 @@ def configuration(parent_package='', top_path=None): config = Configuration('typing', parent_package, top_path) config.add_subpackage('tests') config.add_data_dir('tests/data') + config.add_data_files('*.pyi') return config diff --git a/numpy/typing/tests/data/fail/arithmetic.py b/numpy/typing/tests/data/fail/arithmetic.py new file mode 100644 index 000000000000..f32eddc4bc8d --- /dev/null +++ b/numpy/typing/tests/data/fail/arithmetic.py @@ -0,0 +1,16 @@ +import numpy as np + +b_ = np.bool_() +dt = np.datetime64(0, "D") +td = np.timedelta64(0, "D") + +b_ - b_ # E: No overload variant + +dt + dt # E: Unsupported operand types +td - dt # E: Unsupported operand types +td % 1 # E: Unsupported operand types +td / dt # E: No overload +td % dt # E: Unsupported operand types + +-b_ # E: Unsupported operand type ++b_ # E: Unsupported operand type diff --git a/numpy/typing/tests/data/fail/array_constructors.py b/numpy/typing/tests/data/fail/array_constructors.py new file mode 100644 index 000000000000..9cb59fe5f72e --- /dev/null +++ b/numpy/typing/tests/data/fail/array_constructors.py @@ -0,0 +1,31 @@ +import numpy as np + +a: np.ndarray +generator = (i for i in range(10)) + +np.require(a, requirements=1) # E: No overload variant +np.require(a, requirements="TEST") # E: incompatible type + +np.zeros("test") # E: incompatible type +np.zeros() # E: Too few arguments + +np.ones("test") # E: incompatible type +np.ones() # E: Too few arguments + +np.array(0, float, True) # E: Too many positional + +np.linspace(None, 'bob') # E: No overload variant +np.linspace(0, 2, num=10.0) # E: No overload variant +np.linspace(0, 2, endpoint='True') # E: No overload variant +np.linspace(0, 2, retstep=b'False') # E: No overload variant +np.linspace(0, 2, dtype=0) # E: No overload variant +np.linspace(0, 2, axis=None) # E: No overload variant + +np.logspace(None, 'bob') # E: Argument 1 +np.logspace(0, 2, base=None) # E: Argument "base" + +np.geomspace(None, 'bob') # E: Argument 1 + +np.stack(generator) # E: No overload variant +np.hstack({1, 2}) # E: incompatible type +np.vstack(1) # E: incompatible type diff --git a/numpy/typing/tests/data/fail/bitwise_ops.py b/numpy/typing/tests/data/fail/bitwise_ops.py new file mode 100644 index 000000000000..8a8f89755a36 --- /dev/null +++ b/numpy/typing/tests/data/fail/bitwise_ops.py @@ -0,0 +1,20 @@ +import numpy as np + +i8 = np.int64() +i4 = np.int32() +u8 = np.uint64() +b_ = np.bool_() +i = int() + +f8 = np.float64() + +b_ >> f8 # E: No overload variant +i8 << f8 # E: No overload variant +i | f8 # E: Unsupported operand types +i8 ^ f8 # E: No overload variant +u8 & f8 # E: No overload variant +~f8 # E: Unsupported operand type + +# mypys' error message for `NoReturn` is unfortunately pretty bad +# TODO: Reenable this once we add support for numerical precision for `number`s +# a = u8 | 0 # E: Need type annotation diff --git a/numpy/typing/tests/data/fail/constants.py b/numpy/typing/tests/data/fail/constants.py new file mode 100644 index 000000000000..67ee0e0bc0bf --- /dev/null +++ b/numpy/typing/tests/data/fail/constants.py @@ -0,0 +1,6 @@ +import numpy as np + +np.Inf = np.Inf # E: Cannot assign to final +np.ALLOW_THREADS = np.ALLOW_THREADS # E: Cannot assign to final +np.little_endian = np.little_endian # E: Cannot assign to final +np.UFUNC_PYVALS_NAME = np.UFUNC_PYVALS_NAME # E: Cannot assign to final diff --git a/numpy/typing/tests/data/fail/dtype.py b/numpy/typing/tests/data/fail/dtype.py index 3dc027daf243..7d4783d8f651 100644 --- a/numpy/typing/tests/data/fail/dtype.py +++ b/numpy/typing/tests/data/fail/dtype.py @@ -1,15 +1,16 @@ import numpy as np - class Test: not_dtype = float -np.dtype(Test()) # E: Argument 1 to "dtype" has incompatible type +np.dtype(Test()) # E: No overload variant of "dtype" matches -np.dtype( - { # E: Argument 1 to "dtype" has incompatible type +np.dtype( # E: No overload variant of "dtype" matches + { "field1": (float, 1), "field2": (int, 3), } ) + +np.dtype[np.float64](np.int64) # E: Argument 1 to "dtype" has incompatible type diff --git a/numpy/typing/tests/data/fail/flatiter.py b/numpy/typing/tests/data/fail/flatiter.py index e8a82344f622..544ffbe4a7db 100644 --- a/numpy/typing/tests/data/fail/flatiter.py +++ b/numpy/typing/tests/data/fail/flatiter.py @@ -1,7 +1,7 @@ from typing import Any import numpy as np -from numpy.typing import DtypeLike, _SupportsArray +from numpy.typing import _SupportsArray class Index: diff --git a/numpy/typing/tests/data/fail/linspace.py b/numpy/typing/tests/data/fail/linspace.py deleted file mode 100644 index a9769c5d6fb1..000000000000 --- a/numpy/typing/tests/data/fail/linspace.py +++ /dev/null @@ -1,13 +0,0 @@ -import numpy as np - -np.linspace(None, 'bob') # E: No overload variant -np.linspace(0, 2, num=10.0) # E: No overload variant -np.linspace(0, 2, endpoint='True') # E: No overload variant -np.linspace(0, 2, retstep=b'False') # E: No overload variant -np.linspace(0, 2, dtype=0) # E: No overload variant -np.linspace(0, 2, axis=None) # E: No overload variant - -np.logspace(None, 'bob') # E: Argument 1 -np.logspace(0, 2, base=None) # E: Argument "base" - -np.geomspace(None, 'bob') # E: Argument 1 diff --git a/numpy/typing/tests/data/fail/modules.py b/numpy/typing/tests/data/fail/modules.py index e7ffe89207f0..5e2d820abc85 100644 --- a/numpy/typing/tests/data/fail/modules.py +++ b/numpy/typing/tests/data/fail/modules.py @@ -1,3 +1,10 @@ import numpy as np np.testing.bob # E: Module has no attribute +np.bob # E: Module has no attribute + +# Stdlib modules in the namespace by accident +np.warnings # E: Module has no attribute +np.sys # E: Module has no attribute +np.os # E: Module has no attribute +np.math # E: Module has no attribute diff --git a/numpy/typing/tests/data/fail/ndarray_misc.py b/numpy/typing/tests/data/fail/ndarray_misc.py new file mode 100644 index 000000000000..1e1496bfecca --- /dev/null +++ b/numpy/typing/tests/data/fail/ndarray_misc.py @@ -0,0 +1,21 @@ +""" +Tests for miscellaneous (non-magic) ``np.ndarray``/``np.generic`` methods. + +More extensive tests are performed for the methods' +function-based counterpart in `../from_numeric.py`. + +""" + +import numpy as np + +f8: np.float64 + +f8.argpartition(0) # E: has no attribute +f8.diagonal() # E: has no attribute +f8.dot(1) # E: has no attribute +f8.nonzero() # E: has no attribute +f8.partition(0) # E: has no attribute +f8.put(0, 2) # E: has no attribute +f8.setfield(2, np.float64) # E: has no attribute +f8.sort() # E: has no attribute +f8.trace() # E: has no attribute diff --git a/numpy/typing/tests/data/fail/numerictypes.py b/numpy/typing/tests/data/fail/numerictypes.py index dd03eacc1c97..94537a23b682 100644 --- a/numpy/typing/tests/data/fail/numerictypes.py +++ b/numpy/typing/tests/data/fail/numerictypes.py @@ -10,4 +10,4 @@ np.issubdtype(1, np.int64) # E: incompatible type "int" -np.find_common_type(np.int64, np.int64) # E: incompatible type "Type[int64]" +np.find_common_type(np.int64, np.int64) # E: incompatible type "Type[signedinteger[Any]]" diff --git a/numpy/typing/tests/data/fail/scalars.py b/numpy/typing/tests/data/fail/scalars.py index 47c031163636..5fffa89b142e 100644 --- a/numpy/typing/tests/data/fail/scalars.py +++ b/numpy/typing/tests/data/fail/scalars.py @@ -28,22 +28,6 @@ np.datetime64(0) # E: non-matching overload -dt_64 = np.datetime64(0, "D") -td_64 = np.timedelta64(1, "h") - -dt_64 + dt_64 # E: Unsupported operand types -td_64 - dt_64 # E: Unsupported operand types -td_64 % 1 # E: Unsupported operand types - -# NOTE: The 2 tests below currently don't work due to the broad -# (i.e. untyped) signature of `generic.__truediv__()` and `.__mod__()`. -# TODO: Revisit this once annotations are added to the -# `_ArrayOrScalarCommon` magic methods. - -# td_64 / dt_64 # E: No overload -# td_64 % dt_64 # E: Unsupported operand types - - class A: def __float__(self): return 1.0 @@ -63,11 +47,7 @@ def __float__(self): np.generic(1) # E: Cannot instantiate abstract class np.number(1) # E: Cannot instantiate abstract class np.integer(1) # E: Cannot instantiate abstract class -np.signedinteger(1) # E: Cannot instantiate abstract class -np.unsignedinteger(1) # E: Cannot instantiate abstract class np.inexact(1) # E: Cannot instantiate abstract class -np.floating(1) # E: Cannot instantiate abstract class -np.complexfloating(1) # E: Cannot instantiate abstract class np.character("test") # E: Cannot instantiate abstract class np.flexible(b"test") # E: Cannot instantiate abstract class @@ -84,3 +64,5 @@ def __float__(self): np.bytes_(b"hello", encoding='utf-8') # E: No overload variant np.str_("hello", encoding='utf-8') # E: No overload variant + +complex(np.bytes_("1")) # E: No overload variant diff --git a/numpy/typing/tests/data/fail/simple.py b/numpy/typing/tests/data/fail/simple.py deleted file mode 100644 index 57c08fb7db7e..000000000000 --- a/numpy/typing/tests/data/fail/simple.py +++ /dev/null @@ -1,12 +0,0 @@ -"""Simple expression that should fail with mypy.""" - -import numpy as np - -# Array creation routines checks -np.zeros("test") # E: incompatible type -np.zeros() # E: Too few arguments - -np.ones("test") # E: incompatible type -np.ones() # E: Too few arguments - -np.array(0, float, True) # E: Too many positional diff --git a/numpy/typing/tests/data/pass/arithmetic.py b/numpy/typing/tests/data/pass/arithmetic.py new file mode 100644 index 000000000000..ffbaf2975332 --- /dev/null +++ b/numpy/typing/tests/data/pass/arithmetic.py @@ -0,0 +1,293 @@ +import numpy as np + +c16 = np.complex128(1) +f8 = np.float64(1) +i8 = np.int64(1) +u8 = np.uint64(1) + +c8 = np.complex64(1) +f4 = np.float32(1) +i4 = np.int32(1) +u4 = np.uint32(1) + +dt = np.datetime64(1, "D") +td = np.timedelta64(1, "D") + +b_ = np.bool_(1) + +b = bool(1) +c = complex(1) +f = float(1) +i = int(1) + +AR = np.ones(1, dtype=np.float64) +AR.setflags(write=False) + +# unary ops + +-c16 +-c8 +-f8 +-f4 +-i8 +-i4 +-u8 +-u4 +-td +-AR + ++c16 ++c8 ++f8 ++f4 ++i8 ++i4 ++u8 ++u4 ++td ++AR + +abs(c16) +abs(c8) +abs(f8) +abs(f4) +abs(i8) +abs(i4) +abs(u8) +abs(u4) +abs(td) +abs(b_) +abs(AR) + +# Time structures + +dt + td +dt + i +dt + i4 +dt + i8 +dt - dt +dt - i +dt - i4 +dt - i8 + +td + td +td + i +td + i4 +td + i8 +td - td +td - i +td - i4 +td - i8 +td / f +td / f4 +td / f8 +td / td +td // td +td % td + + +# boolean + +b_ / b +b_ / b_ +b_ / i +b_ / i8 +b_ / i4 +b_ / u8 +b_ / u4 +b_ / f +b_ / f8 +b_ / f4 +b_ / c +b_ / c16 +b_ / c8 + +b / b_ +b_ / b_ +i / b_ +i8 / b_ +i4 / b_ +u8 / b_ +u4 / b_ +f / b_ +f8 / b_ +f4 / b_ +c / b_ +c16 / b_ +c8 / b_ + +# Complex + +c16 + c16 +c16 + f8 +c16 + i8 +c16 + c8 +c16 + f4 +c16 + i4 +c16 + b_ +c16 + b +c16 + c +c16 + f +c16 + i +c16 + AR + +c16 + c16 +f8 + c16 +i8 + c16 +c8 + c16 +f4 + c16 +i4 + c16 +b_ + c16 +b + c16 +c + c16 +f + c16 +i + c16 +AR + c16 + +c8 + c16 +c8 + f8 +c8 + i8 +c8 + c8 +c8 + f4 +c8 + i4 +c8 + b_ +c8 + b +c8 + c +c8 + f +c8 + i +c8 + AR + +c16 + c8 +f8 + c8 +i8 + c8 +c8 + c8 +f4 + c8 +i4 + c8 +b_ + c8 +b + c8 +c + c8 +f + c8 +i + c8 +AR + c8 + +# Float + +f8 + f8 +f8 + i8 +f8 + f4 +f8 + i4 +f8 + b_ +f8 + b +f8 + c +f8 + f +f8 + i +f8 + AR + +f8 + f8 +i8 + f8 +f4 + f8 +i4 + f8 +b_ + f8 +b + f8 +c + f8 +f + f8 +i + f8 +AR + f8 + +f4 + f8 +f4 + i8 +f4 + f4 +f4 + i4 +f4 + b_ +f4 + b +f4 + c +f4 + f +f4 + i +f4 + AR + +f8 + f4 +i8 + f4 +f4 + f4 +i4 + f4 +b_ + f4 +b + f4 +c + f4 +f + f4 +i + f4 +AR + f4 + +# Int + +i8 + i8 +i8 + u8 +i8 + i4 +i8 + u4 +i8 + b_ +i8 + b +i8 + c +i8 + f +i8 + i +i8 + AR + +u8 + u8 +u8 + i4 +u8 + u4 +u8 + b_ +u8 + b +u8 + c +u8 + f +u8 + i +u8 + AR + +i8 + i8 +u8 + i8 +i4 + i8 +u4 + i8 +b_ + i8 +b + i8 +c + i8 +f + i8 +i + i8 +AR + i8 + +u8 + u8 +i4 + u8 +u4 + u8 +b_ + u8 +b + u8 +c + u8 +f + u8 +i + u8 +AR + u8 + +i4 + i8 +i4 + i4 +i4 + i +i4 + b_ +i4 + b +i4 + AR + +u4 + i8 +u4 + i4 +u4 + u8 +u4 + u4 +u4 + i +u4 + b_ +u4 + b +u4 + AR + +i8 + i4 +i4 + i4 +i + i4 +b_ + i4 +b + i4 +AR + i4 + +i8 + u4 +i4 + u4 +u8 + u4 +u4 + u4 +b_ + u4 +b + u4 +i + u4 +AR + u4 diff --git a/numpy/typing/tests/data/pass/array_constructors.py b/numpy/typing/tests/data/pass/array_constructors.py new file mode 100644 index 000000000000..63208f139c39 --- /dev/null +++ b/numpy/typing/tests/data/pass/array_constructors.py @@ -0,0 +1,128 @@ +from typing import List, Any +import numpy as np + +class Index: + def __index__(self) -> int: + return 0 + +class SubClass(np.ndarray): ... + +i8 = np.int64(1) + +A = np.array([1]) +B = A.view(SubClass).copy() +B_stack = np.array([[1], [1]]).view(SubClass) +C = [1] + +def func(i: int, j: int, **kwargs: Any) -> SubClass: + return B + +np.array(1, dtype=float) +np.array(1, copy=False) +np.array(1, order='F') +np.array(1, order=None) +np.array(1, subok=True) +np.array(1, ndmin=3) +np.array(1, str, copy=True, order='C', subok=False, ndmin=2) + +np.asarray(A) +np.asarray(B) +np.asarray(C) + +np.asanyarray(A) +np.asanyarray(B) +np.asanyarray(B, dtype=int) +np.asanyarray(C) + +np.ascontiguousarray(A) +np.ascontiguousarray(B) +np.ascontiguousarray(C) + +np.asfortranarray(A) +np.asfortranarray(B) +np.asfortranarray(C) + +np.require(A) +np.require(B) +np.require(B, dtype=int) +np.require(B, requirements=None) +np.require(B, requirements="E") +np.require(B, requirements=["ENSUREARRAY"]) +np.require(B, requirements={"F", "E"}) +np.require(B, requirements=["C", "OWNDATA"]) +np.require(B, requirements="W") +np.require(B, requirements="A") +np.require(C) + +np.linspace(0, 2) +np.linspace(0.5, [0, 1, 2]) +np.linspace([0, 1, 2], 3) +np.linspace(0j, 2) +np.linspace(0, 2, num=10) +np.linspace(0, 2, endpoint=True) +np.linspace(0, 2, retstep=True) +np.linspace(0j, 2j, retstep=True) +np.linspace(0, 2, dtype=bool) +np.linspace([0, 1], [2, 3], axis=Index()) + +np.logspace(0, 2, base=2) +np.logspace(0, 2, base=2) +np.logspace(0, 2, base=[1j, 2j], num=2) + +np.geomspace(1, 2) + +np.zeros_like(A) +np.zeros_like(C) +np.zeros_like(B) +np.zeros_like(B, dtype=np.int64) + +np.ones_like(A) +np.ones_like(C) +np.ones_like(B) +np.ones_like(B, dtype=np.int64) + +np.empty_like(A) +np.empty_like(C) +np.empty_like(B) +np.empty_like(B, dtype=np.int64) + +np.full_like(A, i8) +np.full_like(C, i8) +np.full_like(B, i8) +np.full_like(B, i8, dtype=np.int64) + +np.ones(1) +np.ones([1, 1, 1]) + +np.full(1, i8) +np.full([1, 1, 1], i8) + +np.indices([1, 2, 3]) +np.indices([1, 2, 3], sparse=True) + +np.fromfunction(func, (3, 5)) + +np.identity(10) + +np.atleast_1d(C) +np.atleast_1d(A) +np.atleast_1d(C, C) +np.atleast_1d(C, A) +np.atleast_1d(A, A) + +np.atleast_2d(C) + +np.atleast_3d(C) + +np.vstack([C, C]) +np.vstack([C, A]) +np.vstack([A, A]) + +np.hstack([C, C]) + +np.stack([C, C]) +np.stack([C, C], axis=0) +np.stack([C, C], out=B_stack) + +np.block([[C, C], [C, C]]) +np.block(A) diff --git a/numpy/typing/tests/data/pass/array_like.py b/numpy/typing/tests/data/pass/array_like.py index 6b823ca7e1af..f85724267448 100644 --- a/numpy/typing/tests/data/pass/array_like.py +++ b/numpy/typing/tests/data/pass/array_like.py @@ -1,7 +1,7 @@ from typing import Any, List, Optional import numpy as np -from numpy.typing import ArrayLike, DtypeLike, _SupportsArray +from numpy.typing import ArrayLike, DTypeLike, _SupportsArray x1: ArrayLike = True x2: ArrayLike = 5 @@ -18,7 +18,7 @@ class A: - def __array__(self, dtype: DtypeLike = None) -> np.ndarray: + def __array__(self, dtype: DTypeLike = None) -> np.ndarray: return np.array([1, 2, 3]) diff --git a/numpy/typing/tests/data/pass/bitwise_ops.py b/numpy/typing/tests/data/pass/bitwise_ops.py new file mode 100644 index 000000000000..67449e2c21d8 --- /dev/null +++ b/numpy/typing/tests/data/pass/bitwise_ops.py @@ -0,0 +1,131 @@ +import numpy as np + +i8 = np.int64(1) +u8 = np.uint64(1) + +i4 = np.int32(1) +u4 = np.uint32(1) + +b_ = np.bool_(1) + +b = bool(1) +i = int(1) + +AR = np.array([0, 1, 2], dtype=np.int32) +AR.setflags(write=False) + + +i8 << i8 +i8 >> i8 +i8 | i8 +i8 ^ i8 +i8 & i8 + +i8 << AR +i8 >> AR +i8 | AR +i8 ^ AR +i8 & AR + +i4 << i4 +i4 >> i4 +i4 | i4 +i4 ^ i4 +i4 & i4 + +i8 << i4 +i8 >> i4 +i8 | i4 +i8 ^ i4 +i8 & i4 + +i8 << i +i8 >> i +i8 | i +i8 ^ i +i8 & i + +i8 << b_ +i8 >> b_ +i8 | b_ +i8 ^ b_ +i8 & b_ + +i8 << b +i8 >> b +i8 | b +i8 ^ b +i8 & b + +u8 << u8 +u8 >> u8 +u8 | u8 +u8 ^ u8 +u8 & u8 + +u8 << AR +u8 >> AR +u8 | AR +u8 ^ AR +u8 & AR + +u4 << u4 +u4 >> u4 +u4 | u4 +u4 ^ u4 +u4 & u4 + +u4 << i4 +u4 >> i4 +u4 | i4 +u4 ^ i4 +u4 & i4 + +u4 << i +u4 >> i +u4 | i +u4 ^ i +u4 & i + +u8 << b_ +u8 >> b_ +u8 | b_ +u8 ^ b_ +u8 & b_ + +u8 << b +u8 >> b +u8 | b +u8 ^ b +u8 & b + +b_ << b_ +b_ >> b_ +b_ | b_ +b_ ^ b_ +b_ & b_ + +b_ << AR +b_ >> AR +b_ | AR +b_ ^ AR +b_ & AR + +b_ << b +b_ >> b +b_ | b +b_ ^ b +b_ & b + +b_ << i +b_ >> i +b_ | i +b_ ^ i +b_ & i + +~i8 +~i4 +~u8 +~u4 +~b_ +~AR diff --git a/numpy/typing/tests/data/pass/linspace.py b/numpy/typing/tests/data/pass/linspace.py deleted file mode 100644 index 8c6d0d56b93b..000000000000 --- a/numpy/typing/tests/data/pass/linspace.py +++ /dev/null @@ -1,22 +0,0 @@ -import numpy as np - -class Index: - def __index__(self) -> int: - return 0 - -np.linspace(0, 2) -np.linspace(0.5, [0, 1, 2]) -np.linspace([0, 1, 2], 3) -np.linspace(0j, 2) -np.linspace(0, 2, num=10) -np.linspace(0, 2, endpoint=True) -np.linspace(0, 2, retstep=True) -np.linspace(0j, 2j, retstep=True) -np.linspace(0, 2, dtype=bool) -np.linspace([0, 1], [2, 3], axis=Index()) - -np.logspace(0, 2, base=2) -np.logspace(0, 2, base=2) -np.logspace(0, 2, base=[1j, 2j], num=2) - -np.geomspace(1, 2) diff --git a/numpy/typing/tests/data/pass/literal.py b/numpy/typing/tests/data/pass/literal.py index 321ce3c2bc0c..8eaeb6afb2ad 100644 --- a/numpy/typing/tests/data/pass/literal.py +++ b/numpy/typing/tests/data/pass/literal.py @@ -31,6 +31,8 @@ (KACF, partial(np.add, 1, 1)), # i.e. np.ufunc.__call__ (ACF, partial(np.reshape, AR, 1)), (KACF, partial(np.ravel, AR)), + (KACF, partial(np.asarray, 1)), + (KACF, partial(np.asanyarray, 1)), ] for order_set, func in order_list: diff --git a/numpy/typing/tests/data/pass/mod.py b/numpy/typing/tests/data/pass/mod.py new file mode 100644 index 000000000000..b5b9afb2a544 --- /dev/null +++ b/numpy/typing/tests/data/pass/mod.py @@ -0,0 +1,149 @@ +import numpy as np + +f8 = np.float64(1) +i8 = np.int64(1) +u8 = np.uint64(1) + +f4 = np.float32(1) +i4 = np.int32(1) +u4 = np.uint32(1) + +td = np.timedelta64(1, "D") +b_ = np.bool_(1) + +b = bool(1) +f = float(1) +i = int(1) + +AR = np.array([1], dtype=np.bool_) +AR.setflags(write=False) + +AR2 = np.array([1], dtype=np.timedelta64) +AR2.setflags(write=False) + +# Time structures + +td % td +td % AR2 +AR2 % td + +divmod(td, td) +divmod(td, AR2) +divmod(AR2, td) + +# Bool + +b_ % b +b_ % i +b_ % f +b_ % b_ +b_ % i8 +b_ % u8 +b_ % f8 +b_ % AR + +divmod(b_, b) +divmod(b_, i) +divmod(b_, f) +divmod(b_, b_) +divmod(b_, i8) +divmod(b_, u8) +divmod(b_, f8) +divmod(b_, AR) + +b % b_ +i % b_ +f % b_ +b_ % b_ +i8 % b_ +u8 % b_ +f8 % b_ +AR % b_ + +divmod(b, b_) +divmod(i, b_) +divmod(f, b_) +divmod(b_, b_) +divmod(i8, b_) +divmod(u8, b_) +divmod(f8, b_) +divmod(AR, b_) + +# int + +i8 % b +i8 % i +i8 % f +i8 % i8 +i8 % f8 +i4 % i8 +i4 % f8 +i4 % i4 +i4 % f4 +i8 % AR + +divmod(i8, b) +divmod(i8, i) +divmod(i8, f) +divmod(i8, i8) +divmod(i8, f8) +divmod(i8, i4) +divmod(i8, f4) +divmod(i4, i4) +divmod(i4, f4) +divmod(i8, AR) + +b % i8 +i % i8 +f % i8 +i8 % i8 +f8 % i8 +i8 % i4 +f8 % i4 +i4 % i4 +f4 % i4 +AR % i8 + +divmod(b, i8) +divmod(i, i8) +divmod(f, i8) +divmod(i8, i8) +divmod(f8, i8) +divmod(i4, i8) +divmod(f4, i8) +divmod(i4, i4) +divmod(f4, i4) +divmod(AR, i8) + +# float + +f8 % b +f8 % i +f8 % f +i8 % f4 +f4 % f4 +f8 % AR + +divmod(f8, b) +divmod(f8, i) +divmod(f8, f) +divmod(f8, f8) +divmod(f8, f4) +divmod(f4, f4) +divmod(f8, AR) + +b % f8 +i % f8 +f % f8 +f8 % f8 +f8 % f8 +f4 % f4 +AR % f8 + +divmod(b, f8) +divmod(i, f8) +divmod(f, f8) +divmod(f8, f8) +divmod(f4, f8) +divmod(f4, f4) +divmod(AR, f8) diff --git a/numpy/typing/tests/data/pass/ndarray_misc.py b/numpy/typing/tests/data/pass/ndarray_misc.py new file mode 100644 index 000000000000..6c6f5d50b986 --- /dev/null +++ b/numpy/typing/tests/data/pass/ndarray_misc.py @@ -0,0 +1,159 @@ +""" +Tests for miscellaneous (non-magic) ``np.ndarray``/``np.generic`` methods. + +More extensive tests are performed for the methods' +function-based counterpart in `../from_numeric.py`. + +""" + +from typing import cast +import numpy as np + +class SubClass(np.ndarray): ... + +i4 = np.int32(1) +A = np.array([[1]], dtype=np.int32) +B0 = np.empty((), dtype=np.int32).view(SubClass) +B1 = np.empty((1,), dtype=np.int32).view(SubClass) +B2 = np.empty((1, 1), dtype=np.int32).view(SubClass) +C = np.array([0, 1, 2], dtype=np.int32) +D = np.empty(3).view(SubClass) + +i4.all() +A.all() +A.all(axis=0) +A.all(keepdims=True) +A.all(out=B0) + +i4.any() +A.any() +A.any(axis=0) +A.any(keepdims=True) +A.any(out=B0) + +i4.argmax() +A.argmax() +A.argmax(axis=0) +A.argmax(out=B0) + +i4.argmin() +A.argmin() +A.argmin(axis=0) +A.argmin(out=B0) + +i4.argsort() +A.argsort() + +i4.choose([()]) +_choices = np.array([[0, 1, 2], [3, 4, 5], [6, 7, 8]], dtype=np.int32) +C.choose(_choices) +C.choose(_choices, out=D) + +i4.clip(1) +A.clip(1) +A.clip(None, 1) +A.clip(1, out=B2) +A.clip(None, 1, out=B2) + +i4.compress([1]) +A.compress([1]) +A.compress([1], out=B1) + +i4.conj() +A.conj() +B0.conj() + +i4.conjugate() +A.conjugate() +B0.conjugate() + +i4.cumprod() +A.cumprod() +A.cumprod(out=B1) + +i4.cumsum() +A.cumsum() +A.cumsum(out=B1) + +i4.max() +A.max() +A.max(axis=0) +A.max(keepdims=True) +A.max(out=B0) + +i4.mean() +A.mean() +A.mean(axis=0) +A.mean(keepdims=True) +A.mean(out=B0) + +i4.min() +A.min() +A.min(axis=0) +A.min(keepdims=True) +A.min(out=B0) + +i4.newbyteorder() +A.newbyteorder() +B0.newbyteorder('|') + +i4.prod() +A.prod() +A.prod(axis=0) +A.prod(keepdims=True) +A.prod(out=B0) + +i4.ptp() +A.ptp() +A.ptp(axis=0) +A.ptp(keepdims=True) +A.astype(int).ptp(out=B0) + +i4.round() +A.round() +A.round(out=B2) + +i4.repeat(1) +A.repeat(1) +B0.repeat(1) + +i4.std() +A.std() +A.std(axis=0) +A.std(keepdims=True) +A.std(out=B0.astype(np.float64)) + +i4.sum() +A.sum() +A.sum(axis=0) +A.sum(keepdims=True) +A.sum(out=B0) + +i4.take(0) +A.take(0) +A.take([0]) +A.take(0, out=B0) +A.take([0], out=B1) + +i4.var() +A.var() +A.var(axis=0) +A.var(keepdims=True) +A.var(out=B0) + +A.argpartition([0]) + +A.diagonal() + +A.dot(1) +A.dot(1, out=B0) + +A.nonzero() + +C.searchsorted(1) + +A.trace() +A.trace(out=B0) + +void = cast(np.void, np.array(1, dtype=[("f", np.float64)]).take(0)) +void.setfield(10, np.float64) diff --git a/numpy/typing/tests/data/pass/numeric.py b/numpy/typing/tests/data/pass/numeric.py new file mode 100644 index 000000000000..34fef7270443 --- /dev/null +++ b/numpy/typing/tests/data/pass/numeric.py @@ -0,0 +1,89 @@ +""" +Tests for :mod:`numpy.core.numeric`. + +Does not include tests which fall under ``array_constructors``. + +""" + +from typing import List +import numpy as np + +class SubClass(np.ndarray): + ... + +i8 = np.int64(1) + +A = np.arange(27).reshape(3, 3, 3) +B: List[List[List[int]]] = A.tolist() +C = np.empty((27, 27)).view(SubClass) + +np.count_nonzero(i8) +np.count_nonzero(A) +np.count_nonzero(B) +np.count_nonzero(A, keepdims=True) +np.count_nonzero(A, axis=0) + +np.isfortran(i8) +np.isfortran(A) + +np.argwhere(i8) +np.argwhere(A) + +np.flatnonzero(i8) +np.flatnonzero(A) + +np.correlate(B[0][0], A.ravel(), mode="valid") +np.correlate(A.ravel(), A.ravel(), mode="same") + +np.convolve(B[0][0], A.ravel(), mode="valid") +np.convolve(A.ravel(), A.ravel(), mode="same") + +np.outer(i8, A) +np.outer(B, A) +np.outer(A, A) +np.outer(A, A, out=C) + +np.tensordot(B, A) +np.tensordot(A, A) +np.tensordot(A, A, axes=0) +np.tensordot(A, A, axes=(0, 1)) + +np.isscalar(i8) +np.isscalar(A) +np.isscalar(B) + +np.roll(A, 1) +np.roll(A, (1, 2)) +np.roll(B, 1) + +np.rollaxis(A, 0, 1) + +np.moveaxis(A, 0, 1) +np.moveaxis(A, (0, 1), (1, 2)) + +np.cross(B, A) +np.cross(A, A) + +np.indices([0, 1, 2]) +np.indices([0, 1, 2], sparse=False) +np.indices([0, 1, 2], sparse=True) + +np.binary_repr(1) + +np.base_repr(1) + +np.allclose(i8, A) +np.allclose(B, A) +np.allclose(A, A) + +np.isclose(i8, A) +np.isclose(B, A) +np.isclose(A, A) + +np.array_equal(i8, A) +np.array_equal(B, A) +np.array_equal(A, A) + +np.array_equiv(i8, A) +np.array_equiv(B, A) +np.array_equiv(A, A) diff --git a/numpy/typing/tests/data/pass/scalars.py b/numpy/typing/tests/data/pass/scalars.py index c02e1ed36f28..04e97498304d 100644 --- a/numpy/typing/tests/data/pass/scalars.py +++ b/numpy/typing/tests/data/pass/scalars.py @@ -1,6 +1,7 @@ import sys import datetime as dt +import pytest import numpy as np @@ -62,16 +63,6 @@ def __float__(self) -> float: np.str_(b"hello", 'utf-8') np.str_(b"hello", encoding='utf-8') -# Protocols -float(np.int8(4)) -int(np.int16(5)) -np.int8(np.float32(6)) - -# TODO(alan): test after https://github.com/python/typeshed/pull/2004 -# complex(np.int32(8)) - -abs(np.int8(4)) - # Array-ish semantics np.int8().real np.int16().imag @@ -108,22 +99,51 @@ def __float__(self) -> float: np.timedelta64(None) np.timedelta64(None, "D") -dt_64 = np.datetime64(0, "D") -td_64 = np.timedelta64(1, "h") - -dt_64 + td_64 -dt_64 - dt_64 -dt_64 - td_64 - -td_64 + td_64 -td_64 - td_64 -td_64 / 1.0 -td_64 / td_64 -td_64 % td_64 - np.void(1) np.void(np.int64(1)) np.void(True) np.void(np.bool_(True)) np.void(b"test") np.void(np.bytes_("test")) + +# Protocols +i8 = np.int64() +u8 = np.uint64() +f8 = np.float64() +c16 = np.complex128() +b_ = np.bool_() +td = np.timedelta64() +U = np.str_("1") +S = np.bytes_("1") +AR = np.array(1, dtype=np.float64) + +int(i8) +int(u8) +int(f8) +int(b_) +int(td) +int(U) +int(S) +int(AR) +with pytest.warns(np.ComplexWarning): + int(c16) + +float(i8) +float(u8) +float(f8) +float(b_) +float(td) +float(U) +float(S) +float(AR) +with pytest.warns(np.ComplexWarning): + float(c16) + +complex(i8) +complex(u8) +complex(f8) +complex(c16) +complex(b_) +complex(td) +complex(U) +complex(AR) diff --git a/numpy/typing/tests/data/pass/simple.py b/numpy/typing/tests/data/pass/simple.py index 52705055702c..243caf229f13 100644 --- a/numpy/typing/tests/data/pass/simple.py +++ b/numpy/typing/tests/data/pass/simple.py @@ -17,23 +17,6 @@ def ndarray_func(x): array == 1 array.dtype == float -# Array creation routines checks -np.array(1, dtype=float) -np.array(1, copy=False) -np.array(1, order='F') -np.array(1, order=None) -np.array(1, subok=True) -np.array(1, ndmin=3) -np.array(1, str, copy=True, order='C', subok=False, ndmin=2) - -ndarray_func(np.zeros([1, 2])) -ndarray_func(np.ones([1, 2])) -ndarray_func(np.empty([1, 2])) - -ndarray_func(np.zeros_like(array)) -ndarray_func(np.ones_like(array)) -ndarray_func(np.empty_like(array)) - # Dtype construction np.dtype(float) np.dtype(np.float64) diff --git a/numpy/typing/tests/data/pass/ufuncs.py b/numpy/typing/tests/data/pass/ufuncs.py index 82172952a61d..ad4d483d4a70 100644 --- a/numpy/typing/tests/data/pass/ufuncs.py +++ b/numpy/typing/tests/data/pass/ufuncs.py @@ -6,7 +6,10 @@ np.matmul(np.ones((2, 2, 2)), np.ones((2, 2, 2)), axes=[(0, 1), (0, 1), (0, 1)]) np.sin(1, signature="D") np.sin(1, extobj=[16, 1, lambda: None]) -np.sin(1) + np.sin(1) +# NOTE: `np.generic` subclasses are not guaranteed to support addition; +# re-enable this we can infer the exact return type of `np.sin(...)`. +# +# np.sin(1) + np.sin(1) np.sin.types[0] np.sin.__name__ diff --git a/numpy/typing/tests/data/reveal/arithmetic.py b/numpy/typing/tests/data/reveal/arithmetic.py new file mode 100644 index 000000000000..8a133c136339 --- /dev/null +++ b/numpy/typing/tests/data/reveal/arithmetic.py @@ -0,0 +1,291 @@ +import numpy as np + +c16 = np.complex128() +f8 = np.float64() +i8 = np.int64() +u8 = np.uint64() + +c8 = np.complex64() +f4 = np.float32() +i4 = np.int32() +u4 = np.uint32() + +dt = np.datetime64(0, "D") +td = np.timedelta64(0, "D") + +b_ = np.bool_() + +b = bool() +c = complex() +f = float() +i = int() + +AR = np.array([0], dtype=np.float64) +AR.setflags(write=False) + +# unary ops + +reveal_type(-c16) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] +reveal_type(-c8) # E: numpy.complexfloating[numpy.typing._32Bit, numpy.typing._32Bit] +reveal_type(-f8) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(-f4) # E: numpy.floating[numpy.typing._32Bit] +reveal_type(-i8) # E: numpy.signedinteger[numpy.typing._64Bit] +reveal_type(-i4) # E: numpy.signedinteger[numpy.typing._32Bit] +reveal_type(-u8) # E: numpy.unsignedinteger[numpy.typing._64Bit] +reveal_type(-u4) # E: numpy.unsignedinteger[numpy.typing._32Bit] +reveal_type(-td) # E: numpy.timedelta64 +reveal_type(-AR) # E: Union[numpy.ndarray*, numpy.generic] + +reveal_type(+c16) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] +reveal_type(+c8) # E: numpy.complexfloating[numpy.typing._32Bit, numpy.typing._32Bit] +reveal_type(+f8) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(+f4) # E: numpy.floating[numpy.typing._32Bit] +reveal_type(+i8) # E: numpy.signedinteger[numpy.typing._64Bit] +reveal_type(+i4) # E: numpy.signedinteger[numpy.typing._32Bit] +reveal_type(+u8) # E: numpy.unsignedinteger[numpy.typing._64Bit] +reveal_type(+u4) # E: numpy.unsignedinteger[numpy.typing._32Bit] +reveal_type(+td) # E: numpy.timedelta64 +reveal_type(+AR) # E: Union[numpy.ndarray*, numpy.generic] + +reveal_type(abs(c16)) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(abs(c8)) # E: numpy.floating[numpy.typing._32Bit] +reveal_type(abs(f8)) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(abs(f4)) # E: numpy.floating[numpy.typing._32Bit] +reveal_type(abs(i8)) # E: numpy.signedinteger[numpy.typing._64Bit] +reveal_type(abs(i4)) # E: numpy.signedinteger[numpy.typing._32Bit] +reveal_type(abs(u8)) # E: numpy.unsignedinteger[numpy.typing._64Bit] +reveal_type(abs(u4)) # E: numpy.unsignedinteger[numpy.typing._32Bit] +reveal_type(abs(td)) # E: numpy.timedelta64 +reveal_type(abs(b_)) # E: numpy.bool_ +reveal_type(abs(AR)) # E: Union[numpy.ndarray*, numpy.generic] + +# Time structures + +reveal_type(dt + td) # E: numpy.datetime64 +reveal_type(dt + i) # E: numpy.datetime64 +reveal_type(dt + i4) # E: numpy.datetime64 +reveal_type(dt + i8) # E: numpy.datetime64 +reveal_type(dt - dt) # E: numpy.timedelta64 +reveal_type(dt - i) # E: numpy.datetime64 +reveal_type(dt - i4) # E: numpy.datetime64 +reveal_type(dt - i8) # E: numpy.datetime64 + +reveal_type(td + td) # E: numpy.timedelta64 +reveal_type(td + i) # E: numpy.timedelta64 +reveal_type(td + i4) # E: numpy.timedelta64 +reveal_type(td + i8) # E: numpy.timedelta64 +reveal_type(td - td) # E: numpy.timedelta64 +reveal_type(td - i) # E: numpy.timedelta64 +reveal_type(td - i4) # E: numpy.timedelta64 +reveal_type(td - i8) # E: numpy.timedelta64 +reveal_type(td / f) # E: numpy.timedelta64 +reveal_type(td / f4) # E: numpy.timedelta64 +reveal_type(td / f8) # E: numpy.timedelta64 +reveal_type(td / td) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(td // td) # E: numpy.signedinteger[numpy.typing._64Bit] + +# boolean + +reveal_type(b_ / b) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(b_ / b_) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(b_ / i) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(b_ / i8) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(b_ / i4) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(b_ / u8) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(b_ / u4) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(b_ / f) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(b_ / f8) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(b_ / f4) # E: numpy.floating[numpy.typing._32Bit] +reveal_type(b_ / c) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] +reveal_type(b_ / c16) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] +reveal_type(b_ / c8) # E: numpy.complexfloating[numpy.typing._32Bit, numpy.typing._32Bit] + +reveal_type(b / b_) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(b_ / b_) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(i / b_) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(i8 / b_) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(i4 / b_) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(u8 / b_) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(u4 / b_) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(f / b_) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(f8 / b_) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(f4 / b_) # E: numpy.floating[numpy.typing._32Bit] +reveal_type(c / b_) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] +reveal_type(c16 / b_) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] +reveal_type(c8 / b_) # E: numpy.complexfloating[numpy.typing._32Bit, numpy.typing._32Bit] + +# Complex + +reveal_type(c16 + c16) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] +reveal_type(c16 + f8) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] +reveal_type(c16 + i8) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] +reveal_type(c16 + c8) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] +reveal_type(c16 + f4) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] +reveal_type(c16 + i4) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] +reveal_type(c16 + b_) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] +reveal_type(c16 + b) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] +reveal_type(c16 + c) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] +reveal_type(c16 + f) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] +reveal_type(c16 + i) # E: numpy.complexfloating[Any, Any] +reveal_type(c16 + AR) # E: Union[numpy.ndarray, numpy.generic] + +reveal_type(c16 + c16) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] +reveal_type(f8 + c16) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] +reveal_type(i8 + c16) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] +reveal_type(c8 + c16) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] +reveal_type(f4 + c16) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] +reveal_type(i4 + c16) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] +reveal_type(b_ + c16) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] +reveal_type(b + c16) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] +reveal_type(c + c16) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] +reveal_type(f + c16) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] +reveal_type(i + c16) # E: numpy.complexfloating[Any, Any] +reveal_type(AR + c16) # E: Union[numpy.ndarray, numpy.generic] + +reveal_type(c8 + c16) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] +reveal_type(c8 + f8) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] +reveal_type(c8 + i8) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] +reveal_type(c8 + c8) # E: numpy.complexfloating[numpy.typing._32Bit, numpy.typing._32Bit] +reveal_type(c8 + f4) # E: numpy.complexfloating[numpy.typing._32Bit, numpy.typing._32Bit] +reveal_type(c8 + i4) # E: numpy.complexfloating[numpy.typing._32Bit, numpy.typing._32Bit] +reveal_type(c8 + b_) # E: numpy.complexfloating[numpy.typing._32Bit, numpy.typing._32Bit] +reveal_type(c8 + b) # E: numpy.complexfloating[numpy.typing._32Bit, numpy.typing._32Bit] +reveal_type(c8 + c) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] +reveal_type(c8 + f) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] +reveal_type(c8 + i) # E: numpy.complexfloating[Any, Any] +reveal_type(c8 + AR) # E: Union[numpy.ndarray, numpy.generic] + +reveal_type(c16 + c8) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] +reveal_type(f8 + c8) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] +reveal_type(i8 + c8) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] +reveal_type(c8 + c8) # E: numpy.complexfloating[numpy.typing._32Bit, numpy.typing._32Bit] +reveal_type(f4 + c8) # E: numpy.complexfloating[numpy.typing._32Bit, numpy.typing._32Bit] +reveal_type(i4 + c8) # E: numpy.complexfloating[numpy.typing._32Bit, numpy.typing._32Bit] +reveal_type(b_ + c8) # E: numpy.complexfloating[numpy.typing._32Bit, numpy.typing._32Bit] +reveal_type(b + c8) # E: numpy.complexfloating[numpy.typing._32Bit, numpy.typing._32Bit] +reveal_type(c + c8) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] +reveal_type(f + c8) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] +reveal_type(i + c8) # E: numpy.complexfloating[Any, Any] +reveal_type(AR + c8) # E: Union[numpy.ndarray, numpy.generic] + +# Float + +reveal_type(f8 + f8) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(f8 + i8) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(f8 + f4) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(f8 + i4) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(f8 + b_) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(f8 + b) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(f8 + c) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] +reveal_type(f8 + f) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(f8 + i) # E: numpy.floating[Any] +reveal_type(f8 + AR) # E: Union[numpy.ndarray, numpy.generic] + +reveal_type(f8 + f8) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(i8 + f8) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(f4 + f8) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(i4 + f8) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(b_ + f8) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(b + f8) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(c + f8) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] +reveal_type(f + f8) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(i + f8) # E: numpy.floating[Any] +reveal_type(AR + f8) # E: Union[numpy.ndarray, numpy.generic] + +reveal_type(f4 + f8) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(f4 + i8) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(f4 + f4) # E: numpy.floating[numpy.typing._32Bit] +reveal_type(f4 + i4) # E: numpy.floating[numpy.typing._32Bit] +reveal_type(f4 + b_) # E: numpy.floating[numpy.typing._32Bit] +reveal_type(f4 + b) # E: numpy.floating[numpy.typing._32Bit] +reveal_type(f4 + c) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] +reveal_type(f4 + f) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(f4 + i) # E: numpy.floating[Any] +reveal_type(f4 + AR) # E: Union[numpy.ndarray, numpy.generic] + +reveal_type(f8 + f4) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(i8 + f4) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(f4 + f4) # E: umpy.floating[numpy.typing._32Bit] +reveal_type(i4 + f4) # E: umpy.floating[numpy.typing._32Bit] +reveal_type(b_ + f4) # E: umpy.floating[numpy.typing._32Bit] +reveal_type(b + f4) # E: umpy.floating[numpy.typing._32Bit] +reveal_type(c + f4) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] +reveal_type(f + f4) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(i + f4) # E: numpy.floating[Any] +reveal_type(AR + f4) # E: Union[numpy.ndarray, numpy.generic] + +# Int + +reveal_type(i8 + i8) # E: numpy.signedinteger[numpy.typing._64Bit] +reveal_type(i8 + u8) # E: Union[numpy.signedinteger[Any], numpy.floating[numpy.typing._64Bit]] +reveal_type(i8 + i4) # E: numpy.signedinteger[numpy.typing._64Bit] +reveal_type(i8 + u4) # E: Union[numpy.signedinteger[Any], numpy.floating[numpy.typing._64Bit]] +reveal_type(i8 + b_) # E: numpy.signedinteger[numpy.typing._64Bit] +reveal_type(i8 + b) # E: numpy.signedinteger[numpy.typing._64Bit] +reveal_type(i8 + c) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] +reveal_type(i8 + f) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(i8 + i) # E: numpy.signedinteger[Any] +reveal_type(i8 + AR) # E: Union[numpy.ndarray, numpy.generic] + +reveal_type(u8 + u8) # E: numpy.unsignedinteger[numpy.typing._64Bit] +reveal_type(u8 + i4) # E: Union[numpy.signedinteger[Any], numpy.floating[numpy.typing._64Bit]] +reveal_type(u8 + u4) # E: numpy.unsignedinteger[numpy.typing._64Bit] +reveal_type(u8 + b_) # E: numpy.unsignedinteger[numpy.typing._64Bit] +reveal_type(u8 + b) # E: numpy.unsignedinteger[numpy.typing._64Bit] +reveal_type(u8 + c) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] +reveal_type(u8 + f) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(u8 + i) # E: Union[numpy.signedinteger[Any], numpy.floating[numpy.typing._64Bit]] +reveal_type(u8 + AR) # E: Union[numpy.ndarray, numpy.generic] + +reveal_type(i8 + i8) # E: numpy.signedinteger[numpy.typing._64Bit] +reveal_type(u8 + i8) # E: Union[numpy.signedinteger[Any], numpy.floating[numpy.typing._64Bit]] +reveal_type(i4 + i8) # E: numpy.signedinteger[numpy.typing._64Bit] +reveal_type(u4 + i8) # E: Union[numpy.signedinteger[Any], numpy.floating[numpy.typing._64Bit]] +reveal_type(b_ + i8) # E: numpy.signedinteger[numpy.typing._64Bit] +reveal_type(b + i8) # E: numpy.signedinteger[numpy.typing._64Bit] +reveal_type(c + i8) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] +reveal_type(f + i8) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(i + i8) # E: numpy.signedinteger[Any] +reveal_type(AR + i8) # E: Union[numpy.ndarray, numpy.generic] + +reveal_type(u8 + u8) # E: numpy.unsignedinteger[numpy.typing._64Bit] +reveal_type(i4 + u8) # E: Union[numpy.signedinteger[Any], numpy.floating[numpy.typing._64Bit]] +reveal_type(u4 + u8) # E: numpy.unsignedinteger[numpy.typing._64Bit] +reveal_type(b_ + u8) # E: numpy.unsignedinteger[numpy.typing._64Bit] +reveal_type(b + u8) # E: numpy.unsignedinteger[numpy.typing._64Bit] +reveal_type(c + u8) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] +reveal_type(f + u8) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(i + u8) # E: Union[numpy.signedinteger[Any], numpy.floating[numpy.typing._64Bit]] +reveal_type(AR + u8) # E: Union[numpy.ndarray, numpy.generic] + +reveal_type(i4 + i8) # E: numpy.signedinteger[numpy.typing._64Bit] +reveal_type(i4 + i4) # E: numpy.signedinteger[numpy.typing._32Bit] +reveal_type(i4 + i) # E: numpy.signedinteger[Any] +reveal_type(i4 + b_) # E: numpy.signedinteger[numpy.typing._32Bit] +reveal_type(i4 + b) # E: numpy.signedinteger[numpy.typing._32Bit] +reveal_type(i4 + AR) # E: Union[numpy.ndarray, numpy.generic] + +reveal_type(u4 + i8) # E: Union[numpy.signedinteger[Any], numpy.floating[numpy.typing._64Bit]] +reveal_type(u4 + i4) # E: Union[numpy.signedinteger[Any], numpy.floating[numpy.typing._64Bit]] +reveal_type(u4 + u8) # E: numpy.unsignedinteger[numpy.typing._64Bit] +reveal_type(u4 + u4) # E: numpy.unsignedinteger[numpy.typing._32Bit] +reveal_type(u4 + i) # E: Union[numpy.signedinteger[Any], numpy.floating[numpy.typing._64Bit]] +reveal_type(u4 + b_) # E: numpy.unsignedinteger[numpy.typing._32Bit] +reveal_type(u4 + b) # E: numpy.unsignedinteger[numpy.typing._32Bit] +reveal_type(u4 + AR) # E: Union[numpy.ndarray, numpy.generic] + +reveal_type(i8 + i4) # E: numpy.signedinteger[numpy.typing._64Bit] +reveal_type(i4 + i4) # E: numpy.signedinteger[numpy.typing._32Bit] +reveal_type(i + i4) # E: numpy.signedinteger[Any] +reveal_type(b_ + i4) # E: numpy.signedinteger[numpy.typing._32Bit] +reveal_type(b + i4) # E: numpy.signedinteger[numpy.typing._32Bit] +reveal_type(AR + i4) # E: Union[numpy.ndarray, numpy.generic] + +reveal_type(i8 + u4) # E: Union[numpy.signedinteger[Any], numpy.floating[numpy.typing._64Bit]] +reveal_type(i4 + u4) # E: Union[numpy.signedinteger[Any], numpy.floating[numpy.typing._64Bit]] +reveal_type(u8 + u4) # E: numpy.unsignedinteger[numpy.typing._64Bit] +reveal_type(u4 + u4) # E: numpy.unsignedinteger[numpy.typing._32Bit] +reveal_type(b_ + u4) # E: numpy.unsignedinteger[numpy.typing._32Bit] +reveal_type(b + u4) # E: numpy.unsignedinteger[numpy.typing._32Bit] +reveal_type(i + u4) # E: Union[numpy.signedinteger[Any], numpy.floating[numpy.typing._64Bit]] +reveal_type(AR + u4) # E: Union[numpy.ndarray, numpy.generic] diff --git a/numpy/typing/tests/data/reveal/array_constructors.py b/numpy/typing/tests/data/reveal/array_constructors.py new file mode 100644 index 000000000000..106174736085 --- /dev/null +++ b/numpy/typing/tests/data/reveal/array_constructors.py @@ -0,0 +1,102 @@ +from typing import List, Any +import numpy as np + +class SubClass(np.ndarray): ... + +i8: np.int64 + +A: np.ndarray +B: SubClass +C: List[int] + +def func(i: int, j: int, **kwargs: Any) -> SubClass: ... + +reveal_type(np.asarray(A)) # E: ndarray +reveal_type(np.asarray(B)) # E: ndarray +reveal_type(np.asarray(C)) # E: ndarray + +reveal_type(np.asanyarray(A)) # E: ndarray +reveal_type(np.asanyarray(B)) # E: SubClass +reveal_type(np.asanyarray(B, dtype=int)) # E: ndarray +reveal_type(np.asanyarray(C)) # E: ndarray + +reveal_type(np.ascontiguousarray(A)) # E: ndarray +reveal_type(np.ascontiguousarray(B)) # E: ndarray +reveal_type(np.ascontiguousarray(C)) # E: ndarray + +reveal_type(np.asfortranarray(A)) # E: ndarray +reveal_type(np.asfortranarray(B)) # E: ndarray +reveal_type(np.asfortranarray(C)) # E: ndarray + +reveal_type(np.require(A)) # E: ndarray +reveal_type(np.require(B)) # E: SubClass +reveal_type(np.require(B, requirements=None)) # E: SubClass +reveal_type(np.require(B, dtype=int)) # E: ndarray +reveal_type(np.require(B, requirements="E")) # E: ndarray +reveal_type(np.require(B, requirements=["ENSUREARRAY"])) # E: ndarray +reveal_type(np.require(B, requirements={"F", "E"})) # E: ndarray +reveal_type(np.require(B, requirements=["C", "OWNDATA"])) # E: SubClass +reveal_type(np.require(B, requirements="W")) # E: SubClass +reveal_type(np.require(B, requirements="A")) # E: SubClass +reveal_type(np.require(C)) # E: ndarray + +reveal_type(np.linspace(0, 10)) # E: numpy.ndarray +reveal_type(np.linspace(0, 10, retstep=True)) # E: Tuple[numpy.ndarray, numpy.inexact[Any]] +reveal_type(np.logspace(0, 10)) # E: numpy.ndarray +reveal_type(np.geomspace(1, 10)) # E: numpy.ndarray + +reveal_type(np.zeros_like(A)) # E: numpy.ndarray +reveal_type(np.zeros_like(C)) # E: numpy.ndarray +reveal_type(np.zeros_like(B)) # E: SubClass +reveal_type(np.zeros_like(B, dtype=np.int64)) # E: numpy.ndarray + +reveal_type(np.ones_like(A)) # E: numpy.ndarray +reveal_type(np.ones_like(C)) # E: numpy.ndarray +reveal_type(np.ones_like(B)) # E: SubClass +reveal_type(np.ones_like(B, dtype=np.int64)) # E: numpy.ndarray + +reveal_type(np.empty_like(A)) # E: numpy.ndarray +reveal_type(np.empty_like(C)) # E: numpy.ndarray +reveal_type(np.empty_like(B)) # E: SubClass +reveal_type(np.empty_like(B, dtype=np.int64)) # E: numpy.ndarray + +reveal_type(np.full_like(A, i8)) # E: numpy.ndarray +reveal_type(np.full_like(C, i8)) # E: numpy.ndarray +reveal_type(np.full_like(B, i8)) # E: SubClass +reveal_type(np.full_like(B, i8, dtype=np.int64)) # E: numpy.ndarray + +reveal_type(np.ones(1)) # E: numpy.ndarray +reveal_type(np.ones([1, 1, 1])) # E: numpy.ndarray + +reveal_type(np.full(1, i8)) # E: numpy.ndarray +reveal_type(np.full([1, 1, 1], i8)) # E: numpy.ndarray + +reveal_type(np.indices([1, 2, 3])) # E: numpy.ndarray +reveal_type(np.indices([1, 2, 3], sparse=True)) # E: tuple[numpy.ndarray] + +reveal_type(np.fromfunction(func, (3, 5))) # E: SubClass + +reveal_type(np.identity(10)) # E: numpy.ndarray + +reveal_type(np.atleast_1d(A)) # E: numpy.ndarray +reveal_type(np.atleast_1d(C)) # E: numpy.ndarray +reveal_type(np.atleast_1d(A, A)) # E: list[numpy.ndarray] +reveal_type(np.atleast_1d(A, C)) # E: list[numpy.ndarray] +reveal_type(np.atleast_1d(C, C)) # E: list[numpy.ndarray] + +reveal_type(np.atleast_2d(A)) # E: numpy.ndarray + +reveal_type(np.atleast_3d(A)) # E: numpy.ndarray + +reveal_type(np.vstack([A, A])) # E: numpy.ndarray +reveal_type(np.vstack([A, C])) # E: numpy.ndarray +reveal_type(np.vstack([C, C])) # E: numpy.ndarray + +reveal_type(np.hstack([A, A])) # E: numpy.ndarray + +reveal_type(np.stack([A, A])) # E: numpy.ndarray +reveal_type(np.stack([A, A], axis=0)) # E: numpy.ndarray +reveal_type(np.stack([A, A], out=B)) # E: SubClass + +reveal_type(np.block([[A, A], [A, A]])) # E: numpy.ndarray +reveal_type(np.block(C)) # E: numpy.ndarray diff --git a/numpy/typing/tests/data/reveal/bitwise_ops.py b/numpy/typing/tests/data/reveal/bitwise_ops.py new file mode 100644 index 000000000000..6883532f2502 --- /dev/null +++ b/numpy/typing/tests/data/reveal/bitwise_ops.py @@ -0,0 +1,131 @@ +import numpy as np + +i8 = np.int64(1) +u8 = np.uint64(1) + +i4 = np.int32(1) +u4 = np.uint32(1) + +b_ = np.bool_(1) + +b = bool(1) +i = int(1) + +AR = np.array([0, 1, 2], dtype=np.int32) +AR.setflags(write=False) + + +reveal_type(i8 << i8) # E: numpy.signedinteger[numpy.typing._64Bit] +reveal_type(i8 >> i8) # E: numpy.signedinteger[numpy.typing._64Bit] +reveal_type(i8 | i8) # E: numpy.signedinteger[numpy.typing._64Bit] +reveal_type(i8 ^ i8) # E: numpy.signedinteger[numpy.typing._64Bit] +reveal_type(i8 & i8) # E: numpy.signedinteger[numpy.typing._64Bit] + +reveal_type(i8 << AR) # E: Union[numpy.ndarray, numpy.integer[Any]] +reveal_type(i8 >> AR) # E: Union[numpy.ndarray, numpy.integer[Any]] +reveal_type(i8 | AR) # E: Union[numpy.ndarray, numpy.integer[Any], numpy.bool_] +reveal_type(i8 ^ AR) # E: Union[numpy.ndarray, numpy.integer[Any], numpy.bool_] +reveal_type(i8 & AR) # E: Union[numpy.ndarray, numpy.integer[Any], numpy.bool_] + +reveal_type(i4 << i4) # E: numpy.signedinteger[numpy.typing._32Bit] +reveal_type(i4 >> i4) # E: numpy.signedinteger[numpy.typing._32Bit] +reveal_type(i4 | i4) # E: numpy.signedinteger[numpy.typing._32Bit] +reveal_type(i4 ^ i4) # E: numpy.signedinteger[numpy.typing._32Bit] +reveal_type(i4 & i4) # E: numpy.signedinteger[numpy.typing._32Bit] + +reveal_type(i8 << i4) # E: numpy.signedinteger[numpy.typing._64Bit] +reveal_type(i8 >> i4) # E: numpy.signedinteger[numpy.typing._64Bit] +reveal_type(i8 | i4) # E: numpy.signedinteger[numpy.typing._64Bit] +reveal_type(i8 ^ i4) # E: numpy.signedinteger[numpy.typing._64Bit] +reveal_type(i8 & i4) # E: numpy.signedinteger[numpy.typing._64Bit] + +reveal_type(i8 << i) # E: numpy.signedinteger[Any] +reveal_type(i8 >> i) # E: numpy.signedinteger[Any] +reveal_type(i8 | i) # E: numpy.signedinteger[Any] +reveal_type(i8 ^ i) # E: numpy.signedinteger[Any] +reveal_type(i8 & i) # E: numpy.signedinteger[Any] + +reveal_type(i8 << b_) # E: numpy.signedinteger[numpy.typing._64Bit] +reveal_type(i8 >> b_) # E: numpy.signedinteger[numpy.typing._64Bit] +reveal_type(i8 | b_) # E: numpy.signedinteger[numpy.typing._64Bit] +reveal_type(i8 ^ b_) # E: numpy.signedinteger[numpy.typing._64Bit] +reveal_type(i8 & b_) # E: numpy.signedinteger[numpy.typing._64Bit] + +reveal_type(i8 << b) # E: numpy.signedinteger[numpy.typing._64Bit] +reveal_type(i8 >> b) # E: numpy.signedinteger[numpy.typing._64Bit] +reveal_type(i8 | b) # E: numpy.signedinteger[numpy.typing._64Bit] +reveal_type(i8 ^ b) # E: numpy.signedinteger[numpy.typing._64Bit] +reveal_type(i8 & b) # E: numpy.signedinteger[numpy.typing._64Bit] + +reveal_type(u8 << u8) # E: numpy.unsignedinteger[numpy.typing._64Bit] +reveal_type(u8 >> u8) # E: numpy.unsignedinteger[numpy.typing._64Bit] +reveal_type(u8 | u8) # E: numpy.unsignedinteger[numpy.typing._64Bit] +reveal_type(u8 ^ u8) # E: numpy.unsignedinteger[numpy.typing._64Bit] +reveal_type(u8 & u8) # E: numpy.unsignedinteger[numpy.typing._64Bit] + +reveal_type(u8 << AR) # E: Union[numpy.ndarray, numpy.integer[Any]] +reveal_type(u8 >> AR) # E: Union[numpy.ndarray, numpy.integer[Any]] +reveal_type(u8 | AR) # E: Union[numpy.ndarray, numpy.integer[Any], numpy.bool_] +reveal_type(u8 ^ AR) # E: Union[numpy.ndarray, numpy.integer[Any], numpy.bool_] +reveal_type(u8 & AR) # E: Union[numpy.ndarray, numpy.integer[Any], numpy.bool_] + +reveal_type(u4 << u4) # E: numpy.unsignedinteger[numpy.typing._32Bit] +reveal_type(u4 >> u4) # E: numpy.unsignedinteger[numpy.typing._32Bit] +reveal_type(u4 | u4) # E: numpy.unsignedinteger[numpy.typing._32Bit] +reveal_type(u4 ^ u4) # E: numpy.unsignedinteger[numpy.typing._32Bit] +reveal_type(u4 & u4) # E: numpy.unsignedinteger[numpy.typing._32Bit] + +reveal_type(u4 << i4) # E: numpy.signedinteger[Any] +reveal_type(u4 >> i4) # E: numpy.signedinteger[Any] +reveal_type(u4 | i4) # E: numpy.signedinteger[Any] +reveal_type(u4 ^ i4) # E: numpy.signedinteger[Any] +reveal_type(u4 & i4) # E: numpy.signedinteger[Any] + +reveal_type(u4 << i) # E: numpy.signedinteger[Any] +reveal_type(u4 >> i) # E: numpy.signedinteger[Any] +reveal_type(u4 | i) # E: numpy.signedinteger[Any] +reveal_type(u4 ^ i) # E: numpy.signedinteger[Any] +reveal_type(u4 & i) # E: numpy.signedinteger[Any] + +reveal_type(u8 << b_) # E: numpy.unsignedinteger[numpy.typing._64Bit] +reveal_type(u8 >> b_) # E: numpy.unsignedinteger[numpy.typing._64Bit] +reveal_type(u8 | b_) # E: numpy.unsignedinteger[numpy.typing._64Bit] +reveal_type(u8 ^ b_) # E: numpy.unsignedinteger[numpy.typing._64Bit] +reveal_type(u8 & b_) # E: numpy.unsignedinteger[numpy.typing._64Bit] + +reveal_type(u8 << b) # E: numpy.unsignedinteger[numpy.typing._64Bit] +reveal_type(u8 >> b) # E: numpy.unsignedinteger[numpy.typing._64Bit] +reveal_type(u8 | b) # E: numpy.unsignedinteger[numpy.typing._64Bit] +reveal_type(u8 ^ b) # E: numpy.unsignedinteger[numpy.typing._64Bit] +reveal_type(u8 & b) # E: numpy.unsignedinteger[numpy.typing._64Bit] + +reveal_type(b_ << b_) # E: numpy.signedinteger[numpy.typing._8Bit] +reveal_type(b_ >> b_) # E: numpy.signedinteger[numpy.typing._8Bit] +reveal_type(b_ | b_) # E: numpy.bool_ +reveal_type(b_ ^ b_) # E: numpy.bool_ +reveal_type(b_ & b_) # E: numpy.bool_ + +reveal_type(b_ << AR) # E: Union[numpy.ndarray, numpy.integer[Any]] +reveal_type(b_ >> AR) # E: Union[numpy.ndarray, numpy.integer[Any]] +reveal_type(b_ | AR) # E: Union[numpy.ndarray, numpy.integer[Any], numpy.bool_] +reveal_type(b_ ^ AR) # E: Union[numpy.ndarray, numpy.integer[Any], numpy.bool_] +reveal_type(b_ & AR) # E: Union[numpy.ndarray, numpy.integer[Any], numpy.bool_] + +reveal_type(b_ << b) # E: numpy.signedinteger[numpy.typing._8Bit] +reveal_type(b_ >> b) # E: numpy.signedinteger[numpy.typing._8Bit] +reveal_type(b_ | b) # E: numpy.bool_ +reveal_type(b_ ^ b) # E: numpy.bool_ +reveal_type(b_ & b) # E: numpy.bool_ + +reveal_type(b_ << i) # E: numpy.signedinteger[Any] +reveal_type(b_ >> i) # E: numpy.signedinteger[Any] +reveal_type(b_ | i) # E: numpy.signedinteger[Any] +reveal_type(b_ ^ i) # E: numpy.signedinteger[Any] +reveal_type(b_ & i) # E: numpy.signedinteger[Any] + +reveal_type(~i8) # E: numpy.signedinteger[numpy.typing._64Bit] +reveal_type(~i4) # E: numpy.signedinteger[numpy.typing._32Bit] +reveal_type(~u8) # E: numpy.unsignedinteger[numpy.typing._64Bit] +reveal_type(~u4) # E: numpy.unsignedinteger[numpy.typing._32Bit] +reveal_type(~b_) # E: numpy.bool_ +reveal_type(~AR) # E: Union[numpy.ndarray*, numpy.integer[Any], numpy.bool_] diff --git a/numpy/typing/tests/data/reveal/constants.py b/numpy/typing/tests/data/reveal/constants.py index 8e00810bd7c4..b2382e8611a3 100644 --- a/numpy/typing/tests/data/reveal/constants.py +++ b/numpy/typing/tests/data/reveal/constants.py @@ -40,5 +40,13 @@ reveal_type(np.SHIFT_UNDERFLOW) # E: int reveal_type(np.UFUNC_BUFSIZE_DEFAULT) # E: int reveal_type(np.WRAP) # E: int -reveal_type(np.little_endian) # E: int reveal_type(np.tracemalloc_domain) # E: int + +reveal_type(np.little_endian) # E: bool +reveal_type(np.True_) # E: numpy.bool_ +reveal_type(np.False_) # E: numpy.bool_ + +reveal_type(np.UFUNC_PYVALS_NAME) # E: str + +reveal_type(np.sctypeDict) # E: dict +reveal_type(np.sctypes) # E: TypedDict diff --git a/numpy/typing/tests/data/reveal/dtype.py b/numpy/typing/tests/data/reveal/dtype.py new file mode 100644 index 000000000000..d414f2c4934f --- /dev/null +++ b/numpy/typing/tests/data/reveal/dtype.py @@ -0,0 +1,33 @@ +import numpy as np + +reveal_type(np.dtype(np.float64)) # E: numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(np.dtype(np.int64)) # E: numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]] + +# String aliases +reveal_type(np.dtype("float64")) # E: numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(np.dtype("float32")) # E: numpy.dtype[numpy.floating[numpy.typing._32Bit]] +reveal_type(np.dtype("int64")) # E: numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]] +reveal_type(np.dtype("int32")) # E: numpy.dtype[numpy.signedinteger[numpy.typing._32Bit]] +reveal_type(np.dtype("bool")) # E: numpy.dtype[numpy.bool_] +reveal_type(np.dtype("bytes")) # E: numpy.dtype[numpy.bytes_] +reveal_type(np.dtype("str")) # E: numpy.dtype[numpy.str_] + +# Python types +reveal_type(np.dtype(complex)) # E: numpy.dtype[numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit]] +reveal_type(np.dtype(float)) # E: numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(np.dtype(int)) # E: numpy.dtype +reveal_type(np.dtype(bool)) # E: numpy.dtype[numpy.bool_] +reveal_type(np.dtype(str)) # E: numpy.dtype[numpy.str_] +reveal_type(np.dtype(bytes)) # E: numpy.dtype[numpy.bytes_] + +# Special case for None +reveal_type(np.dtype(None)) # E: numpy.dtype[numpy.floating[numpy.typing._64Bit]] + +# Dtypes of dtypes +reveal_type(np.dtype(np.dtype(np.float64))) # E: numpy.dtype[numpy.floating[numpy.typing._64Bit]] + +# Parameterized dtypes +reveal_type(np.dtype("S8")) # E: numpy.dtype + +# Void +reveal_type(np.dtype(("U", 10))) # E: numpy.dtype[numpy.void] diff --git a/numpy/typing/tests/data/reveal/fromnumeric.py b/numpy/typing/tests/data/reveal/fromnumeric.py index 06501f6e2f87..75865c285f36 100644 --- a/numpy/typing/tests/data/reveal/fromnumeric.py +++ b/numpy/typing/tests/data/reveal/fromnumeric.py @@ -13,7 +13,7 @@ d = np.array(1.0, dtype=np.float32) # writeable reveal_type(np.take(a, 0)) # E: numpy.bool_ -reveal_type(np.take(b, 0)) # E: numpy.float32 +reveal_type(np.take(b, 0)) # E: numpy.floating[numpy.typing._32Bit] reveal_type( np.take(c, 0) # E: Union[numpy.generic, datetime.datetime, datetime.timedelta] ) @@ -66,8 +66,8 @@ reveal_type(np.partition(A, 0)) # E: numpy.ndarray reveal_type(np.partition(B, 0)) # E: numpy.ndarray -reveal_type(np.argpartition(a, 0)) # E: numpy.integer -reveal_type(np.argpartition(b, 0)) # E: numpy.integer +reveal_type(np.argpartition(a, 0)) # E: numpy.integer[Any] +reveal_type(np.argpartition(b, 0)) # E: numpy.integer[Any] reveal_type(np.argpartition(c, 0)) # E: numpy.ndarray reveal_type(np.argpartition(A, 0)) # E: numpy.ndarray reveal_type(np.argpartition(B, 0)) # E: numpy.ndarray @@ -78,18 +78,18 @@ reveal_type(np.argsort(A, 0)) # E: numpy.ndarray reveal_type(np.argsort(B, 0)) # E: numpy.ndarray -reveal_type(np.argmax(A)) # E: numpy.integer -reveal_type(np.argmax(B)) # E: numpy.integer -reveal_type(np.argmax(A, axis=0)) # E: Union[numpy.integer, numpy.ndarray] -reveal_type(np.argmax(B, axis=0)) # E: Union[numpy.integer, numpy.ndarray] +reveal_type(np.argmax(A)) # E: numpy.integer[Any] +reveal_type(np.argmax(B)) # E: numpy.integer[Any] +reveal_type(np.argmax(A, axis=0)) # E: Union[numpy.integer[Any], numpy.ndarray] +reveal_type(np.argmax(B, axis=0)) # E: Union[numpy.integer[Any], numpy.ndarray] -reveal_type(np.argmin(A)) # E: numpy.integer -reveal_type(np.argmin(B)) # E: numpy.integer -reveal_type(np.argmin(A, axis=0)) # E: Union[numpy.integer, numpy.ndarray] -reveal_type(np.argmin(B, axis=0)) # E: Union[numpy.integer, numpy.ndarray] +reveal_type(np.argmin(A)) # E: numpy.integer[Any] +reveal_type(np.argmin(B)) # E: numpy.integer[Any] +reveal_type(np.argmin(A, axis=0)) # E: Union[numpy.integer[Any], numpy.ndarray] +reveal_type(np.argmin(B, axis=0)) # E: Union[numpy.integer[Any], numpy.ndarray] -reveal_type(np.searchsorted(A[0], 0)) # E: numpy.integer -reveal_type(np.searchsorted(B[0], 0)) # E: numpy.integer +reveal_type(np.searchsorted(A[0], 0)) # E: numpy.integer[Any] +reveal_type(np.searchsorted(B[0], 0)) # E: numpy.integer[Any] reveal_type(np.searchsorted(A[0], [0])) # E: numpy.ndarray reveal_type(np.searchsorted(B[0], [0])) # E: numpy.ndarray @@ -100,7 +100,7 @@ reveal_type(np.resize(B, (5, 5))) # E: numpy.ndarray reveal_type(np.squeeze(a)) # E: numpy.bool_ -reveal_type(np.squeeze(b)) # E: numpy.float32 +reveal_type(np.squeeze(b)) # E: numpy.floating[numpy.typing._32Bit] reveal_type(np.squeeze(c)) # E: numpy.ndarray reveal_type(np.squeeze(A)) # E: numpy.ndarray reveal_type(np.squeeze(B)) # E: numpy.ndarray @@ -108,8 +108,8 @@ reveal_type(np.diagonal(A)) # E: numpy.ndarray reveal_type(np.diagonal(B)) # E: numpy.ndarray -reveal_type(np.trace(A)) # E: Union[numpy.number, numpy.ndarray] -reveal_type(np.trace(B)) # E: Union[numpy.number, numpy.ndarray] +reveal_type(np.trace(A)) # E: Union[numpy.number[Any], numpy.ndarray] +reveal_type(np.trace(B)) # E: Union[numpy.number[Any], numpy.ndarray] reveal_type(np.ravel(a)) # E: numpy.ndarray reveal_type(np.ravel(b)) # E: numpy.ndarray @@ -135,19 +135,19 @@ reveal_type(np.compress([True], A)) # E: numpy.ndarray reveal_type(np.compress([True], B)) # E: numpy.ndarray -reveal_type(np.clip(a, 0, 1.0)) # E: numpy.number -reveal_type(np.clip(b, -1, 1)) # E: numpy.float32 -reveal_type(np.clip(c, 0, 1)) # E: numpy.number -reveal_type(np.clip(A, 0, 1)) # E: Union[numpy.number, numpy.ndarray] -reveal_type(np.clip(B, 0, 1)) # E: Union[numpy.number, numpy.ndarray] +reveal_type(np.clip(a, 0, 1.0)) # E: numpy.number[Any] +reveal_type(np.clip(b, -1, 1)) # E: numpy.floating[numpy.typing._32Bit] +reveal_type(np.clip(c, 0, 1)) # E: numpy.number[Any] +reveal_type(np.clip(A, 0, 1)) # E: Union[numpy.number[Any], numpy.ndarray] +reveal_type(np.clip(B, 0, 1)) # E: Union[numpy.number[Any], numpy.ndarray] -reveal_type(np.sum(a)) # E: numpy.number -reveal_type(np.sum(b)) # E: numpy.float32 -reveal_type(np.sum(c)) # E: numpy.number -reveal_type(np.sum(A)) # E: numpy.number -reveal_type(np.sum(B)) # E: numpy.number -reveal_type(np.sum(A, axis=0)) # E: Union[numpy.number, numpy.ndarray] -reveal_type(np.sum(B, axis=0)) # E: Union[numpy.number, numpy.ndarray] +reveal_type(np.sum(a)) # E: numpy.number[Any] +reveal_type(np.sum(b)) # E: numpy.floating[numpy.typing._32Bit] +reveal_type(np.sum(c)) # E: numpy.number[Any] +reveal_type(np.sum(A)) # E: numpy.number[Any] +reveal_type(np.sum(B)) # E: numpy.number[Any] +reveal_type(np.sum(A, axis=0)) # E: Union[numpy.number[Any], numpy.ndarray] +reveal_type(np.sum(B, axis=0)) # E: Union[numpy.number[Any], numpy.ndarray] reveal_type(np.all(a)) # E: numpy.bool_ reveal_type(np.all(b)) # E: numpy.bool_ @@ -175,45 +175,45 @@ reveal_type(np.cumsum(A)) # E: numpy.ndarray reveal_type(np.cumsum(B)) # E: numpy.ndarray -reveal_type(np.ptp(a)) # E: numpy.number -reveal_type(np.ptp(b)) # E: numpy.float32 -reveal_type(np.ptp(c)) # E: numpy.number -reveal_type(np.ptp(A)) # E: numpy.number -reveal_type(np.ptp(B)) # E: numpy.number -reveal_type(np.ptp(A, axis=0)) # E: Union[numpy.number, numpy.ndarray] -reveal_type(np.ptp(B, axis=0)) # E: Union[numpy.number, numpy.ndarray] -reveal_type(np.ptp(A, keepdims=True)) # E: Union[numpy.number, numpy.ndarray] -reveal_type(np.ptp(B, keepdims=True)) # E: Union[numpy.number, numpy.ndarray] - -reveal_type(np.amax(a)) # E: numpy.number -reveal_type(np.amax(b)) # E: numpy.float32 -reveal_type(np.amax(c)) # E: numpy.number -reveal_type(np.amax(A)) # E: numpy.number -reveal_type(np.amax(B)) # E: numpy.number -reveal_type(np.amax(A, axis=0)) # E: Union[numpy.number, numpy.ndarray] -reveal_type(np.amax(B, axis=0)) # E: Union[numpy.number, numpy.ndarray] -reveal_type(np.amax(A, keepdims=True)) # E: Union[numpy.number, numpy.ndarray] -reveal_type(np.amax(B, keepdims=True)) # E: Union[numpy.number, numpy.ndarray] - -reveal_type(np.amin(a)) # E: numpy.number -reveal_type(np.amin(b)) # E: numpy.float32 -reveal_type(np.amin(c)) # E: numpy.number -reveal_type(np.amin(A)) # E: numpy.number -reveal_type(np.amin(B)) # E: numpy.number -reveal_type(np.amin(A, axis=0)) # E: Union[numpy.number, numpy.ndarray] -reveal_type(np.amin(B, axis=0)) # E: Union[numpy.number, numpy.ndarray] -reveal_type(np.amin(A, keepdims=True)) # E: Union[numpy.number, numpy.ndarray] -reveal_type(np.amin(B, keepdims=True)) # E: Union[numpy.number, numpy.ndarray] - -reveal_type(np.prod(a)) # E: numpy.number -reveal_type(np.prod(b)) # E: numpy.float32 -reveal_type(np.prod(c)) # E: numpy.number -reveal_type(np.prod(A)) # E: numpy.number -reveal_type(np.prod(B)) # E: numpy.number -reveal_type(np.prod(A, axis=0)) # E: Union[numpy.number, numpy.ndarray] -reveal_type(np.prod(B, axis=0)) # E: Union[numpy.number, numpy.ndarray] -reveal_type(np.prod(A, keepdims=True)) # E: Union[numpy.number, numpy.ndarray] -reveal_type(np.prod(B, keepdims=True)) # E: Union[numpy.number, numpy.ndarray] +reveal_type(np.ptp(a)) # E: numpy.number[Any] +reveal_type(np.ptp(b)) # E: numpy.floating[numpy.typing._32Bit] +reveal_type(np.ptp(c)) # E: numpy.number[Any] +reveal_type(np.ptp(A)) # E: numpy.number[Any] +reveal_type(np.ptp(B)) # E: numpy.number[Any] +reveal_type(np.ptp(A, axis=0)) # E: Union[numpy.number[Any], numpy.ndarray] +reveal_type(np.ptp(B, axis=0)) # E: Union[numpy.number[Any], numpy.ndarray] +reveal_type(np.ptp(A, keepdims=True)) # E: Union[numpy.number[Any], numpy.ndarray] +reveal_type(np.ptp(B, keepdims=True)) # E: Union[numpy.number[Any], numpy.ndarray] + +reveal_type(np.amax(a)) # E: numpy.number[Any] +reveal_type(np.amax(b)) # E: numpy.floating[numpy.typing._32Bit] +reveal_type(np.amax(c)) # E: numpy.number[Any] +reveal_type(np.amax(A)) # E: numpy.number[Any] +reveal_type(np.amax(B)) # E: numpy.number[Any] +reveal_type(np.amax(A, axis=0)) # E: Union[numpy.number[Any], numpy.ndarray] +reveal_type(np.amax(B, axis=0)) # E: Union[numpy.number[Any], numpy.ndarray] +reveal_type(np.amax(A, keepdims=True)) # E: Union[numpy.number[Any], numpy.ndarray] +reveal_type(np.amax(B, keepdims=True)) # E: Union[numpy.number[Any], numpy.ndarray] + +reveal_type(np.amin(a)) # E: numpy.number[Any] +reveal_type(np.amin(b)) # E: numpy.floating[numpy.typing._32Bit] +reveal_type(np.amin(c)) # E: numpy.number[Any] +reveal_type(np.amin(A)) # E: numpy.number[Any] +reveal_type(np.amin(B)) # E: numpy.number[Any] +reveal_type(np.amin(A, axis=0)) # E: Union[numpy.number[Any], numpy.ndarray] +reveal_type(np.amin(B, axis=0)) # E: Union[numpy.number[Any], numpy.ndarray] +reveal_type(np.amin(A, keepdims=True)) # E: Union[numpy.number[Any], numpy.ndarray] +reveal_type(np.amin(B, keepdims=True)) # E: Union[numpy.number[Any], numpy.ndarray] + +reveal_type(np.prod(a)) # E: numpy.number[Any] +reveal_type(np.prod(b)) # E: numpy.floating[numpy.typing._32Bit] +reveal_type(np.prod(c)) # E: numpy.number[Any] +reveal_type(np.prod(A)) # E: numpy.number[Any] +reveal_type(np.prod(B)) # E: numpy.number[Any] +reveal_type(np.prod(A, axis=0)) # E: Union[numpy.number[Any], numpy.ndarray] +reveal_type(np.prod(B, axis=0)) # E: Union[numpy.number[Any], numpy.ndarray] +reveal_type(np.prod(A, keepdims=True)) # E: Union[numpy.number[Any], numpy.ndarray] +reveal_type(np.prod(B, keepdims=True)) # E: Union[numpy.number[Any], numpy.ndarray] reveal_type(np.prod(b, out=d)) # E: numpy.ndarray reveal_type(np.prod(B, out=d)) # E: numpy.ndarray @@ -235,44 +235,44 @@ reveal_type(np.size(A)) # E: int reveal_type(np.size(B)) # E: int -reveal_type(np.around(a)) # E: numpy.number -reveal_type(np.around(b)) # E: numpy.float32 -reveal_type(np.around(c)) # E: numpy.number +reveal_type(np.around(a)) # E: numpy.number[Any] +reveal_type(np.around(b)) # E: numpy.floating[numpy.typing._32Bit] +reveal_type(np.around(c)) # E: numpy.number[Any] reveal_type(np.around(A)) # E: numpy.ndarray reveal_type(np.around(B)) # E: numpy.ndarray -reveal_type(np.mean(a)) # E: numpy.number -reveal_type(np.mean(b)) # E: numpy.number -reveal_type(np.mean(c)) # E: numpy.number -reveal_type(np.mean(A)) # E: numpy.number -reveal_type(np.mean(B)) # E: numpy.number -reveal_type(np.mean(A, axis=0)) # E: Union[numpy.number, numpy.ndarray] -reveal_type(np.mean(B, axis=0)) # E: Union[numpy.number, numpy.ndarray] -reveal_type(np.mean(A, keepdims=True)) # E: Union[numpy.number, numpy.ndarray] -reveal_type(np.mean(B, keepdims=True)) # E: Union[numpy.number, numpy.ndarray] +reveal_type(np.mean(a)) # E: numpy.number[Any] +reveal_type(np.mean(b)) # E: numpy.number[Any] +reveal_type(np.mean(c)) # E: numpy.number[Any] +reveal_type(np.mean(A)) # E: numpy.number[Any] +reveal_type(np.mean(B)) # E: numpy.number[Any] +reveal_type(np.mean(A, axis=0)) # E: Union[numpy.number[Any], numpy.ndarray] +reveal_type(np.mean(B, axis=0)) # E: Union[numpy.number[Any], numpy.ndarray] +reveal_type(np.mean(A, keepdims=True)) # E: Union[numpy.number[Any], numpy.ndarray] +reveal_type(np.mean(B, keepdims=True)) # E: Union[numpy.number[Any], numpy.ndarray] reveal_type(np.mean(b, out=d)) # E: numpy.ndarray reveal_type(np.mean(B, out=d)) # E: numpy.ndarray -reveal_type(np.std(a)) # E: numpy.number -reveal_type(np.std(b)) # E: numpy.number -reveal_type(np.std(c)) # E: numpy.number -reveal_type(np.std(A)) # E: numpy.number -reveal_type(np.std(B)) # E: numpy.number -reveal_type(np.std(A, axis=0)) # E: Union[numpy.number, numpy.ndarray] -reveal_type(np.std(B, axis=0)) # E: Union[numpy.number, numpy.ndarray] -reveal_type(np.std(A, keepdims=True)) # E: Union[numpy.number, numpy.ndarray] -reveal_type(np.std(B, keepdims=True)) # E: Union[numpy.number, numpy.ndarray] +reveal_type(np.std(a)) # E: numpy.number[Any] +reveal_type(np.std(b)) # E: numpy.number[Any] +reveal_type(np.std(c)) # E: numpy.number[Any] +reveal_type(np.std(A)) # E: numpy.number[Any] +reveal_type(np.std(B)) # E: numpy.number[Any] +reveal_type(np.std(A, axis=0)) # E: Union[numpy.number[Any], numpy.ndarray] +reveal_type(np.std(B, axis=0)) # E: Union[numpy.number[Any], numpy.ndarray] +reveal_type(np.std(A, keepdims=True)) # E: Union[numpy.number[Any], numpy.ndarray] +reveal_type(np.std(B, keepdims=True)) # E: Union[numpy.number[Any], numpy.ndarray] reveal_type(np.std(b, out=d)) # E: numpy.ndarray reveal_type(np.std(B, out=d)) # E: numpy.ndarray -reveal_type(np.var(a)) # E: numpy.number -reveal_type(np.var(b)) # E: numpy.number -reveal_type(np.var(c)) # E: numpy.number -reveal_type(np.var(A)) # E: numpy.number -reveal_type(np.var(B)) # E: numpy.number -reveal_type(np.var(A, axis=0)) # E: Union[numpy.number, numpy.ndarray] -reveal_type(np.var(B, axis=0)) # E: Union[numpy.number, numpy.ndarray] -reveal_type(np.var(A, keepdims=True)) # E: Union[numpy.number, numpy.ndarray] -reveal_type(np.var(B, keepdims=True)) # E: Union[numpy.number, numpy.ndarray] +reveal_type(np.var(a)) # E: numpy.number[Any] +reveal_type(np.var(b)) # E: numpy.number[Any] +reveal_type(np.var(c)) # E: numpy.number[Any] +reveal_type(np.var(A)) # E: numpy.number[Any] +reveal_type(np.var(B)) # E: numpy.number[Any] +reveal_type(np.var(A, axis=0)) # E: Union[numpy.number[Any], numpy.ndarray] +reveal_type(np.var(B, axis=0)) # E: Union[numpy.number[Any], numpy.ndarray] +reveal_type(np.var(A, keepdims=True)) # E: Union[numpy.number[Any], numpy.ndarray] +reveal_type(np.var(B, keepdims=True)) # E: Union[numpy.number[Any], numpy.ndarray] reveal_type(np.var(b, out=d)) # E: numpy.ndarray reveal_type(np.var(B, out=d)) # E: numpy.ndarray diff --git a/numpy/typing/tests/data/reveal/linspace.py b/numpy/typing/tests/data/reveal/linspace.py deleted file mode 100644 index cfbbdf390264..000000000000 --- a/numpy/typing/tests/data/reveal/linspace.py +++ /dev/null @@ -1,6 +0,0 @@ -import numpy as np - -reveal_type(np.linspace(0, 10)) # E: numpy.ndarray -reveal_type(np.linspace(0, 10, retstep=True)) # E: Tuple[numpy.ndarray, numpy.inexact] -reveal_type(np.logspace(0, 10)) # E: numpy.ndarray -reveal_type(np.geomspace(1, 10)) # E: numpy.ndarray diff --git a/numpy/typing/tests/data/reveal/mod.py b/numpy/typing/tests/data/reveal/mod.py new file mode 100644 index 000000000000..9cd2c2fcfd60 --- /dev/null +++ b/numpy/typing/tests/data/reveal/mod.py @@ -0,0 +1,149 @@ +import numpy as np + +f8 = np.float64() +i8 = np.int64() +u8 = np.uint64() + +f4 = np.float32() +i4 = np.int32() +u4 = np.uint32() + +td = np.timedelta64(0, "D") +b_ = np.bool_() + +b = bool() +f = float() +i = int() + +AR = np.array([1], dtype=np.bool_) +AR.setflags(write=False) + +AR2 = np.array([1], dtype=np.timedelta64) +AR2.setflags(write=False) + +# Time structures + +reveal_type(td % td) # E: numpy.timedelta64 +reveal_type(AR2 % td) # E: Union[numpy.ndarray, numpy.generic] +reveal_type(td % AR2) # E: Union[numpy.ndarray, numpy.generic] + +reveal_type(divmod(td, td)) # E: Tuple[numpy.signedinteger[numpy.typing._64Bit], numpy.timedelta64] +reveal_type(divmod(AR2, td)) # E: Union[Tuple[numpy.ndarray, numpy.ndarray], Tuple[numpy.generic, numpy.generic]] +reveal_type(divmod(td, AR2)) # E: Union[Tuple[numpy.ndarray, numpy.ndarray], Tuple[numpy.generic, numpy.generic]] + +# Bool + +reveal_type(b_ % b) # E: numpy.signedinteger[numpy.typing._8Bit] +reveal_type(b_ % i) # E: numpy.signedinteger[Any] +reveal_type(b_ % f) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(b_ % b_) # E: numpy.signedinteger[numpy.typing._8Bit] +reveal_type(b_ % i8) # E: numpy.signedinteger[numpy.typing._64Bit] +reveal_type(b_ % u8) # E: numpy.unsignedinteger[numpy.typing._64Bit] +reveal_type(b_ % f8) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(b_ % AR) # E: Union[numpy.ndarray, numpy.generic] + +reveal_type(divmod(b_, b)) # E: Tuple[numpy.signedinteger[numpy.typing._8Bit], numpy.signedinteger[numpy.typing._8Bit]] +reveal_type(divmod(b_, i)) # E: Tuple[numpy.signedinteger[Any], numpy.signedinteger[Any]] +reveal_type(divmod(b_, f)) # E: Tuple[numpy.floating[numpy.typing._64Bit], numpy.floating[numpy.typing._64Bit]] +reveal_type(divmod(b_, b_)) # E: Tuple[numpy.signedinteger[numpy.typing._8Bit], numpy.signedinteger[numpy.typing._8Bit]] +reveal_type(divmod(b_, i8)) # E: Tuple[numpy.signedinteger[numpy.typing._64Bit], numpy.signedinteger[numpy.typing._64Bit]] +reveal_type(divmod(b_, u8)) # E: Tuple[numpy.unsignedinteger[numpy.typing._64Bit], numpy.unsignedinteger[numpy.typing._64Bit]] +reveal_type(divmod(b_, f8)) # E: Tuple[numpy.floating[numpy.typing._64Bit], numpy.floating[numpy.typing._64Bit]] +reveal_type(divmod(b_, AR)) # E: Union[Tuple[numpy.ndarray, numpy.ndarray], Tuple[numpy.generic, numpy.generic]] + +reveal_type(b % b_) # E: numpy.signedinteger[numpy.typing._8Bit] +reveal_type(i % b_) # E: numpy.signedinteger[Any] +reveal_type(f % b_) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(b_ % b_) # E: numpy.signedinteger[numpy.typing._8Bit] +reveal_type(i8 % b_) # E: numpy.signedinteger[numpy.typing._64Bit] +reveal_type(u8 % b_) # E: numpy.unsignedinteger[numpy.typing._64Bit] +reveal_type(f8 % b_) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(AR % b_) # E: Union[numpy.ndarray, numpy.generic] + +reveal_type(divmod(b, b_)) # E: Tuple[numpy.signedinteger[numpy.typing._8Bit], numpy.signedinteger[numpy.typing._8Bit]] +reveal_type(divmod(i, b_)) # E: Tuple[numpy.signedinteger[Any], numpy.signedinteger[Any]] +reveal_type(divmod(f, b_)) # E: Tuple[numpy.floating[numpy.typing._64Bit], numpy.floating[numpy.typing._64Bit]] +reveal_type(divmod(b_, b_)) # E: Tuple[numpy.signedinteger[numpy.typing._8Bit], numpy.signedinteger[numpy.typing._8Bit]] +reveal_type(divmod(i8, b_)) # E: Tuple[numpy.signedinteger[numpy.typing._64Bit], numpy.signedinteger[numpy.typing._64Bit]] +reveal_type(divmod(u8, b_)) # E: Tuple[numpy.unsignedinteger[numpy.typing._64Bit], numpy.unsignedinteger[numpy.typing._64Bit]] +reveal_type(divmod(f8, b_)) # E: Tuple[numpy.floating[numpy.typing._64Bit], numpy.floating[numpy.typing._64Bit]] +reveal_type(divmod(AR, b_)) # E: Union[Tuple[numpy.ndarray, numpy.ndarray], Tuple[numpy.generic, numpy.generic]] + +# int + +reveal_type(i8 % b) # E: numpy.signedinteger[numpy.typing._64Bit] +reveal_type(i8 % i) # E: numpy.signedinteger[Any] +reveal_type(i8 % f) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(i8 % i8) # E: numpy.signedinteger[numpy.typing._64Bit] +reveal_type(i8 % f8) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(i4 % i8) # E: numpy.signedinteger[numpy.typing._64Bit] +reveal_type(i4 % f8) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(i4 % i4) # E: numpy.signedinteger[numpy.typing._32Bit] +reveal_type(i4 % f4) # E: numpy.floating[numpy.typing._32Bit] +reveal_type(i8 % AR) # E: Union[numpy.ndarray, numpy.generic] + +reveal_type(divmod(i8, b)) # E: Tuple[numpy.signedinteger[numpy.typing._64Bit], numpy.signedinteger[numpy.typing._64Bit]] +reveal_type(divmod(i8, i)) # E: Tuple[numpy.signedinteger[Any], numpy.signedinteger[Any]] +reveal_type(divmod(i8, f)) # E: Tuple[numpy.floating[numpy.typing._64Bit], numpy.floating[numpy.typing._64Bit]] +reveal_type(divmod(i8, i8)) # E: Tuple[numpy.signedinteger[numpy.typing._64Bit], numpy.signedinteger[numpy.typing._64Bit]] +reveal_type(divmod(i8, f8)) # E: Tuple[numpy.floating[numpy.typing._64Bit], numpy.floating[numpy.typing._64Bit]] +reveal_type(divmod(i8, i4)) # E: Tuple[numpy.signedinteger[numpy.typing._64Bit], numpy.signedinteger[numpy.typing._64Bit]] +reveal_type(divmod(i8, f4)) # E: Tuple[numpy.floating[numpy.typing._64Bit], numpy.floating[numpy.typing._64Bit]] +reveal_type(divmod(i4, i4)) # E: Tuple[numpy.signedinteger[numpy.typing._32Bit], numpy.signedinteger[numpy.typing._32Bit]] +reveal_type(divmod(i4, f4)) # E: Tuple[numpy.floating[numpy.typing._32Bit], numpy.floating[numpy.typing._32Bit]] +reveal_type(divmod(i8, AR)) # E: Union[Tuple[numpy.ndarray, numpy.ndarray], Tuple[numpy.generic, numpy.generic]] + +reveal_type(b % i8) # E: numpy.signedinteger[numpy.typing._64Bit] +reveal_type(i % i8) # E: numpy.signedinteger[Any] +reveal_type(f % i8) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(i8 % i8) # E: numpy.signedinteger[numpy.typing._64Bit] +reveal_type(f8 % i8) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(i8 % i4) # E: numpy.signedinteger[numpy.typing._64Bit] +reveal_type(f8 % i4) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(i4 % i4) # E: numpy.signedinteger[numpy.typing._32Bit] +reveal_type(f4 % i4) # E: numpy.floating[numpy.typing._32Bit] +reveal_type(AR % i8) # E: Union[numpy.ndarray, numpy.generic] + +reveal_type(divmod(b, i8)) # E: Tuple[numpy.signedinteger[numpy.typing._64Bit], numpy.signedinteger[numpy.typing._64Bit]] +reveal_type(divmod(i, i8)) # E: Tuple[numpy.signedinteger[Any], numpy.signedinteger[Any]] +reveal_type(divmod(f, i8)) # E: Tuple[numpy.floating[numpy.typing._64Bit], numpy.floating[numpy.typing._64Bit]] +reveal_type(divmod(i8, i8)) # E: Tuple[numpy.signedinteger[numpy.typing._64Bit], numpy.signedinteger[numpy.typing._64Bit]] +reveal_type(divmod(f8, i8)) # E: Tuple[numpy.floating[numpy.typing._64Bit], numpy.floating[numpy.typing._64Bit]] +reveal_type(divmod(i4, i8)) # E: Tuple[numpy.signedinteger[numpy.typing._64Bit], numpy.signedinteger[numpy.typing._64Bit]] +reveal_type(divmod(f4, i8)) # E: Tuple[numpy.floating[numpy.typing._64Bit], numpy.floating[numpy.typing._64Bit]] +reveal_type(divmod(i4, i4)) # E: Tuple[numpy.signedinteger[numpy.typing._32Bit], numpy.signedinteger[numpy.typing._32Bit]] +reveal_type(divmod(f4, i4)) # E: Tuple[numpy.floating[numpy.typing._32Bit], numpy.floating[numpy.typing._32Bit]] +reveal_type(divmod(AR, i8)) # E: Union[Tuple[numpy.ndarray, numpy.ndarray], Tuple[numpy.generic, numpy.generic]] + +# float + +reveal_type(f8 % b) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(f8 % i) # E: numpy.floating[Any] +reveal_type(f8 % f) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(i8 % f4) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(f4 % f4) # E: numpy.floating[numpy.typing._32Bit] +reveal_type(f8 % AR) # E: Union[numpy.ndarray, numpy.generic] + +reveal_type(divmod(f8, b)) # E: Tuple[numpy.floating[numpy.typing._64Bit], numpy.floating[numpy.typing._64Bit]] +reveal_type(divmod(f8, i)) # E: Tuple[numpy.floating[Any], numpy.floating[Any]] +reveal_type(divmod(f8, f)) # E: Tuple[numpy.floating[numpy.typing._64Bit], numpy.floating[numpy.typing._64Bit]] +reveal_type(divmod(f8, f8)) # E: Tuple[numpy.floating[numpy.typing._64Bit], numpy.floating[numpy.typing._64Bit]] +reveal_type(divmod(f8, f4)) # E: Tuple[numpy.floating[numpy.typing._64Bit], numpy.floating[numpy.typing._64Bit]] +reveal_type(divmod(f4, f4)) # E: Tuple[numpy.floating[numpy.typing._32Bit], numpy.floating[numpy.typing._32Bit]] +reveal_type(divmod(f8, AR)) # E: Union[Tuple[numpy.ndarray, numpy.ndarray], Tuple[numpy.generic, numpy.generic]] + +reveal_type(b % f8) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(i % f8) # E: numpy.floating[Any] +reveal_type(f % f8) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(f8 % f8) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(f8 % f8) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(f4 % f4) # E: numpy.floating[numpy.typing._32Bit] +reveal_type(AR % f8) # E: Union[numpy.ndarray, numpy.generic] + +reveal_type(divmod(b, f8)) # E: Tuple[numpy.floating[numpy.typing._64Bit], numpy.floating[numpy.typing._64Bit]] +reveal_type(divmod(i, f8)) # E: Tuple[numpy.floating[Any], numpy.floating[Any]] +reveal_type(divmod(f, f8)) # E: Tuple[numpy.floating[numpy.typing._64Bit], numpy.floating[numpy.typing._64Bit]] +reveal_type(divmod(f8, f8)) # E: Tuple[numpy.floating[numpy.typing._64Bit], numpy.floating[numpy.typing._64Bit]] +reveal_type(divmod(f4, f8)) # E: Tuple[numpy.floating[numpy.typing._64Bit], numpy.floating[numpy.typing._64Bit]] +reveal_type(divmod(f4, f4)) # E: Tuple[numpy.floating[numpy.typing._32Bit], numpy.floating[numpy.typing._32Bit]] +reveal_type(divmod(AR, f8)) # E: Union[Tuple[numpy.ndarray, numpy.ndarray], Tuple[numpy.generic, numpy.generic]] diff --git a/numpy/typing/tests/data/reveal/nbit_base_example.py b/numpy/typing/tests/data/reveal/nbit_base_example.py new file mode 100644 index 000000000000..0c4c53f9b1f1 --- /dev/null +++ b/numpy/typing/tests/data/reveal/nbit_base_example.py @@ -0,0 +1,18 @@ +from typing import TypeVar, Union +import numpy as np +import numpy.typing as npt + +T = TypeVar("T", bound=npt.NBitBase) + +def add(a: np.floating[T], b: np.integer[T]) -> np.floating[T]: + return a + b + +i8: np.int64 +i4: np.int32 +f8: np.float64 +f4: np.float32 + +reveal_type(add(f8, i8)) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(add(f4, i8)) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(add(f8, i4)) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(add(f4, i4)) # E: numpy.floating[numpy.typing._32Bit] diff --git a/numpy/typing/tests/data/reveal/ndarray_misc.py b/numpy/typing/tests/data/reveal/ndarray_misc.py new file mode 100644 index 000000000000..826c8aaa6d12 --- /dev/null +++ b/numpy/typing/tests/data/reveal/ndarray_misc.py @@ -0,0 +1,150 @@ +""" +Tests for miscellaneous (non-magic) ``np.ndarray``/``np.generic`` methods. + +More extensive tests are performed for the methods' +function-based counterpart in `../from_numeric.py`. + +""" + +import numpy as np + +class SubClass(np.ndarray): ... + +f8: np.float64 +A: np.ndarray +B: SubClass + +reveal_type(f8.all()) # E: numpy.bool_ +reveal_type(A.all()) # E: numpy.bool_ +reveal_type(A.all(axis=0)) # E: Union[numpy.bool_, numpy.ndarray] +reveal_type(A.all(keepdims=True)) # E: Union[numpy.bool_, numpy.ndarray] +reveal_type(A.all(out=B)) # E: SubClass + +reveal_type(f8.any()) # E: numpy.bool_ +reveal_type(A.any()) # E: numpy.bool_ +reveal_type(A.any(axis=0)) # E: Union[numpy.bool_, numpy.ndarray] +reveal_type(A.any(keepdims=True)) # E: Union[numpy.bool_, numpy.ndarray] +reveal_type(A.any(out=B)) # E: SubClass + +reveal_type(f8.argmax()) # E: numpy.signedinteger[Any] +reveal_type(A.argmax()) # E: numpy.signedinteger[Any] +reveal_type(A.argmax(axis=0)) # E: Union[numpy.signedinteger[Any], numpy.ndarray] +reveal_type(A.argmax(out=B)) # E: SubClass + +reveal_type(f8.argmin()) # E: numpy.signedinteger[Any] +reveal_type(A.argmin()) # E: numpy.signedinteger[Any] +reveal_type(A.argmin(axis=0)) # E: Union[numpy.signedinteger[Any], numpy.ndarray] +reveal_type(A.argmin(out=B)) # E: SubClass + +reveal_type(f8.argsort()) # E: numpy.ndarray +reveal_type(A.argsort()) # E: numpy.ndarray + +reveal_type(f8.astype(np.int64).choose([()])) # E: numpy.ndarray +reveal_type(A.choose([0])) # E: numpy.ndarray +reveal_type(A.choose([0], out=B)) # E: SubClass + +reveal_type(f8.clip(1)) # E: Union[numpy.number[Any], numpy.ndarray] +reveal_type(A.clip(1)) # E: Union[numpy.number[Any], numpy.ndarray] +reveal_type(A.clip(None, 1)) # E: Union[numpy.number[Any], numpy.ndarray] +reveal_type(A.clip(1, out=B)) # E: SubClass +reveal_type(A.clip(None, 1, out=B)) # E: SubClass + +reveal_type(f8.compress([0])) # E: numpy.ndarray +reveal_type(A.compress([0])) # E: numpy.ndarray +reveal_type(A.compress([0], out=B)) # E: SubClass + +reveal_type(f8.conj()) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(A.conj()) # E: numpy.ndarray +reveal_type(B.conj()) # E: SubClass + +reveal_type(f8.conjugate()) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(A.conjugate()) # E: numpy.ndarray +reveal_type(B.conjugate()) # E: SubClass + +reveal_type(f8.cumprod()) # E: numpy.ndarray +reveal_type(A.cumprod()) # E: numpy.ndarray +reveal_type(A.cumprod(out=B)) # E: SubClass + +reveal_type(f8.cumsum()) # E: numpy.ndarray +reveal_type(A.cumsum()) # E: numpy.ndarray +reveal_type(A.cumsum(out=B)) # E: SubClass + +reveal_type(f8.max()) # E: numpy.number[Any] +reveal_type(A.max()) # E: numpy.number[Any] +reveal_type(A.max(axis=0)) # E: Union[numpy.number[Any], numpy.ndarray] +reveal_type(A.max(keepdims=True)) # E: Union[numpy.number[Any], numpy.ndarray] +reveal_type(A.max(out=B)) # E: SubClass + +reveal_type(f8.mean()) # E: numpy.number[Any] +reveal_type(A.mean()) # E: numpy.number[Any] +reveal_type(A.mean(axis=0)) # E: Union[numpy.number[Any], numpy.ndarray] +reveal_type(A.mean(keepdims=True)) # E: Union[numpy.number[Any], numpy.ndarray] +reveal_type(A.mean(out=B)) # E: SubClass + +reveal_type(f8.min()) # E: numpy.number[Any] +reveal_type(A.min()) # E: numpy.number[Any] +reveal_type(A.min(axis=0)) # E: Union[numpy.number[Any], numpy.ndarray] +reveal_type(A.min(keepdims=True)) # E: Union[numpy.number[Any], numpy.ndarray] +reveal_type(A.min(out=B)) # E: SubClass + +reveal_type(f8.newbyteorder()) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(A.newbyteorder()) # E: numpy.ndarray +reveal_type(B.newbyteorder('|')) # E: SubClass + +reveal_type(f8.prod()) # E: numpy.number[Any] +reveal_type(A.prod()) # E: numpy.number[Any] +reveal_type(A.prod(axis=0)) # E: Union[numpy.number[Any], numpy.ndarray] +reveal_type(A.prod(keepdims=True)) # E: Union[numpy.number[Any], numpy.ndarray] +reveal_type(A.prod(out=B)) # E: SubClass + +reveal_type(f8.ptp()) # E: numpy.number[Any] +reveal_type(A.ptp()) # E: numpy.number[Any] +reveal_type(A.ptp(axis=0)) # E: Union[numpy.number[Any], numpy.ndarray] +reveal_type(A.ptp(keepdims=True)) # E: Union[numpy.number[Any], numpy.ndarray] +reveal_type(A.ptp(out=B)) # E: SubClass + +reveal_type(f8.round()) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(A.round()) # E: numpy.ndarray +reveal_type(A.round(out=B)) # E: SubClass + +reveal_type(f8.repeat(1)) # E: numpy.ndarray +reveal_type(A.repeat(1)) # E: numpy.ndarray +reveal_type(B.repeat(1)) # E: numpy.ndarray + +reveal_type(f8.std()) # E: numpy.number[Any] +reveal_type(A.std()) # E: numpy.number[Any] +reveal_type(A.std(axis=0)) # E: Union[numpy.number[Any], numpy.ndarray] +reveal_type(A.std(keepdims=True)) # E: Union[numpy.number[Any], numpy.ndarray] +reveal_type(A.std(out=B)) # E: SubClass + +reveal_type(f8.sum()) # E: numpy.number[Any] +reveal_type(A.sum()) # E: numpy.number[Any] +reveal_type(A.sum(axis=0)) # E: Union[numpy.number[Any], numpy.ndarray] +reveal_type(A.sum(keepdims=True)) # E: Union[numpy.number[Any], numpy.ndarray] +reveal_type(A.sum(out=B)) # E: SubClass + +reveal_type(f8.take(0)) # E: numpy.generic +reveal_type(A.take(0)) # E: numpy.generic +reveal_type(A.take([0])) # E: numpy.ndarray +reveal_type(A.take(0, out=B)) # E: SubClass +reveal_type(A.take([0], out=B)) # E: SubClass + +reveal_type(f8.var()) # E: numpy.number[Any] +reveal_type(A.var()) # E: numpy.number[Any] +reveal_type(A.var(axis=0)) # E: Union[numpy.number[Any], numpy.ndarray] +reveal_type(A.var(keepdims=True)) # E: Union[numpy.number[Any], numpy.ndarray] +reveal_type(A.var(out=B)) # E: SubClass + +reveal_type(A.argpartition([0])) # E: numpy.ndarray + +reveal_type(A.diagonal()) # E: numpy.ndarray + +reveal_type(A.dot(1)) # E: Union[numpy.number[Any], numpy.ndarray] +reveal_type(A.dot(1, out=B)) # E: SubClass + +reveal_type(A.nonzero()) # E: tuple[numpy.ndarray] + +reveal_type(A.searchsorted([1])) # E: numpy.ndarray + +reveal_type(A.trace()) # E: Union[numpy.number[Any], numpy.ndarray] +reveal_type(A.trace(out=B)) # E: SubClass diff --git a/numpy/typing/tests/data/reveal/numeric.py b/numpy/typing/tests/data/reveal/numeric.py new file mode 100644 index 000000000000..5cbfa4ac765c --- /dev/null +++ b/numpy/typing/tests/data/reveal/numeric.py @@ -0,0 +1,89 @@ +""" +Tests for :mod:`numpy.core.numeric`. + +Does not include tests which fall under ``array_constructors``. + +""" + +from typing import List +import numpy as np + +class SubClass(np.ndarray): + ... + +i8: np.int64 + +A: np.ndarray +B: List[int] +C: SubClass + +reveal_type(np.count_nonzero(i8)) # E: int +reveal_type(np.count_nonzero(A)) # E: int +reveal_type(np.count_nonzero(B)) # E: int +reveal_type(np.count_nonzero(A, keepdims=True)) # E: Union[numpy.signedinteger[Any], numpy.ndarray] +reveal_type(np.count_nonzero(A, axis=0)) # E: Union[numpy.signedinteger[Any], numpy.ndarray] + +reveal_type(np.isfortran(i8)) # E: bool +reveal_type(np.isfortran(A)) # E: bool + +reveal_type(np.argwhere(i8)) # E: numpy.ndarray +reveal_type(np.argwhere(A)) # E: numpy.ndarray + +reveal_type(np.flatnonzero(i8)) # E: numpy.ndarray +reveal_type(np.flatnonzero(A)) # E: numpy.ndarray + +reveal_type(np.correlate(B, A, mode="valid")) # E: numpy.ndarray +reveal_type(np.correlate(A, A, mode="same")) # E: numpy.ndarray + +reveal_type(np.convolve(B, A, mode="valid")) # E: numpy.ndarray +reveal_type(np.convolve(A, A, mode="same")) # E: numpy.ndarray + +reveal_type(np.outer(i8, A)) # E: numpy.ndarray +reveal_type(np.outer(B, A)) # E: numpy.ndarray +reveal_type(np.outer(A, A)) # E: numpy.ndarray +reveal_type(np.outer(A, A, out=C)) # E: SubClass + +reveal_type(np.tensordot(B, A)) # E: numpy.ndarray +reveal_type(np.tensordot(A, A)) # E: numpy.ndarray +reveal_type(np.tensordot(A, A, axes=0)) # E: numpy.ndarray +reveal_type(np.tensordot(A, A, axes=(0, 1))) # E: numpy.ndarray + +reveal_type(np.isscalar(i8)) # E: bool +reveal_type(np.isscalar(A)) # E: bool +reveal_type(np.isscalar(B)) # E: bool + +reveal_type(np.roll(A, 1)) # E: numpy.ndarray +reveal_type(np.roll(A, (1, 2))) # E: numpy.ndarray +reveal_type(np.roll(B, 1)) # E: numpy.ndarray + +reveal_type(np.rollaxis(A, 0, 1)) # E: numpy.ndarray + +reveal_type(np.moveaxis(A, 0, 1)) # E: numpy.ndarray +reveal_type(np.moveaxis(A, (0, 1), (1, 2))) # E: numpy.ndarray + +reveal_type(np.cross(B, A)) # E: numpy.ndarray +reveal_type(np.cross(A, A)) # E: numpy.ndarray + +reveal_type(np.indices([0, 1, 2])) # E: numpy.ndarray +reveal_type(np.indices([0, 1, 2], sparse=False)) # E: numpy.ndarray +reveal_type(np.indices([0, 1, 2], sparse=True)) # E: tuple[numpy.ndarray] + +reveal_type(np.binary_repr(1)) # E: str + +reveal_type(np.base_repr(1)) # E: str + +reveal_type(np.allclose(i8, A)) # E: bool +reveal_type(np.allclose(B, A)) # E: bool +reveal_type(np.allclose(A, A)) # E: bool + +reveal_type(np.isclose(i8, A)) # E: Union[numpy.bool_, numpy.ndarray] +reveal_type(np.isclose(B, A)) # E: Union[numpy.bool_, numpy.ndarray] +reveal_type(np.isclose(A, A)) # E: Union[numpy.bool_, numpy.ndarray] + +reveal_type(np.array_equal(i8, A)) # E: bool +reveal_type(np.array_equal(B, A)) # E: bool +reveal_type(np.array_equal(A, A)) # E: bool + +reveal_type(np.array_equiv(i8, A)) # E: bool +reveal_type(np.array_equiv(B, A)) # E: bool +reveal_type(np.array_equiv(A, A)) # E: bool diff --git a/numpy/typing/tests/data/reveal/scalars.py b/numpy/typing/tests/data/reveal/scalars.py index 882fe96128d6..72a3e4d2e628 100644 --- a/numpy/typing/tests/data/reveal/scalars.py +++ b/numpy/typing/tests/data/reveal/scalars.py @@ -2,32 +2,18 @@ x = np.complex64(3 + 2j) -reveal_type(x.real) # E: numpy.float32 -reveal_type(x.imag) # E: numpy.float32 +reveal_type(x.real) # E: numpy.floating[numpy.typing._32Bit] +reveal_type(x.imag) # E: numpy.floating[numpy.typing._32Bit] -reveal_type(x.real.real) # E: numpy.float32 -reveal_type(x.real.imag) # E: numpy.float32 +reveal_type(x.real.real) # E: numpy.floating[numpy.typing._32Bit] +reveal_type(x.real.imag) # E: numpy.floating[numpy.typing._32Bit] reveal_type(x.itemsize) # E: int reveal_type(x.shape) # E: tuple[builtins.int] reveal_type(x.strides) # E: tuple[builtins.int] -# Time structures -dt = np.datetime64(0, "D") -td = np.timedelta64(0, "D") +reveal_type(np.complex64().real) # E: numpy.floating[numpy.typing._32Bit] +reveal_type(np.complex128().imag) # E: numpy.floating[numpy.typing._64Bit] -reveal_type(dt + td) # E: numpy.datetime64 -reveal_type(dt + 1) # E: numpy.datetime64 -reveal_type(dt - dt) # E: numpy.timedelta64 -reveal_type(dt - 1) # E: numpy.timedelta64 - -reveal_type(td + td) # E: numpy.timedelta64 -reveal_type(td + 1) # E: numpy.timedelta64 -reveal_type(td - td) # E: numpy.timedelta64 -reveal_type(td - 1) # E: numpy.timedelta64 -reveal_type(td / 1.0) # E: numpy.timedelta64 -reveal_type(td / td) # E: float -reveal_type(td % td) # E: numpy.timedelta64 - -reveal_type(np.complex64().real) # E: numpy.float32 -reveal_type(np.complex128().imag) # E: numpy.float64 +reveal_type(np.unicode_('foo')) # E: numpy.str_ +reveal_type(np.str0('foo')) # E: numpy.str_ diff --git a/numpy/typing/tests/test_isfile.py b/numpy/typing/tests/test_isfile.py new file mode 100644 index 000000000000..e6b80f995919 --- /dev/null +++ b/numpy/typing/tests/test_isfile.py @@ -0,0 +1,34 @@ +import os +from pathlib import Path + +import numpy as np +from numpy.testing import assert_ + +ROOT = Path(np.__file__).parents[0] +FILES = [ + ROOT / "py.typed", + ROOT / "__init__.pyi", + ROOT / "char.pyi", + ROOT / "ctypeslib.pyi", + ROOT / "emath.pyi", + ROOT / "rec.pyi", + ROOT / "version.pyi", + ROOT / "core" / "__init__.pyi", + ROOT / "distutils" / "__init__.pyi", + ROOT / "f2py" / "__init__.pyi", + ROOT / "fft" / "__init__.pyi", + ROOT / "lib" / "__init__.pyi", + ROOT / "linalg" / "__init__.pyi", + ROOT / "ma" / "__init__.pyi", + ROOT / "matrixlib" / "__init__.pyi", + ROOT / "polynomial" / "__init__.pyi", + ROOT / "random" / "__init__.pyi", + ROOT / "testing" / "__init__.pyi", +] + + +class TestIsFile: + def test_isfile(self): + """Test if all ``.pyi`` files are properly installed.""" + for file in FILES: + assert_(os.path.isfile(file)) diff --git a/numpy/typing/tests/test_typing.py b/numpy/typing/tests/test_typing.py index beb53ddecefa..90de4fd6dc41 100644 --- a/numpy/typing/tests/test_typing.py +++ b/numpy/typing/tests/test_typing.py @@ -3,6 +3,7 @@ import os import re from collections import defaultdict +from typing import Optional import pytest try: @@ -36,6 +37,7 @@ def get_test_cases(directory): ) +@pytest.mark.slow @pytest.mark.skipif(NO_MYPY, reason="Mypy is not installed") @pytest.mark.parametrize("path", get_test_cases(PASS_DIR)) def test_success(path): @@ -50,9 +52,12 @@ def test_success(path): assert re.match(r"Success: no issues found in \d+ source files?", stdout.strip()) +@pytest.mark.slow @pytest.mark.skipif(NO_MYPY, reason="Mypy is not installed") @pytest.mark.parametrize("path", get_test_cases(FAIL_DIR)) def test_fail(path): + __tracebackhide__ = True + stdout, stderr, exitcode = api.run([ "--config-file", MYPY_INI, @@ -93,15 +98,37 @@ def test_fail(path): target_line = lines[lineno - 1] if "# E:" in target_line: marker = target_line.split("# E:")[-1].strip() - assert lineno in errors, f'Extra error "{marker}"' - assert marker in errors[lineno] + expected_error = errors.get(lineno) + _test_fail(path, marker, expected_error, lineno) else: pytest.fail(f"Error {repr(errors[lineno])} not found") +_FAIL_MSG1 = """Extra error at line {} + +Extra error: {!r} +""" + +_FAIL_MSG2 = """Error mismatch at line {} + +Expected error: {!r} +Observed error: {!r} +""" + + +def _test_fail(path: str, error: str, expected_error: Optional[str], lineno: int) -> None: + if expected_error is None: + raise AssertionError(_FAIL_MSG1.format(lineno, error)) + elif error not in expected_error: + raise AssertionError(_FAIL_MSG2.format(lineno, expected_error, error)) + + +@pytest.mark.slow @pytest.mark.skipif(NO_MYPY, reason="Mypy is not installed") @pytest.mark.parametrize("path", get_test_cases(REVEAL_DIR)) def test_reveal(path): + __tracebackhide__ = True + stdout, stderr, exitcode = api.run([ "--config-file", MYPY_INI, @@ -111,9 +138,10 @@ def test_reveal(path): ]) with open(path) as fin: - lines = fin.readlines() + lines = fin.read().replace('*', '').split("\n") - for error_line in stdout.split("\n"): + stdout_list = stdout.replace('*', '').split("\n") + for error_line in stdout_list: error_line = error_line.strip() if not error_line: continue @@ -124,12 +152,26 @@ def test_reveal(path): ) if match is None: raise ValueError(f"Unexpected reveal line format: {error_line}") - lineno = int(match.group('lineno')) + lineno = int(match.group('lineno')) - 1 assert "Revealed type is" in error_line - marker = lines[lineno - 1].split("# E:")[-1].strip() - assert marker in error_line + + marker = lines[lineno].split("# E:")[-1].strip() + _test_reveal(path, marker, error_line, 1 + lineno) + + +_REVEAL_MSG = """Reveal mismatch at line {} + +Expected reveal: {!r} +Observed reveal: {!r} +""" + + +def _test_reveal(path: str, reveal: str, expected_reveal: str, lineno: int) -> None: + if reveal not in expected_reveal: + raise AssertionError(_REVEAL_MSG.format(lineno, expected_reveal, reveal)) +@pytest.mark.slow @pytest.mark.skipif(NO_MYPY, reason="Mypy is not installed") @pytest.mark.parametrize("path", get_test_cases(PASS_DIR)) def test_code_runs(path): diff --git a/release_requirements.txt b/release_requirements.txt new file mode 100644 index 000000000000..dc840b9359ec --- /dev/null +++ b/release_requirements.txt @@ -0,0 +1,10 @@ +# These packages are needed for a release in addition to those needed +# for building, testing, and the creation of documentation. + +# download-wheels.py +urllib3 +beatifulsoup4 + +# changelog.py +pygithub +gitpython diff --git a/runtests.py b/runtests.py index 2f07749f8147..87e26768b351 100755 --- a/runtests.py +++ b/runtests.py @@ -52,7 +52,7 @@ import sys -import os +import os, glob # In case we are run from the source directory, we don't want to import the # project from there: @@ -122,6 +122,9 @@ def main(argv): help="Specify a list of dispatched CPU optimizations"), parser.add_argument("--disable-optimization", action="store_true", help="Disable CPU optimized code(dispatch,simd,fast...)"), + parser.add_argument("--simd-test", default=None, + help="Specify a list of CPU optimizations to be " + "tested against NumPy SIMD interface"), parser.add_argument("--show-build-log", action="store_true", help="Show build output rather than using a log file") parser.add_argument("--bench", action="store_true", @@ -310,8 +313,16 @@ def main(argv): out = subprocess.check_output(['git', 'rev-parse', commit_a]) commit_a = out.strip().decode('ascii') + # generate config file with the required build options + asv_cfpath = [ + '--config', asv_compare_config( + os.path.join(ROOT_DIR, 'benchmarks'), args, + # to clear the cache if the user changed build options + (commit_a, commit_b) + ) + ] cmd = ['asv', 'continuous', '-e', '-f', '1.05', - commit_a, commit_b] + bench_args + commit_a, commit_b] + asv_cfpath + bench_args ret = subprocess.call(cmd, cwd=os.path.join(ROOT_DIR, 'benchmarks')) sys.exit(ret) @@ -361,7 +372,6 @@ def main(argv): else: sys.exit(1) - def build_project(args): """ Build a dev version of the project. @@ -432,6 +442,8 @@ def build_project(args): cmd += ["--cpu-dispatch", args.cpu_dispatch] if args.disable_optimization: cmd += ["--disable-optimization"] + if args.simd_test is not None: + cmd += ["--simd-test", args.simd_test] # Install; avoid producing eggs so numpy can be imported from dst_dir. cmd += ['install', '--prefix=' + dst_dir, '--single-version-externally-managed', @@ -491,6 +503,98 @@ def build_project(args): return site_dir, site_dir_noarch +def asv_compare_config(bench_path, args, h_commits): + """ + Fill the required build options through custom variable + 'numpy_build_options' and return the generated config path. + """ + conf_path = os.path.join(bench_path, "asv_compare.conf.json.tpl") + nconf_path = os.path.join(bench_path, "_asv_compare.conf.json") + + # add custom build + build = [] + if args.parallel > 1: + build += ["-j", str(args.parallel)] + if args.cpu_baseline: + build += ["--cpu-baseline", args.cpu_baseline] + if args.cpu_dispatch: + build += ["--cpu-dispatch", args.cpu_dispatch] + if args.disable_optimization: + build += ["--disable-optimization"] + + is_cached = asv_substitute_config(conf_path, nconf_path, + numpy_build_options = ' '.join([f'\\"{v}\\"' for v in build]), + ) + if not is_cached: + asv_clear_cache(bench_path, h_commits) + return nconf_path + +def asv_clear_cache(bench_path, h_commits, env_dir="env"): + """ + Force ASV to clear the cache according to specified commit hashes. + """ + # FIXME: only clear the cache from the current environment dir + asv_build_pattern = os.path.join(bench_path, env_dir, "*", "asv-build-cache") + for asv_build_cache in glob.glob(asv_build_pattern, recursive=True): + for c in h_commits: + try: shutil.rmtree(os.path.join(asv_build_cache, c)) + except OSError: pass + +def asv_substitute_config(in_config, out_config, **custom_vars): + """ + A workaround to allow substituting custom tokens within + ASV configuration file since there's no official way to add custom + variables(e.g. env vars). + + Parameters + ---------- + in_config : str + The path of ASV configuration file, e.g. '/path/to/asv.conf.json' + out_config : str + The path of generated configuration file, + e.g. '/path/to/asv_substituted.conf.json'. + + The other keyword arguments represent the custom variables. + + Returns + ------- + True(is cached) if 'out_config' is already generated with + the same '**custom_vars' and updated with latest 'in_config', + False otherwise. + + Examples + -------- + See asv_compare_config(). + """ + assert in_config != out_config + assert len(custom_vars) > 0 + + def sdbm_hash(*factors): + chash = 0 + for f in factors: + for char in str(f): + chash = ord(char) + (chash << 6) + (chash << 16) - chash + chash &= 0xFFFFFFFF + return chash + + vars_hash = sdbm_hash(custom_vars, os.path.getmtime(in_config)) + try: + with open(out_config, "r") as wfd: + hash_line = wfd.readline().split('hash:') + if len(hash_line) > 1 and int(hash_line[1]) == vars_hash: + return True + except IOError: + pass + + custom_vars = {f'{{{k}}}':v for k, v in custom_vars.items()} + with open(in_config, "r") as rfd, open(out_config, "w") as wfd: + wfd.write(f"// hash:{vars_hash}\n") + wfd.write("// This file is automatically generated by runtests.py\n") + for line in rfd: + for key, val in custom_vars.items(): + line = line.replace(key, val) + wfd.write(line) + return False # # GCOV support diff --git a/setup.py b/setup.py index 1f5212676310..6d7e90c6a15a 100755 --- a/setup.py +++ b/setup.py @@ -23,7 +23,7 @@ import sys import subprocess import textwrap -import sysconfig +import warnings if sys.version_info[:2] < (3, 6): @@ -36,17 +36,19 @@ Development Status :: 5 - Production/Stable Intended Audience :: Science/Research Intended Audience :: Developers -License :: OSI Approved +License :: OSI Approved :: BSD License Programming Language :: C Programming Language :: Python Programming Language :: Python :: 3 Programming Language :: Python :: 3.6 Programming Language :: Python :: 3.7 Programming Language :: Python :: 3.8 +Programming Language :: Python :: 3.9 Programming Language :: Python :: 3 :: Only Programming Language :: Python :: Implementation :: CPython Topic :: Software Development Topic :: Scientific/Engineering +Typing :: Typed Operating System :: Microsoft :: Windows Operating System :: POSIX Operating System :: Unix @@ -59,6 +61,14 @@ ISRELEASED = False VERSION = '%d.%d.%d' % (MAJOR, MINOR, MICRO) +# The first version not in the `Programming Language :: Python :: ...` classifiers above +if sys.version_info >= (3, 10): + warnings.warn( + f"NumPy {VERSION} may not yet support Python " + f"{sys.version_info.major}.{sys.version_info.minor}.", + RuntimeWarning, + ) + # Return the git revision as a string def git_version(): @@ -88,6 +98,7 @@ def _minimal_ext_cmd(cmd): return GIT_REVISION + # BEFORE importing setuptools, remove MANIFEST. Otherwise it may not be # properly updated when the contents of directories change (true for distutils, # not sure about setuptools). @@ -150,7 +161,7 @@ def write_version_py(filename='numpy/version.py'): a.close() -def configuration(parent_package='',top_path=None): +def configuration(parent_package='', top_path=None): from numpy.distutils.misc_util import Configuration config = Configuration(None, parent_package, top_path) @@ -163,7 +174,7 @@ def configuration(parent_package='',top_path=None): config.add_data_files(('numpy', 'LICENSE.txt')) config.add_data_files(('numpy', 'numpy/*.pxd')) - config.get_version('numpy/version.py') # sets config.version + config.get_version('numpy/version.py') # sets config.version return config @@ -175,13 +186,12 @@ def check_submodules(): if not os.path.exists('.git'): return with open('.gitmodules') as f: - for l in f: - if 'path' in l: - p = l.split('=')[-1].strip() + for line in f: + if 'path' in line: + p = line.split('=')[-1].strip() if not os.path.exists(p): raise ValueError('Submodule {} missing'.format(p)) - proc = subprocess.Popen(['git', 'submodule', 'status'], stdout=subprocess.PIPE) status, _ = proc.communicate() @@ -273,9 +283,9 @@ def generate_cython(): print("Cythonizing sources") for d in ('random',): p = subprocess.call([sys.executable, - os.path.join(cwd, 'tools', 'cythonize.py'), - 'numpy/{0}'.format(d)], - cwd=cwd) + os.path.join(cwd, 'tools', 'cythonize.py'), + 'numpy/{0}'.format(d)], + cwd=cwd) if p != 0: raise RuntimeError("Running cythonize failed!") @@ -346,7 +356,6 @@ def parse_setuppy_commands(): """)) return False - # The following commands aren't supported. They can only be executed when # the user explicitly adds a --force command-line argument. bad_commands = dict( @@ -384,8 +393,8 @@ def parse_setuppy_commands(): ) bad_commands['nosetests'] = bad_commands['test'] for command in ('upload_docs', 'easy_install', 'bdist', 'bdist_dumb', - 'register', 'check', 'install_data', 'install_headers', - 'install_lib', 'install_scripts', ): + 'register', 'check', 'install_data', 'install_headers', + 'install_lib', 'install_scripts', ): bad_commands[command] = "`setup.py %s` is not supported" % command for command in bad_commands.keys(): @@ -405,7 +414,8 @@ def parse_setuppy_commands(): # If we got here, we didn't detect what setup.py command was given import warnings warnings.warn("Unrecognized setuptools command, proceeding with " - "generating Cython sources and expanding templates", stacklevel=2) + "generating Cython sources and expanding templates", + stacklevel=2) return True @@ -413,7 +423,7 @@ def get_docs_url(): if not ISRELEASED: return "https://numpy.org/devdocs" else: - # For releaeses, this URL ends up on pypi. + # For releases, this URL ends up on pypi. # By pinning the version, users looking at old PyPI releases can get # to the associated docs easily. return "https://numpy.org/doc/{}.{}".format(MAJOR, MINOR) @@ -440,25 +450,24 @@ def setup_package(): 'f2py%s.%s = numpy.f2py.f2py2e:main' % sys.version_info[:2], ] - cmdclass={"sdist": sdist_checked, - } + cmdclass = {"sdist": sdist_checked, } metadata = dict( - name = 'numpy', - maintainer = "NumPy Developers", - maintainer_email = "numpy-discussion@python.org", - description = DOCLINES[0], - long_description = "\n".join(DOCLINES[2:]), - url = "https://www.numpy.org", - author = "Travis E. Oliphant et al.", - download_url = "https://pypi.python.org/pypi/numpy", + name='numpy', + maintainer="NumPy Developers", + maintainer_email="numpy-discussion@python.org", + description=DOCLINES[0], + long_description="\n".join(DOCLINES[2:]), + url="https://www.numpy.org", + author="Travis E. Oliphant et al.", + download_url="https://pypi.python.org/pypi/numpy", project_urls={ "Bug Tracker": "https://github.com/numpy/numpy/issues", "Documentation": get_docs_url(), "Source Code": "https://github.com/numpy/numpy", }, - license = 'BSD', + license='BSD', classifiers=[_f for _f in CLASSIFIERS.split('\n') if _f], - platforms = ["Windows", "Linux", "Solaris", "Mac OS-X", "Unix"], + platforms=["Windows", "Linux", "Solaris", "Mac OS-X", "Unix"], test_suite='pytest', cmdclass=cmdclass, python_requires='>=3.6', @@ -479,8 +488,7 @@ def setup_package(): # patches distutils, even though we don't use it import setuptools # noqa: F401 from numpy.distutils.core import setup - cwd = os.path.abspath(os.path.dirname(__file__)) - if not 'sdist' in sys.argv: + if 'sdist' not in sys.argv: # Generate Cython sources, unless we're generating an sdist generate_cython() diff --git a/test_requirements.txt b/test_requirements.txt index 7ef91125c4e4..8b17ccc133f1 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -1,9 +1,9 @@ cython==0.29.21 wheel setuptools<49.2.0 -hypothesis==5.33.0 -pytest==6.0.1 -pytz==2020.1 +hypothesis==5.41.0 +pytest==6.0.2 +pytz==2020.4 pytest-cov==2.10.1 pickle5; python_version == '3.7' pickle5; python_version == '3.6' and platform_python_implementation != 'PyPy' @@ -13,5 +13,5 @@ cffi # - Mypy relies on C API features not present in PyPy # - Mypy doesn't currently work on Python 3.9 # - Python 3.6 doesn't work because it doesn't understand py.typed -mypy==0.782; platform_python_implementation != "PyPy" and python_version > "3.6" +mypy==0.790; platform_python_implementation != "PyPy" and python_version > "3.6" typing_extensions diff --git a/tools/changelog.py b/tools/changelog.py index 92f33af24453..920f5b87faa3 100755 --- a/tools/changelog.py +++ b/tools/changelog.py @@ -45,16 +45,17 @@ this_repo = Repo(os.path.join(os.path.dirname(__file__), "..")) author_msg =\ -u""" +""" A total of %d people contributed to this release. People with a "+" by their names contributed a patch for the first time. """ pull_request_msg =\ -u""" +""" A total of %d pull requests were merged for this release. """ + def get_authors(revision_range): pat = u'^.*\\t(.*)$' lst_release, cur_release = [r.strip() for r in revision_range.split('..')] diff --git a/tools/download-wheels.py b/tools/download-wheels.py index 941440ca9afc..28b3fc7ad6a9 100644 --- a/tools/download-wheels.py +++ b/tools/download-wheels.py @@ -1,9 +1,29 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 +# -*- encoding:utf-8 -*- """ -Download NumPy wheels from Anaconda staging area. +Script to download NumPy wheels from the Anaconda staging area. + +Usage:: + + $ ./tools/download-wheels.py -w + +The default wheelhouse is ``release/installers``. + +Dependencies +------------ + +- beautifulsoup4 +- urllib3 + +Examples +-------- + +While in the repository root:: + + $ python tools/download-wheels.py 1.19.0 + $ python tools/download-wheels.py 1.19.0 -w ~/wheelhouse """ -import sys import os import re import shutil @@ -18,6 +38,7 @@ STAGING_URL = 'https://anaconda.org/multibuild-wheels-staging/numpy' PREFIX = 'numpy' + def get_wheel_names(version): """ Get wheel names from Anaconda HTML directory. @@ -69,7 +90,7 @@ def download_wheels(version, wheelhouse): parser = argparse.ArgumentParser() parser.add_argument( "version", - help="NumPy version to download.") + help="NumPy version to download.") parser.add_argument( "-w", "--wheelhouse", default=os.path.join(os.getcwd(), "release", "installers"), diff --git a/tools/functions_missing_types.py b/tools/functions_missing_types.py index a32e72dad8b1..c2fe156f0716 100755 --- a/tools/functions_missing_types.py +++ b/tools/functions_missing_types.py @@ -27,6 +27,9 @@ "division", "print_function", "warnings", + "sys", + "os", + "math", # Accidentally public, deprecated, or shouldn't be used "Tester", "alen", @@ -34,6 +37,7 @@ "add_newdoc", "add_newdoc_ufunc", "core", + "compat", "fastCopyAndTranspose", "get_array_wrap", "int_asbuffer", diff --git a/tools/openblas_support.py b/tools/openblas_support.py index 46803742c986..3d966a8f601a 100644 --- a/tools/openblas_support.py +++ b/tools/openblas_support.py @@ -12,9 +12,9 @@ from urllib.request import urlopen, Request from urllib.error import HTTPError -OPENBLAS_V = '0.3.10' +OPENBLAS_V = '0.3.12' # Temporary build of OpenBLAS to test a fix for dynamic detection of CPU -OPENBLAS_LONG = 'v0.3.9-452-g349b722d' +OPENBLAS_LONG = 'v0.3.12' BASE_LOC = 'https://anaconda.org/multibuild-wheels-staging/openblas-libs' BASEURL = f'{BASE_LOC}/{OPENBLAS_LONG}/download' ARCHITECTURES = ['', 'windows', 'darwin', 'aarch64', 'x86_64', @@ -143,9 +143,9 @@ def download_openblas(target, arch, ilp64, is_32bit): typ = 'tar.gz' elif arch == 'windows': if is_32bit: - suffix = 'win32-gcc_7_1_0.zip' + suffix = 'win32-gcc_8_1_0.zip' else: - suffix = 'win_amd64-gcc_7_1_0.zip' + suffix = 'win_amd64-gcc_8_1_0.zip' filename = f'{BASEURL}/openblas{fnsuffix}-{OPENBLAS_LONG}-{suffix}' typ = 'zip' if not filename: diff --git a/tools/pypy-test.sh b/tools/pypy-test.sh index 32b7968d8e9a..e6c6ae719c91 100755 --- a/tools/pypy-test.sh +++ b/tools/pypy-test.sh @@ -28,8 +28,8 @@ include_dirs = $target/lib:$LIB runtime_library_dirs = $target/lib EOF -echo getting PyPy 3.6-v7.3.1 -wget -q https://downloads.python.org/pypy/pypy3.6-v7.3.1-linux64.tar.bz2 -O pypy.tar.bz2 +echo getting PyPy 3.6-v7.3.2 +wget -q https://downloads.python.org/pypy/pypy3.6-v7.3.2-linux64.tar.bz2 -O pypy.tar.bz2 mkdir -p pypy3 (cd pypy3; tar --strip-components=1 -xf ../pypy.tar.bz2) pypy3/bin/pypy3 -mensurepip diff --git a/tools/swig/numpy.i b/tools/swig/numpy.i index 6b69ce96e0ed..60930c098330 100644 --- a/tools/swig/numpy.i +++ b/tools/swig/numpy.i @@ -115,7 +115,7 @@ if (py_obj == Py_None ) return "Python None" ; if (PyCallable_Check(py_obj)) return "callable" ; if (PyBytes_Check( py_obj)) return "string" ; - if (PyInt_Check( py_obj)) return "int" ; + if (PyLong_Check( py_obj)) return "int" ; if (PyFloat_Check( py_obj)) return "float" ; if (PyDict_Check( py_obj)) return "dict" ; if (PyList_Check( py_obj)) return "list" ; @@ -2002,7 +2002,7 @@ (PyObject* array = NULL) { npy_intp dims[1]; - if (!PyInt_Check($input)) + if (!PyLong_Check($input)) { const char* typestring = pytype_string($input); PyErr_Format(PyExc_TypeError, @@ -2010,7 +2010,8 @@ typestring); SWIG_fail; } - $2 = (DIM_TYPE) PyInt_AsLong($input); + $2 = (DIM_TYPE) PyLong_AsSsize_t($input); + if ($2 == -1 && PyErr_Occurred()) SWIG_fail; dims[0] = (npy_intp) $2; array = PyArray_SimpleNew(1, dims, DATA_TYPECODE); if (!array) SWIG_fail; @@ -2030,7 +2031,7 @@ (PyObject* array = NULL) { npy_intp dims[1]; - if (!PyInt_Check($input)) + if (!PyLong_Check($input)) { const char* typestring = pytype_string($input); PyErr_Format(PyExc_TypeError, @@ -2038,7 +2039,8 @@ typestring); SWIG_fail; } - $1 = (DIM_TYPE) PyInt_AsLong($input); + $1 = (DIM_TYPE) PyLong_AsSsize_t($input); + if ($1 == -1 && PyErr_Occurred()) SWIG_fail; dims[0] = (npy_intp) $1; array = PyArray_SimpleNew(1, dims, DATA_TYPECODE); if (!array) SWIG_fail; @@ -2492,9 +2494,9 @@ if (!array) SWIG_fail; %#ifdef SWIGPY_USE_CAPSULE - PyObject* cap = PyCapsule_New((void*)(*$1), SWIGPY_CAPSULE_NAME, free_cap); + PyObject* cap = PyCapsule_New((void*)(*$2), SWIGPY_CAPSULE_NAME, free_cap); %#else - PyObject* cap = PyCObject_FromVoidPtr((void*)(*$1), free); + PyObject* cap = PyCObject_FromVoidPtr((void*)(*$2), free); %#endif %#if NPY_API_VERSION < 0x00000007 @@ -2562,9 +2564,9 @@ if (!array) SWIG_fail; %#ifdef SWIGPY_USE_CAPSULE - PyObject* cap = PyCapsule_New((void*)(*$1), SWIGPY_CAPSULE_NAME, free_cap); + PyObject* cap = PyCapsule_New((void*)(*$3), SWIGPY_CAPSULE_NAME, free_cap); %#else - PyObject* cap = PyCObject_FromVoidPtr((void*)(*$1), free); + PyObject* cap = PyCObject_FromVoidPtr((void*)(*$3), free); %#endif %#if NPY_API_VERSION < 0x00000007 @@ -2632,9 +2634,9 @@ if (!array || !require_fortran(array)) SWIG_fail; %#ifdef SWIGPY_USE_CAPSULE - PyObject* cap = PyCapsule_New((void*)(*$1), SWIGPY_CAPSULE_NAME, free_cap); + PyObject* cap = PyCapsule_New((void*)(*$3), SWIGPY_CAPSULE_NAME, free_cap); %#else - PyObject* cap = PyCObject_FromVoidPtr((void*)(*$1), free); + PyObject* cap = PyCObject_FromVoidPtr((void*)(*$3), free); %#endif %#if NPY_API_VERSION < 0x00000007 @@ -2706,9 +2708,9 @@ if (!array) SWIG_fail; %#ifdef SWIGPY_USE_CAPSULE - PyObject* cap = PyCapsule_New((void*)(*$1), SWIGPY_CAPSULE_NAME, free_cap); + PyObject* cap = PyCapsule_New((void*)(*$4), SWIGPY_CAPSULE_NAME, free_cap); %#else - PyObject* cap = PyCObject_FromVoidPtr((void*)(*$1), free); + PyObject* cap = PyCObject_FromVoidPtr((void*)(*$4), free); %#endif %#if NPY_API_VERSION < 0x00000007 @@ -2780,9 +2782,9 @@ if (!array || !require_fortran(array)) SWIG_fail; %#ifdef SWIGPY_USE_CAPSULE - PyObject* cap = PyCapsule_New((void*)(*$1), SWIGPY_CAPSULE_NAME, free_cap); + PyObject* cap = PyCapsule_New((void*)(*$4), SWIGPY_CAPSULE_NAME, free_cap); %#else - PyObject* cap = PyCObject_FromVoidPtr((void*)(*$1), free); + PyObject* cap = PyCObject_FromVoidPtr((void*)(*$4), free); %#endif %#if NPY_API_VERSION < 0x00000007 @@ -2856,9 +2858,9 @@ if (!array) SWIG_fail; %#ifdef SWIGPY_USE_CAPSULE - PyObject* cap = PyCapsule_New((void*)(*$1), SWIGPY_CAPSULE_NAME, free_cap); + PyObject* cap = PyCapsule_New((void*)(*$5), SWIGPY_CAPSULE_NAME, free_cap); %#else - PyObject* cap = PyCObject_FromVoidPtr((void*)(*$1), free); + PyObject* cap = PyCObject_FromVoidPtr((void*)(*$5), free); %#endif %#if NPY_API_VERSION < 0x00000007 @@ -2932,9 +2934,9 @@ if (!array || !require_fortran(array)) SWIG_fail; %#ifdef SWIGPY_USE_CAPSULE - PyObject* cap = PyCapsule_New((void*)(*$1), SWIGPY_CAPSULE_NAME, free_cap); + PyObject* cap = PyCapsule_New((void*)(*$5), SWIGPY_CAPSULE_NAME, free_cap); %#else - PyObject* cap = PyCObject_FromVoidPtr((void*)(*$1), free); + PyObject* cap = PyCObject_FromVoidPtr((void*)(*$5), free); %#endif %#if NPY_API_VERSION < 0x00000007 @@ -3008,9 +3010,9 @@ if (!array) SWIG_fail; %#ifdef SWIGPY_USE_CAPSULE - PyObject* cap = PyCapsule_New((void*)(*$1), SWIGPY_CAPSULE_NAME, free_cap); + PyObject* cap = PyCapsule_New((void*)(*$5), SWIGPY_CAPSULE_NAME, free_cap); %#else - PyObject* cap = PyCObject_FromVoidPtr((void*)(*$1), free); + PyObject* cap = PyCObject_FromVoidPtr((void*)(*$5), free); %#endif %#if NPY_API_VERSION < 0x00000007 @@ -3084,9 +3086,9 @@ if (!array || !require_fortran(array)) SWIG_fail; %#ifdef SWIGPY_USE_CAPSULE - PyObject* cap = PyCapsule_New((void*)(*$1), SWIGPY_CAPSULE_NAME, free_cap); + PyObject* cap = PyCapsule_New((void*)(*$5), SWIGPY_CAPSULE_NAME, free_cap); %#else - PyObject* cap = PyCObject_FromVoidPtr((void*)(*$1), free); + PyObject* cap = PyCObject_FromVoidPtr((void*)(*$5), free); %#endif %#if NPY_API_VERSION < 0x00000007 diff --git a/tools/swig/pyfragments.swg b/tools/swig/pyfragments.swg index ce2452f8020d..558633733da3 100644 --- a/tools/swig/pyfragments.swg +++ b/tools/swig/pyfragments.swg @@ -22,12 +22,9 @@ SWIGINTERN int SWIG_AsVal_dec(long)(PyObject * obj, long * val) { - if (PyInt_Check(obj)) { - if (val) *val = PyInt_AsLong(obj); - return SWIG_OK; - } else if (PyLong_Check(obj)) { + if (PyLong_Check(obj)) { long v = PyLong_AsLong(obj); - if (!PyErr_Occurred()) { + if (v != -1 || !PyErr_Occurred()) { if (val) *val = v; return SWIG_OK; } else { @@ -37,8 +34,8 @@ %#ifdef SWIG_PYTHON_CAST_MODE { int dispatch = 0; - long v = PyInt_AsLong(obj); - if (!PyErr_Occurred()) { + long v = PyLong_AsLong(obj); + if (v != -1 || !PyErr_Occurred()) { if (val) *val = v; return SWIG_AddCast(SWIG_OK); } else {