Sync randomgen

.circleci/config.yml

@@ -20,7 +20,7 @@ jobs:
           command: |
             python3 -m venv venv
             . venv/bin/activate
-            pip install cython sphinx>=1.8.3 matplotlib
+            pip install cython sphinx==1.8.5 matplotlib ipython
             sudo apt-get update
             sudo apt-get install -y graphviz texlive-fonts-recommended texlive-latex-recommended texlive-latex-extra texlive-generic-extra latexmk texlive-xetex
 

.gitignore

@@ -167,12 +167,16 @@ numpy/core/src/umath/test_rational.c
 numpy/core/src/umath/umath_tests.c
 numpy/distutils/__config__.py
 numpy/linalg/umath_linalg.c
-doc/source/reference/generated
+doc/source/**/generated/
 benchmarks/results
 benchmarks/html
 benchmarks/env
 benchmarks/numpy
 # cythonized files
 cythonize.dat
 numpy/random/mtrand/mtrand.c
+numpy/random/randomgen/*.c
+numpy/random/randomgen/legacy/*.c
 numpy/random/mtrand/randint_helpers.pxi
+numpy/random/randomgen/bounded_integers.pyx
+numpy/random/randomgen/bounded_integers.pxd

.travis.yml

@@ -69,6 +69,8 @@ matrix:
       env:
        # for matrix annotation only
        - PPC64_LE=1
+       # use POWER8 OpenBLAS build, not system ATLAS
+       - ATLAS=None
 
 before_install:
   - ./tools/travis-before-install.sh

LICENSE.txt

@@ -58,3 +58,8 @@ Name: dragon4
 Files: numpy/core/src/multiarray/dragon4.c
 License: One of a kind
   For license text, see numpy/core/src/multiarray/dragon4.c
+
+Name: randomgen
+Files: numpy/random/randomgen and docs/source/random
+Licence: NCSA
+  For license text, see numpy/random/randomgen/LICENSE.md

azure-pipelines.yml

@@ -28,9 +28,11 @@ jobs:
            F77=gfortran-5 F90=gfortran-5 \
            CFLAGS='-UNDEBUG -std=c99' python3 runtests.py --mode=full -- -rsx --junitxml=junit/test-results.xml"
     displayName: 'Run 32-bit Ubuntu Docker Build / Tests'
+    continueOnError: true
   - task: PublishTestResults@2
     inputs:
       testResultsFiles: '**/test-*.xml'
+      failTaskOnFailedTests: true
       testRunTitle: 'Publish test results for Python 3.6-32 bit full Linux'
 - job: macOS
   pool:
@@ -81,7 +83,7 @@ jobs:
     displayName: 'install pre-built openblas'
   - script: python -m pip install --upgrade pip setuptools wheel
     displayName: 'Install tools'
-  - script: python -m pip install cython nose pytz pytest pickle5 vulture docutils sphinx>=1.8.3 numpydoc matplotlib
+  - script: python -m pip install cython nose pytz pytest pickle5 vulture docutils sphinx==1.8.5 numpydoc matplotlib
     displayName: 'Install dependencies; some are optional to avoid test skips'
   - script: /bin/bash -c "! vulture . --min-confidence 100 --exclude doc/,numpy/distutils/ | grep 'unreachable'"
     displayName: 'Check for unreachable code paths in Python modules'
@@ -99,9 +101,11 @@ jobs:
     displayName: 'Run Refuide Check'
   - script: python runtests.py --mode=full -- -rsx --junitxml=junit/test-results.xml
     displayName: 'Run Full NumPy Test Suite'
+    continueOnError: true
   - task: PublishTestResults@2
     inputs:
       testResultsFiles: '**/test-*.xml'
+      failTaskOnFailedTests: true
       testRunTitle: 'Publish test results for Python 3.6 64-bit full Mac OS'
 - job: Windows
   pool:
@@ -200,7 +204,9 @@ jobs:
     displayName: 'For gh-12667; Windows DLL resolution'
   - script: python runtests.py -n --show-build-log --mode=$(TEST_MODE) -- -rsx --junitxml=junit/test-results.xml
     displayName: 'Run NumPy Test Suite'
+    continueOnError: true
   - task: PublishTestResults@2
     inputs:
       testResultsFiles: '**/test-*.xml'
+      failTaskOnFailedTests: true
       testRunTitle: 'Publish test results for Python $(PYTHON_VERSION) $(BITS)-bit $(TEST_MODE) Windows'

benchmarks/benchmarks/bench_function_base.py

@@ -199,6 +199,7 @@ class Sort(Benchmark):
         ['quick', 'merge', 'heap'],
         ['float64', 'int64', 'uint64'],
         [
+            ('random',),
             ('ordered',),
             ('reversed',),
             ('uniform',),

benchmarks/benchmarks/bench_io.py

@@ -66,7 +66,8 @@ def setup(self):
         self.squares = get_squares()
 
     def time_vb_savez_squares(self):
-        np.savez('tmp.npz', self.squares)
+        np.savez('tmp.npz', **self.squares)
+
 
 class LoadtxtCSVComments(Benchmark):
     # benchmarks for np.loadtxt comment handling

benchmarks/benchmarks/bench_random.py

@@ -4,6 +4,8 @@
 
 import numpy as np
 
+from numpy.random import RandomState
+from numpy.random.randomgen import RandomGenerator
 
 class Random(Benchmark):
     params = ['normal', 'uniform', 'weibull 1', 'binomial 10 0.5',
@@ -80,3 +82,100 @@ def time_permutation_2d(self):
 
     def time_permutation_int(self):
         np.random.permutation(self.n)
+
+nom_size = 100000
+
+class RNG(Benchmark):
+    param_names = ['rng']
+    params = ['DSFMT', 'PCG64', 'PCG32', 'MT19937', 'Xoroshiro128',
+              'Xorshift1024', 'Xoshiro256StarStar', 'Xoshiro512StarStar',
+              'Philox', 'ThreeFry', 'ThreeFry32', 'numpy']
+
+    def setup(self, brng):
+        if brng == 'numpy':
+            self.rg = np.random.RandomState()
+        else:
+            self.rg = RandomGenerator(getattr(np.random.randomgen, brng)())
+        self.rg.random_sample()
+        self.int32info = np.iinfo(np.int32)
+        self.uint32info = np.iinfo(np.uint32)
+        self.uint64info = np.iinfo(np.uint64)
+
+    def time_raw(self, brng):
+        if brng == 'numpy':
+            self.rg.random_integers(self.int32info.max, size=nom_size)
+        else:
+            self.rg.random_integers(self.int32info.max, size=nom_size)
+
+    def time_32bit(self, brng):
+        min, max = self.uint32info.min, self.uint32info.max
+        self.rg.randint(min, max + 1, nom_size, dtype=np.uint32)
+
+    def time_64bit(self, brng):
+        min, max = self.uint64info.min, self.uint64info.max
+        self.rg.randint(min, max + 1, nom_size, dtype=np.uint64)
+
+    def time_normal_zig(self, brng):
+        self.rg.standard_normal(nom_size)
+
+class Bounded(Benchmark):
+    u8 = np.uint8
+    u16 = np.uint16
+    u32 = np.uint32
+    u64 = np.uint64
+    param_names = ['rng', 'dt_max_masked']
+    params = [['DSFMT', 'PCG64', 'PCG32', 'MT19937', 'Xoroshiro128',
+               'Xorshift1024', 'Xoshiro256StarStar', 'Xoshiro512StarStar',
+               'Philox', 'ThreeFry', 'ThreeFry32', 'numpy'],
+              [[u8,  95, True],
+               [u8,  64, False],  # Worst case for legacy
+               [u8,  95, False],   # Typ. avg. case for legacy
+               [u8, 127, False],  # Best case for legacy
+               [u16,   95, True],
+               [u16, 1024, False],  # Worst case for legacy
+               [u16, 1535, False],   # Typ. avg. case for legacy
+               [u16, 2047, False],  # Best case for legacy
+               [u32,   95, True],
+               [u32, 1024, False],  # Worst case for legacy
+               [u32, 1535, False],   # Typ. avg. case for legacy
+               [u32, 2047, False],  # Best case for legacy
+               [u64,   95, True],
+               [u64, 1024, False],  # Worst case for legacy
+               [u64, 1535, False],   # Typ. avg. case for legacy
+               [u64, 2047, False],  # Best case for legacy
+             ]]
+
+    def setup(self, brng, args):
+        if brng == 'numpy':
+            self.rg = np.random.RandomState()
+        else:
+            self.rg = RandomGenerator(getattr(np.random.randomgen, brng)())
+        self.rg.random_sample()
+
+    def time_bounded(self, brng, args):
+            """
+            Timer for 8-bit bounded values.
+
+            Parameters (packed as args)
+            ----------
+            dt : {uint8, uint16, uint32, unit64}
+                output dtype
+            max : int
+                Upper bound for range. Lower is always 0.  Must be <= 2**bits.
+            use_masked: bool
+                If True, masking and rejection sampling is used to generate a random
+                number in an interval. If False, Lemire's algorithm is used if
+                available to generate a random number in an interval.
+
+            Notes
+            -----
+            Lemire's algorithm has improved performance when max+1 is not a
+            power of two.
+            """
+            dt, max, use_masked = args
+            if brng == 'numpy':
+                self.rg.randint(0, max + 1, nom_size, dtype=dt)
+            else:
+                self.rg.randint(0, max + 1, nom_size, dtype=dt,
+                                use_masked=use_masked)
+

doc/DISTUTILS.rst.txt

@@ -319,11 +319,7 @@ and :c:data:`/**end repeat**/` lines, which may also be nested using
 consecutively numbered delimiting lines such as :c:data:`/**begin repeat1`
 and :c:data:`/**end repeat1**/`. String replacement specifications are started
 and terminated using :c:data:`#`. This may be clearer in the following
-template source example:
-
-.. code-block:: C
-   :linenos:
-   :emphasize-lines: 3, 13, 29, 31
+template source example::
 
     /* TIMEDELTA to non-float types */
 

doc/Makefile

@@ -20,7 +20,8 @@ FILES=
 # Internal variables.
 PAPEROPT_a4     = -D latex_paper_size=a4
 PAPEROPT_letter = -D latex_paper_size=letter
-ALLSPHINXOPTS   = -d build/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source
+ALLSPHINXOPTS   = -WT --keep-going -d build/doctrees $(PAPEROPT_$(PAPER)) \
+  $(SPHINXOPTS) source
 
 .PHONY: help clean html web pickle htmlhelp latex changes linkcheck \
         dist dist-build gitwash-update
@@ -41,7 +42,8 @@ help:
 	@echo "  upload USERNAME=... RELEASE=... to upload built docs to docs.scipy.org"
 
 clean:
-	-rm -rf build/* source/reference/generated
+	-rm -rf build/* 
+	find . -name generated -type d -prune -exec rm -rf "{}" ";"
 
 gitwash-update:
 	rm -rf source/dev/gitwash

doc/release/1.17.0-notes.rst

@@ -18,11 +18,11 @@ New functions
 Deprecations
 ============
 
-``np.polynomial`` functions warn when passed ``float``s in place of ``int``s
-----------------------------------------------------------------------------
-Previously functions in this module would accept ``float``s provided their
-values were integral. For consistency with the rest of numpy, doing so is now
-deprecated, and in future will raise a ``TypeError``.
+``np.polynomial`` functions warn when passed ``float`` in place of ``int``
+--------------------------------------------------------------------------
+Previously functions in this module would accept ``float`` values provided they
+were integral (``1.0``, ``2.0``, etc). For consistency with the rest of numpy,
+doing so is now deprecated, and in future will raise a ``TypeError``.
 
 Similarly, passing a float like ``0.5`` in place of an integer will now raise a
 ``TypeError`` instead of the previous ``ValueError``.
@@ -72,6 +72,11 @@ Looking up ``__buffer__`` attribute in `numpy.frombuffer` was undocumented and
 non-functional. This code was removed. If needed, use
 ``frombuffer(memoryview(obj), ...)`` instead.
 
+``out``is buffered for memory overlaps in ``np.take``, ``np.choose``, ``np.put``
+--------------------------------------------------------------------------------
+If the out argument to these functions is provided and has memory overlap with
+the other arguments, it is now buffered to avoid order-dependent behavior.
+
 
 C API changes
 =============
@@ -119,6 +124,10 @@ divmod operation is now supported for two ``timedelta64`` operands
 The divmod operator now handles two ``np.timedelta64`` operands, with
 type signature mm->qm.
 
+New mode "empty" for ``np.pad``
+-------------------------------
+This mode pads an array to a desired shape without initializing the new
+entries.
 
 Improvements
 ============
@@ -171,11 +180,29 @@ but with this change, you can do::
 
 thereby saving a level of indentation
 
+Improve performance of ``np.pad``
+---------------------------------
+The performance of the function has been improved for most cases by filling in
+a preallocated array with the desired padded shape instead of using
+concatenation.
+
 ``np.interp`` handles infinities more robustly
 ----------------------------------------------
 In some cases where ``np.interp`` would previously return ``np.nan``, it now
 returns an appropriate infinity.
 
+Specialized ``np.isnan``, ``np.isinf``, and ``np.isfinite`` ufuncs for bool and int types
+-----------------------------------------------------------------------------------------
+The boolean and integer types are incapable of storing ``np.nan`` and ``np.inf`` values,
+which allows us to provide specialized ufuncs that are up to 250x faster than the current
+approach.
+
+New keywords added to ``np.nan_to_num``
+---------------------------------------
+``np.nan_to_num`` now accepts keywords ``nan``, ``posinf`` and ``neginf`` allowing the
+user to define the value to replace the ``nan``, positive and negative ``np.inf`` values 
+respectively.
+
 
 Changes
 =======

doc/source/conf.py

@@ -19,11 +19,19 @@
 
 sys.path.insert(0, os.path.abspath('../sphinxext'))
 
-extensions = ['sphinx.ext.autodoc', 'numpydoc',
-              'sphinx.ext.intersphinx', 'sphinx.ext.coverage',
-              'sphinx.ext.doctest', 'sphinx.ext.autosummary',
-              'sphinx.ext.graphviz', 'sphinx.ext.ifconfig',
-              'matplotlib.sphinxext.plot_directive']
+extensions = [
+    'sphinx.ext.autodoc',
+    'numpydoc',
+    'sphinx.ext.intersphinx',
+    'sphinx.ext.coverage',
+    'sphinx.ext.doctest',
+    'sphinx.ext.autosummary',
+    'sphinx.ext.graphviz',
+    'sphinx.ext.ifconfig',
+    'matplotlib.sphinxext.plot_directive',
+    'IPython.sphinxext.ipython_console_highlighting',
+    'IPython.sphinxext.ipython_directive',
+]
 
 if sphinx.__version__ >= "1.4":
     extensions.append('sphinx.ext.imgmath')
@@ -234,7 +242,7 @@ def setup(app):
 # -----------------------------------------------------------------------------
 
 import glob
-autosummary_generate = glob.glob("reference/*.rst")
+autosummary_generate = True
 
 # -----------------------------------------------------------------------------
 # Coverage checker
@@ -355,3 +363,8 @@ def linkcode_resolve(domain, info):
     else:
         return "https://github.com/numpy/numpy/blob/v%s/numpy/%s%s" % (
            numpy.__version__, fn, linespec)
+
+doctest_global_setup = '''
+import numpy as np
+from numpy.random import randomgen
+'''

doc/source/dev/development_environment.rst

@@ -127,6 +127,9 @@ the interpreter, tests can be run like this::
     >>> np.test('full')   # Also run tests marked as slow
     >>> np.test('full', verbose=2)   # Additionally print test name/file
 
+    An example of a successful test :
+    ``4686 passed, 362 skipped, 9 xfailed, 5 warnings in 213.99 seconds``
+
 Or a similar way from the command line::
 
     $ python -c "import numpy as np; np.test()"

doc/source/docs/howto_build_docs.rst

@@ -5,7 +5,7 @@ Building the NumPy API and reference docs
 =========================================
 
 We currently use Sphinx_ for generating the API and reference
-documentation for NumPy.  You will need Sphinx 1.0.1 or newer.
+documentation for NumPy.  You will need Sphinx 1.8.3 or newer.
 
 If you only want to get the documentation, note that pre-built
 versions can be found at