From 804ca08c3728447daf22d487b7e3c15b09daa29f Mon Sep 17 00:00:00 2001 From: Oscar Benjamin Date: Sat, 31 Aug 2024 21:00:16 +0100 Subject: [PATCH 1/4] build: Check Cython version in meson build --- meson.build | 87 +++++++++++++++++++++++++++++------------------------ 1 file changed, 48 insertions(+), 39 deletions(-) diff --git a/meson.build b/meson.build index 71863628..6417ea4a 100644 --- a/meson.build +++ b/meson.build @@ -1,58 +1,77 @@ project('python-flint', 'cython', 'c') +# +# The minimum versions are because we know that it will not work with earlier +# versions. The maximum versions are because python-flint was not tested +# against future versions that didn't exist at the time of release. In future +# if it seems like new releases do not always break the build of python-flint +# then we can consider not using a speculative upper version cap here. +# +flint_lower = '>=3.0' +flint_upper = '<3.2' +cython_lower = '>=3.0' +cython_upper = '<3.2' py = import('python').find_installation(pure: false) dep_py = py.dependency() cc = meson.get_compiler('c') - -flint_ver_lower = '3.0' # >=3.0 -flint_ver_upper = '3.2' # <3.2 +cy = meson.get_compiler('cython') gmp_dep = dependency('gmp') mpfr_dep = dependency('mpfr') -flint_dep = dependency('flint', version: ['>=' + flint_ver_lower]) +flint_dep = dependency('flint') # -# The minimum Flint version is because we know that it will not work with -# earlier versions. The maximum version is because python-flint has not been -# tested against versions that didn't exist at the time of release. In future -# if it seems like new Flint releases do not break the build of python-flint -# every time, then we can consider not using a speculative upper version cap -# here. -# -# For the source release, we should by default fail for newer versions of Flint -# that are untested with a nice error message. +# For the source release, we should by default fail for new untested versions +# with a clear error message about the version mismatch. # # We need an option to disable this though so that we can test newer versions # of Flint. Also good to have an escape hatch for users since we don't know # that future versions of Flint will not work. # -if get_option('flint_version_check') - if flint_dep.version().version_compare('>=' + flint_ver_upper) - error(''' +ver_message = ''' + +Invalid @0@ version: +Version needed is: @0@ @2@, @3@ +Version found is: @0@ == @1@ - Invalid Flint version: - Version needed is: @0@ <= flint < @1@ - Version found is: flint == @2@ +By default, python-flint will only build against @0@ versions that have +been tested. If you are sure you want to use this version of @0@, you can +disable this check with -Dflint_version_check=false. - By default, python-flint will only build against Flint versions that have - been tested. If you are sure you want to use this version of Flint, you can - disable this check with -Dflint_version_check=false. +If building from the source directory using meson directly, you can do this +with: - If building from the source directory using meson directly, you can do this - with: + meson setup build -Dflint_version_check=false - meson setup build -Dflint_version_check=false +If you are installing with pip, you can do this with: - If you are installing with pip, you can do this with: + pip install --config-settings=setup-args="-Dflint_version_check=false" python-flint - pip install --config-settings=setup-args="-Dflint_version_check=false" python-flint +Other build frontends have similar options for passing this to meson. - Other build frontends have similar options for passing this to meson. - '''.format(flint_ver_lower, flint_ver_upper, flint_dep.version())) +''' +if get_option('flint_version_check') + if not (flint_dep.version().version_compare(flint_lower) and + flint_dep.version().version_compare(flint_upper)) + error(ver_message.format('FLINT', flint_dep.version(), flint_lower, flint_upper)) endif + if not (cy.version().version_compare(cython_lower) and + cy.version().version_compare(cython_upper)) + error(ver_message.format('Cython', cy.version(), cython_lower, cython_upper)) + endif +endif + +# flint.pc was missing -lflint until Flint 3.1.0 +if flint_dep.version().version_compare('<3.1') + flint_dep = cc.find_library('flint') + have_acb_theta = false +else + have_acb_theta = true endif +pyflint_deps = [dep_py, gmp_dep, mpfr_dep, flint_dep] + add_project_arguments( '-X', 'embedsignature=True', '-X', 'emit_code_comments=True', @@ -74,14 +93,4 @@ if get_option('add_flint_rpath') ) endif -# flint.pc was missing -lflint until Flint 3.1.0 -if flint_dep.version().version_compare('<3.1') - flint_dep = cc.find_library('flint') - have_acb_theta = false -else - have_acb_theta = true -endif - -pyflint_deps = [dep_py, gmp_dep, mpfr_dep, flint_dep] - subdir('src/flint') From 73fbb625390149136f72a797cccda08eb499e16f Mon Sep 17 00:00:00 2001 From: Oscar Benjamin Date: Sun, 1 Sep 2024 01:37:58 +0100 Subject: [PATCH 2/4] doc: Add a building from source guide --- bin/cibw_before_all_windows.sh | 5 +- doc/source/build.rst | 460 +++++++++++++++++++++++++++++++++ doc/source/index.rst | 3 +- doc/source/install.rst | 99 +++++++ doc/source/setup.rst | 104 -------- 5 files changed, 565 insertions(+), 106 deletions(-) create mode 100644 doc/source/build.rst create mode 100644 doc/source/install.rst delete mode 100644 doc/source/setup.rst diff --git a/bin/cibw_before_all_windows.sh b/bin/cibw_before_all_windows.sh index 1b45da64..8395f6de 100755 --- a/bin/cibw_before_all_windows.sh +++ b/bin/cibw_before_all_windows.sh @@ -9,6 +9,9 @@ set -o errexit # Make a setup.cfg to specify compiling with mingw64 (even though it says # mingw32...) # + +# This is not needed any more for python-flint >= 0.7.0 because meson is now +# used as the build system rather than setuptools: echo '[build]' > setup.cfg echo 'compiler = mingw32' >> setup.cfg cat setup.cfg @@ -25,5 +28,5 @@ pacman -S --noconfirm \ libtool\ # -# This takes ~30mins +# This is slow with MinGW: bin/build_dependencies_unix.sh --use-gmp-github-mirror diff --git a/doc/source/build.rst b/doc/source/build.rst new file mode 100644 index 00000000..f2c1edf5 --- /dev/null +++ b/doc/source/build.rst @@ -0,0 +1,460 @@ +Build from source +================= + +.. _supported_versions: + +Compatibility and supported versions +------------------------------------ + +.. note:: + The compatibility information here is mostly only relevant when building + ``python-flint`` from source. For most users it is recommended to install + pre-built binaries from ``PyPI`` or ``conda-forge``. See + :ref:`install_pip_conda`. + +Generally each release of python-flint will be compatible with a range of +Python versions as described in `SPEC 0 +`_. Since python-flint 0.5.0 +the minimum supported FLINT version is ``3.0`` and each release of python-flint +supports all versions of ``FLINT >= 3.0`` available at the time of release. + +Compatible versions (note that 0.7.0 is not yet released): + +.. list-table:: python-flint compatibility + :header-rows: 1 + + * - python-flint + - Release date + - CPython + - FLINT + - Cython + * - 0.7.0 + - Not yet + - 3.10-3.13 + - 3.0-3.2? + - 3.0-3.1? + * - 0.6.0 + - 1st Feb 2024 + - 3.9-3.12 + - 3.0 only + - 3.0 only + * - 0.5.0 + - 22nd Oct 2023 + - 3.9-3.12 + - 3.0 only + - 3.0 only + +If installing binaries from PyPI or conda-forge then the Python (CPython) +version is the only column in this table that matters. The minimum and maximum +versions of Python represent the versions that are tested in CI and for which +binaries are provided on PyPI. It is likely that ``python-flint`` will work +with other versions of Python (particularly older Python versions) but this is +not tested and requires building from source. It is possible that +``conda-forge`` may provide binaries for other versions of Python. + +The minimum versions of Cython and FLINT are needed because it is known that +python-flint will not even build with older versions of these libraries. The +maximum versions of all dependencies are speculative and are based on the +versions that are known to work at the time of release. It is possible that +newer versions of Cython and FLINT will work but from ``python-flint 0.4.0`` +through to the time of writing (``python-flint 0.7.0``) minor releases of +Cython, FLINT, or CPython have frequently required changes to the +``python-flint`` source code to be able to build at all. In particular the +following releases of Cython, FLINT and CPython have had changes that would +prevent building all versions of ``python-flint`` existing at the time of the +release: + +- Flint 3.0 (Arb and Flint merged, lots of changes) +- Flint 3.1 (Function signature for ``fmpz_mod_mat`` changed) +- Flint 3.2 (``flint_randinit`` function name changed) +- Cython 3.0 (Handling of dunders changed) +- Cython 3.1 (Removal of ``PyInt_*`` functions) +- CPython 3.12 (Removal of distutils) + +In fact out of python-flint's three core dependencies (CPython, Cython, FLINT), +the only minor release that has not required changes to the python-flint source +code during this time has been CPython ``3.13``. It is expected then that any +future untested ``3.x`` version of Cython, FLINT, or CPython will not be +compatible with past versions of ``python-flint`` which is why the table above +only lists the versions that are known to work. + +As of python-flint 0.7.0, CPython ``3.13t`` free-threaded builds are tested in +CI but wheels are not provided on PyPI. There are no known issues related to +using python-flint in a `PEP 703 `_ +free-threaded build but it is likely that mutating objects shared by multiple +threads is not safe. + +It is also possible to build and use python-flint for PyPy. Other Python +implementations may work but are not tested. + + +.. _build_from_source: + +Building from source +-------------------- + +.. note:: + Building from source is not the recommended way for most users to install + ``python-flint``. For most users it is recommended to use the binaries from + ``PyPI`` or ``conda-forge`` except in cases where a binary is not available + for the platform. See :ref:`install_pip_conda`. + +.. note:: + The instructions here assume that you want to build and install ``python-flint`` + on a platform for which a binary is not available on PyPI or conda-forge. If + you are interested in contributing to ``python-flint`` then see the + + - XXX: Add link to separate development page... + +The source code for ``python-flint`` is available on `GitHub +`_ and source distributions can +be downloaded from PyPI. + +To build from source you must first install the dependencies (see +:ref:`install_dependencies` below for instructions). Once the dependencies are +installed the following command will download the ``python-flint`` source code +from PyPI, then build and install it into the active Python environment:: + + pip install python-flint + +This will download, build and install the latest release of ``python-flint`` +from PyPI. + +.. note:: + If you have more than one Python environment on your system then you need to + ensure that you are installing ``python-flint`` into the correct one. This + may require using the full path to ``pip`` or something like ``python3 -m + pip`` or by activating the correct environment first. + +To install a specific version of ``python-flint`` from PyPI use:: + + pip install python-flint==0.6.0 + +To download and install the latest ``python-flint`` from git master you can +use:: + + pip install git+https://github.com/flintlib/python-flint.git@master + +If you already have the source code downloaded or checked out from git, you can +``cd`` in and build and install with:: + + pip install . + +Alternatively if you would like to build a wheel you can use +``pypa/build`` (first ``pip install build``):: + + python -m build + +Note that wheels built in this way will not include the dependencies (unlike +those distributed on PyPI) and cannot generally be installed on other systems. + +For ``python-flint < 0.6.0`` the source distribution did not include +``pyproject.toml`` and did not list the build requirements. Also for +``python-flint < 0.7.0`` there were no version constraints on the build +requirements. If you are building an older version of ``python-flint`` then you +may need to install the build requirements manually and disable build isolation +with ``--no-build-isolation``:: + + pip install Cython==3.0 setuptools numpy + pip install --no-build-isolation . + +To build without build isolation with ``python-flint >= 0.7.0`` the needed +dependencies are different:: + + pip install Cython==3.0 meson meson-python ninja + pip install --no-build-isolation . + +Since ``python-flint 0.7.0`` the build system is ``meson`` and the build +requirements and version constraints are listed in ``pyproject.toml``. When +using build isolation the build requirements are installed in a temporary +virtual environment and so it should not be necessary to install them in the +active Python environment before running ``pip install``. + +The ``meson`` build system will also detect the version of ``FLINT`` installed +in the system and will fail if it is not a version that was known to be +compatible at time of the release of ``python-flint``. To build against new, +untested versions of ``FLINT`` you can pass the ``-Dflint_version_check=false`` +option to the ``meson`` build system:: + + pip install --config-settings=setup-args="-Dflint_version_check=false" . + +This is useful for testing new versions of ``FLINT`` with ``python-flint`` for +example if you want to build ``python-flint`` against the latest git version of +``FLINT``. See :ref:`supported_versions` above for the versions of ``FLINT`` +and Cython that are supported by each version of ``python-flint``. + + +.. _install_dependencies: + +Installing the dependencies +--------------------------- + +.. note:: + It is not necessary to install the dependencies manually if you install + from PyPI or conda-forge as is recommended. When installing with ``conda`` + the packages for the dependencies will also be installed from conda-forge + automatically. The binaries on PyPI are built with the dependencies bundled + in the wheel so that they do not need to be installed separately. + + The following instructions are only for when building ``python-flint`` from + source if needed because a binary is not available for your platform. See + :ref:`install_pip_conda`. + +The dependencies for building ``python-flint`` have changed over time. See +:ref:`supported_versions` above for the versions of the dependencies that are +supported by each version of ``python-flint``. + +As of ``python-flint 0.7.0`` the runtime dependencies are Python and FLINT (at +least version 3.0) and the build-time dependencies are a C compiler, +``Cython``, ``meson``, ``meson-python`` and ``ninja``. Commands shown above +such as ``pip install .`` will install dependencies like ``Cython``, ``meson`` +etc automatically. If you already have Python and a C compiler then what needs +to be installed before building ``python-flint`` is ``FLINT``. + +At the time of writing, few Linux distributions provide ``FLINT >= 3.0`` in +their package repositories but for example on ``Ubuntu 24.04`` (but not any +earlier Ubuntu versions) you can install ``FLINT 3.0.1`` with:: + + sudo apt-get install libflint-dev + +On MacOS you can install FLINT from homebrew with:: + + brew install flint + +Other package managers may also provide ``FLINT`` but make sure that it is at +least version ``3.0``. + +Once ``FLINT`` is installed it should be possible to build ``python-flint`` +with any of the commands shown above e.g.:: + + pip install . + +If it is not possible to install FLINT from a package manager then you need to +install GMP and MPFR and then build FLINT. You may still be able to install GMP +and MPFR from a package manager for example on Ubuntu:: + + sudo apt-get install libgmp-dev libmpfr-dev + +The python-flint git repo has a script `bin/install_flint_ubuntu.sh +`_ +that uses ``apt-get`` to install all dependencies needed to build ``FLINT``, +then builds ``FLINT`` from git using a specified git ref, and then installs +``FLINT`` system-wide:: + + bin/install_flint_ubuntu.sh v3.0.1 # version 3.0.1 + bin/install_flint_ubuntu.sh main # latest git + +The script can be adapted for other Linux distributions or MacOS to use +something other than ``apt-get`` to install dependencies. + +If the whole stack needs to be built from source then download the source for +all three (`GMP `_, `MPFR +`_, `FLINT +`_) and build each with the standard:: + + ./configure + make + make install + +Adapt the ``configure`` commands as needed. Once these are installed you should +again be able to install ``python-flint`` with:: + + pip install . + +In the ``python-flint`` git repo there is a script +`bin/build_dependencies_unix.sh +`_ +which will download and build GMP, MPFR and FLINT and install them in the +current directory under ``~/.local``. The versions used and the installation +directory can be changed by editing the `bin/build_variables.sh +`_ +script. This script is useful for building ``python-flint`` on systems where +the system-wide ``FLINT`` is too old or if precise control over the versions of +GMP, MPFR and FLINT is needed. This script is used for building the binaries +for PyPI and also takes care of ensuring that ``GMP`` and ``FLINT`` are built +as redistributable shared libraries (this is not the default behaviour of the +``configure`` scripts for these libraries and disables some optimisation +features of ``FLINT`` on some ``x86_64`` micro-architectures). Since this +installation is not system-wide, see :ref:`non_standard_location` below for +instructions on how to build and use ``python-flint`` in this case. + + +Building on Windows +------------------- + +.. note:: + Building from source is not the recommended way for most users to install + ``python-flint``, especially on Windows. For most users it is recommended to + use the binaries from ``PyPI`` or ``conda-forge`` except in cases where a + binary is not available for the platform. See :ref:`install_pip_conda`. + +The instructions in :ref:`install_dependencies` above are for Unix-like systems +(e.g. Linux or MacOS). On Windows the dependencies can be built in a similar +way using MSYS2 or WSL. It is also possible to build ``python-flint`` and its +dependencies using MSVC but we do not currently provide instructions for this. +The `conda-forge recipe +`_ for ``python-flint`` +builds on Windows using MSVC. + +The `MSYS2 `_ project provides a Unix-like environment +for Windows and a package manager that can be used to install the dependencies. +The git repo for ``python-flint`` has a script `bin/cibw_before_all_windows.sh +`_ +that installs the dependencies under MSYS2 and builds ``GMP``, ``MPFR``, +``FLINT``. This script is used for building the Windows binaries for PyPI. We +use the ``MinGW64`` (``mingw-w64-x86_64``) toolchain for building on Windows +rather than MSVC because it makes it possible to have a fat build of ``GMP`` +(``--enable-fat``) which bundles micro-architecture specific optimisations for +``x86_64`` in a redistributable shared library. This is important for +performance on modern ``x86_64`` CPUs and is not possible if building ``GMP`` +with MSVC. Since we need to use ``MinGW64`` for building ``GMP`` it is simplest +to use it for building ``MPFR``, ``FLINT`` and ``python-flint`` as well and +means that the same Unix-style build scripts can be used for all platforms. + +The ``python-flint`` project does not have much experience using MSVC. Possibly +it would be better to build ``GMP`` using ``MinGW64`` and then build ``MPFR``, +``FLINT`` and ``python-flint`` using MSVC. It is also possible that it would be +better to build ``GMP``, ``MPFR``, ``FLINT`` using MinGW64 and then build +``python-flint`` using MSVC. Someone with more experience with MSVC would need +to help with this. We would welcome contributions that explain how to build +``python-flint`` and its dependencies using MSVC and/or that improve the build +process for distributed binaries on Windows. + + +.. _non_standard_location: + +Using ``FLINT`` from a non-standard location +-------------------------------------------- + +.. note:: + This section is only relevant when building ``python-flint`` from source. + For most users it is recommended to use the binaries from ``PyPI`` or + ``conda-forge``. See :ref:`install_pip_conda`. The instructions here are + also not needed if you have installed ``FLINT`` and its dependencies + system-wide. + +If you have installed ``FLINT`` in a non-standard location then you will need +to instruct the ``python-flint`` build system where to find it and ensure that +the ``FLINT`` shared library can be found at runtime. + +Since ``python-flint 0.7.0`` the build system is `meson +`_ and uses `pkg-config +`_ to find the +dependencies ``FLINT``, ``GMP`` and ``MPFR``. If these are installed in a +non-standard location then you can set the ``PKG_CONFIG_PATH`` environment +variable to point to the directory containing the ``.pc`` files for these +libraries. For example if you have installed ``FLINT`` in ``~/.local`` then you +can set the environment variable like this:: + + export PKG_CONFIG_PATH=$(pwd)/.local/lib/pkgconfig + +Note that in some systems the ``lib/pkgconfig`` directory may be in a different +location e.g. ``lib64/pkgconfig``. It is also possible to pass the path to the +``pkg-config`` files to the ``meson-python`` build backend. For example if +building with ``pip``:: + + pip install --config-settings=setup-args="--pkg-config-path=$(pwd)/.local/lib/pkgconfig" \ + python-flint + +Setting the path to the ``pkg-config`` files in this way will allow the +``python-flint`` build system to find the ``FLINT`` library at build time. At +runtime the ``GMP``, ``MPFR`` and ``FLINT`` shared libraries must be in a +location where the dynamic linker can find them. On Linux the environment +variable ``LD_LIBRARY_PATH`` can be used to add the directory containing the +shared libraries to the search path. On MacOS the environment variable is +``DYLD_LIBRARY_PATH`` and on Windows it is ``PATH``. For example on Linux if +``FLINT`` is installed in ``~/.local/lib`` then you can set the environment +variable:: + + export LD_LIBRARY_PATH=$(pwd)/.local/lib + +Using the environment variable like this means that it needs to be set every +time you run Python and use ``python-flint`` (the git repo provides ``source +bin/activate`` for doing this). A better option on Unix-like systems is to +install ``RPATH`` entries into the ``python-flint`` extension modules. On some +platforms this is done automatically by the ``meson`` build system but on +others it needs to be enabled explicitly. This can be done by passing the +``-Dadd_flint_rpath=true`` option to the ``meson`` build system:: + + pip install --config-settings=setup-args="--pkg-config-path=$(pwd)/.local/lib/pkgconfig" \ + --config-settings=setup-args="-Dadd_flint_rpath=true" \ + python-flint + +For versions of ``python-flint`` before ``0.7.0`` the build system is +``setuptools`` (or ``numpy.distutils`` for ``Python < 3.12``). In this case +``pkg-config`` is not used. The following environment variables can be used to +set the location of the ``FLINT`` and other shared libraries at build time or +runtime:: + + C_INCLUDE_PATH=$(pwd)/.local/include # build-time + LIBRARY_PATH=$(pwd)/.local/lib # build-time + LDFLAGS=-Wl,-rpath=$(pwd)/.local/lib # build-time Linux or MacOS + LD_LIBRARY_PATH=$(pwd)/.local/lib # run-time Linux + DYLD_LIBRARY_PATH=$(pwd)/.local/lib # run-time MacOS + PATH=$(pwd)/.local/bin:$PATH # run-time Windows + +A future improvement for ``python-flint`` could be if the meson build system +could build all dependencies (``GMP``, ``MPFR``, ``FLINT``) as shared libraries +and bundle them into ``python-flint`` although `this is not currently possible +with meson-python +`_. Otherwise +perhaps it could be possible to link ``FLINT`` and the other libraries +statically into ``python-flint``. + + +Editable install +---------------- + +.. note:: + For working on ``python-flint`` itself it is not recommended to install the + package into the active Python environment. Instead the development + workflow uses ``spin`` and ``meson`` to manage a local build of + ``python-flint``. See + + - XXX: Add link to separate development page... + + for more information on how to develop ``python-flint``. + +If you are building and testing ``python-flint`` while working on another +project then it may be useful to install ``python-flint`` in editable mode. +This allows making changes to the code of ``python-flint`` and seeing the +changes reflected in the other environment without needing to reinstall +``python-flint`` each time. This might be useful for example if you are using +``git bisect`` to find a change in ``python-flint`` (although it will not work +if you go back to versions before ``0.7.0``). + +Since ``0.7.0`` it is possible to install ``python-flint`` as a +`meson-python editable install +`_. +To install ``python-flint`` in editable mode, first install ``FLINT`` and +then:: + + git clone https://github.com/flintlib/python-flint.git + cd python-flint + pip install meson meson-python cython ninja + pip install --no-build-isolation --editable . + python -m flint.test # recommended if you have made changes + +This requires ``--no-build-isolation`` so that the build directory is not +deleted after install. Once installed in editable mode, each time Python is +restarted and ``python-flint`` is imported (``import flint``) an import hook +will check if the source code has changed and if so will rebuild the extension +modules and update the Python files. The rebuild uses ``meson`` for fast, +parallel, incremental rebuilds. Note that for the rebuild to happen and for the +changes to take effect it is necessary to start a new Python process e.g. by +running ``python`` again or by restarting the Jupyter kernel. + +If you have installed ``FLINT`` in a non-standard location then you should set +the ``pkg-config`` path as described in :ref:`non_standard_location` above:: + + pip install --no-build-isolation \ + --config-settings=setup-args="--pkg-config-path=$(pwd)/.local/lib/pkgconfig" \ + --editable . + +To fully remove the editable install you can run:: + + pip uninstall python-flint + +and then delete the ``build`` directory that was created in the root of the +``python-flint`` git repo. + diff --git a/doc/source/index.rst b/doc/source/index.rst index 03d62af7..747d6512 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -28,8 +28,9 @@ Introduction .. toctree:: :maxdepth: 2 - setup.rst + install.rst general.rst + build.rst Reference diff --git a/doc/source/install.rst b/doc/source/install.rst new file mode 100644 index 00000000..d106dd81 --- /dev/null +++ b/doc/source/install.rst @@ -0,0 +1,99 @@ +Install +======= + +.. _install_pip_conda: + +Install with pip or conda +------------------------- + +The recommended way to install ``python-flint`` for general use is to install a +prebuilt binary package. Python-FLINT is available on PyPI, the `Python Package +Index `_. The latest release can be +installed using:: + + pip install python-flint + +.. note:: + If you have more than one Python environment on your system then you should + ensure that you are installing ``python-flint`` into the correct Python + environment. This may require using the full path to ``pip`` or using + something like ``python3 -m pip`` to ensure that ``pip`` is run from the + correct Python environment. + +Python-FLINT is also available from `conda-forge +`_ and can be installed with +``conda``:: + + conda install -c conda-forge python-flint + +Both PyPI and conda-forge provide prebuilt binary packages for ``python-flint`` +for common platforms such as Windows, MacOS and Linux for a range of different +architectures and Python versions. If binaries are available for your platform +then installing from PyPI or conda-forge is simplest and does not require any +other dependencies or configuration. It is also possible that other software +distribution channels may provide binaries for ``python-flint``. + +A specific version of ``python-flint`` can be installed with ``pip`` by +specifying the version number, for example:: + + pip install python-flint==0.6.0 + +At the time of writing the next release of ``python-flint`` will be ``0.7.0`` +and binaries will be provided for Python 3.10, 3.11, 3.12 and 3.13 for the +following platforms: + +- Windows 64-bit (``x86_64``) +- MacOS 64-bit Intel and 64-bit ARM (i.e. Apple Silicon) +- Linux 64-bit (``x86_64``, ``manylinux``) + +Notably, at the time of writing the following platforms do *not* have binaries +available: + +- Windows on ARM. +- Linux aarch64 (``conda-forge`` has binaries but ``PyPI`` does not, see + `gh-105 `_). +- non-glibc Linux distros (``musllinux`` rather than ``manylinux``) + +You can see which binaries are provided for the latest release of +``python-flint`` `from PyPI `_ +and which are provided `from conda-forge +`_. A list of supported versions +of Python and other ``python-flint`` dependencies can be found at +:ref:`supported_versions`. + +If a binary is not available for your platform then you can build from source +as described in :ref:`build_from_source`. + + +Testing your installation +------------------------- + +However you install ``python-flint``, you can test the installation by running +the test suite (for ``python-flint >= 0.5.0``):: + + python -m flint.test + +This does not take long and will run all the tests and doctests and should +hopefully show something like this:: + + $ python -m flint.test --quiet + Running tests... + Running doctests... + flint.test: all 54 tests passed! + flint.test: all 4283 doctests passed! + ---------------------------------------- + OK: Your installation of python-flint seems to be working just fine! + +From Python you can instead run the same tests with:: + + from flint.test.__main__ import main + main() + +If you have installed ``python-flint`` from PyPI or conda-forge as described +above and your installation passes the tests then you are ready to use +``python-flint``. If the tests fail (or do not complete) then please report the +problem on the `python-flint issue tracker +`_. + +If the tests have all passed then ``python-flint`` is installed and ready to +use! diff --git a/doc/source/setup.rst b/doc/source/setup.rst deleted file mode 100644 index 8865850b..00000000 --- a/doc/source/setup.rst +++ /dev/null @@ -1,104 +0,0 @@ -Setup -=============================================================================== - -First install both FLINT (version 2.9 or later) and Arb (version 2.23 or later). -See: - -* http://flintlib.org/ -* http://arblib.org/ - -Python-FLINT is available on PyPI, the Python Package Index -(https://pypi.org/project/python-flint/). -The latest release can be installed using:: - - pip install python-flint - -Binary wheels are provided for Windows amd64, Linux (manylinux 2_17) x86_64, -macOS x86_64 and macOS arm64. For other platforms, pip will attempt to build -Python-FLINT from source which requires a C compiler and the FLINT and Arb -header files and library files (libflint.so and libarb.so) to be available as -well as the Python development headers and Cython and numpy. - -Python-FLINT is also available on conda-forge for Linux and macOS. -(https://anaconda.org/conda-forge/python-flint). -It can be installed using:: - - conda install -c conda-forge python-flint - -Python-FLINT can also be installed from a local git checkout or a source archive -as follows:: - - pip install . - -To build Python-FLINT manually, you first need to install some build -dependencies:: - - pip install Cython numpy - -Then run:: - - python setup.py build_ext - python setup.py install - -Run the test suite:: - - python -m flint.test - -Build the documentation:: - - cd doc - make html - cd .. - -Additional paths ----------------- - -The FLINT and Arb header files and library files (libflint.so and libarb.so) -must be available at compile time. If they are in a nonstandard location -(for example, if they have been built but not installed), -use a command such as the following to build:: - - python ./setup.py build_ext --include-dirs=/home/fredrik/src/flint2:/home/fredrik/src/arb --library-dirs=/home/fredrik/src/flint2:/home/fredrik/src/arb - -Likewise, before starting the Python interpreter, tell the linker -where to find the library files using something like:: - - export LD_LIBRARY_PATH=/home/fredrik/src/flint2:/home/fredrik/src/arb:$LD_LIBRARY_PATH - -Build all dependencies from source ----------------------------------- - -From a VCS checkout, to build python-flint and all dependencies from source, -using the exact versions that are tested in CI and used for the binary PyPI -wheels, run the following in a unix shell:: - - source bin/activate - bin/build_dependencies_unix.sh - -The script will download and build GMP, MPFR, FLINT and Arb and build them all -in a ``.local`` directory. The ``bin/activate`` script sets the appropriate -path environment variables for C headers and libraries which is needed for -the ``build_dependencies_unix.sh`` script to work. After running the script, -you can then build Python-FLINT in place with:: - - python setup.py build_ext --in-place - -and run the test suite with:: - - python -m flint.test - -This way of building Python-FLINT depends on the ``bin/activate`` script to -locate the shared libraries at runtime. The script will also set ``PYTHONPATH`` -so that the in-place build of Python-FLINT can be imported. - -These steps will also work under MinGW with the mingw64 toolchain, but you -should first run:: - - echo '[build]' > setup.cfg - echo 'compiler = mingw32' >> setup.cfg - - # Install the mingw-w64 toolchain - pacman -S --noconfirm mingw-w64-x86_64-gcc m4 make mingw-w64-x86_64-tools-git - -To change the versions of the dependencies that are built, edit the -``bin/build_variables.sh`` script. From 2f7dd2274b37a9ffab40cde715a5faf13e6fa954 Mon Sep 17 00:00:00 2001 From: Oscar Benjamin Date: Sun, 1 Sep 2024 14:20:04 +0100 Subject: [PATCH 3/4] doc: rearrange build and install instructions --- doc/source/build.rst | 222 ++++++++++++++++++++++++++--------------- doc/source/install.rst | 62 +++++++++--- 2 files changed, 192 insertions(+), 92 deletions(-) diff --git a/doc/source/build.rst b/doc/source/build.rst index f2c1edf5..2c843e14 100644 --- a/doc/source/build.rst +++ b/doc/source/build.rst @@ -1,6 +1,55 @@ Build from source ================= +.. note:: + The instructions here are for building ``python-flint`` from source. For + most users it is recommended to install prebuilt binaries from ``PyPI`` or + ``conda-forge`` instead. The instructions here are only needed if a binary + is not available for the platform. See :ref:`install_pip_conda`. + + +Simple build instructions +------------------------- + +The simple explanation of how to build ``python-flint`` from source is that +there are two steps: + +- Install ``FLINT >= 3.0`` (see :ref:`install_dependencies` below). +- Run ``pip install --no-binary python-flint python-flint``. + +For example on Ubuntu 24.04 (but not older versions of Ubuntu) and when installing +``python-flint >= 0.7.0`` these two steps are:: + + sudo apt-get install libflint-dev + pip install --no-binary python-flint python-flint + +The first command installs ``FLINT 3.0.1`` system-wide. With the second command +``pip`` will download the source code for the latest release of +``python-flint`` from PyPI, build it and install it into the active Python +environment. When building, ``pip`` will create a temporary isolated build +environment and will install the Python build dependencies (``Cython``, +``meson``, ...) into this environment so it is not necessary to install them +manually before running ``pip install``. + +If you have the source code locally then you can build and install with:: + + pip install path/to/python-flint-directory-or-archive + +After installing from source it is recommended to run the tests to check that +everything is working correctly as described in :ref:`test_installation`. + +The remainder of this page provides more detailed instructions for building +``python-flint`` from source including how to install the dependencies, how to +build older versions of ``python-flint`` (``< 0.7.0``), how to install from +git, and other more advanced topics. + +.. note:: + If you have more than one Python environment in your system then you need to + ensure that you are installing ``python-flint`` into the correct one. This + may require using the full path to ``pip`` or something like ``python3 -m + pip`` or you may need to activate the environment first. + + .. _supported_versions: Compatibility and supported versions @@ -36,33 +85,40 @@ Compatible versions (note that 0.7.0 is not yet released): * - 0.6.0 - 1st Feb 2024 - 3.9-3.12 - - 3.0 only - - 3.0 only + - 3.0 + - 3.0 * - 0.5.0 - 22nd Oct 2023 - 3.9-3.12 - - 3.0 only - - 3.0 only - -If installing binaries from PyPI or conda-forge then the Python (CPython) -version is the only column in this table that matters. The minimum and maximum -versions of Python represent the versions that are tested in CI and for which -binaries are provided on PyPI. It is likely that ``python-flint`` will work -with other versions of Python (particularly older Python versions) but this is -not tested and requires building from source. It is possible that -``conda-forge`` may provide binaries for other versions of Python. + - 3.0 + - 3.0 + * - 0.4.0 + - 8th Aug 2023 + - 3.9-3.11 + - ``2.9.0`` (``Arb 2.23.0``) + - 3.0 + * - 0.3.0 + - 7th Dec 2018 + - older Python versions + - ``< 3.0`` + - ``< 3.0`` + +The minimum and maximum versions of Python represent the versions that are +tested in CI and for which binaries are provided on PyPI. It is likely that +``python-flint`` will work with other versions of Python (particularly older +Python versions) but this is not tested. It is possible that ``conda-forge`` +may provide binaries for other versions of Python. The minimum versions of Cython and FLINT are needed because it is known that python-flint will not even build with older versions of these libraries. The maximum versions of all dependencies are speculative and are based on the versions that are known to work at the time of release. It is possible that -newer versions of Cython and FLINT will work but from ``python-flint 0.4.0`` -through to the time of writing (``python-flint 0.7.0``) minor releases of -Cython, FLINT, or CPython have frequently required changes to the -``python-flint`` source code to be able to build at all. In particular the -following releases of Cython, FLINT and CPython have had changes that would -prevent building all versions of ``python-flint`` existing at the time of the -release: +newer versions of Cython and FLINT will work but unlikely. During the year +following the release of ``python-flint 0.4.0`` every non-patch release of +Cython, FLINT, or CPython has required changes to the ``python-flint`` source +code to be able to build at all. In particular the following releases of +Cython, FLINT and CPython had changes that would prevent building all versions +of ``python-flint`` existing at the time: - Flint 3.0 (Arb and Flint merged, lots of changes) - Flint 3.1 (Function signature for ``fmpz_mod_mat`` changed) @@ -71,12 +127,10 @@ release: - Cython 3.1 (Removal of ``PyInt_*`` functions) - CPython 3.12 (Removal of distutils) -In fact out of python-flint's three core dependencies (CPython, Cython, FLINT), -the only minor release that has not required changes to the python-flint source -code during this time has been CPython ``3.13``. It is expected then that any -future untested ``3.x`` version of Cython, FLINT, or CPython will not be -compatible with past versions of ``python-flint`` which is why the table above -only lists the versions that are known to work. +It is expected then that any future untested ``3.x`` version of Cython, FLINT, +or CPython will not be compatible with past versions of ``python-flint`` which +is why the table above lists the versions that were known to work at the time +of release. As of python-flint 0.7.0, CPython ``3.13t`` free-threaded builds are tested in CI but wheels are not provided on PyPI. There are no known issues related to @@ -90,21 +144,19 @@ implementations may work but are not tested. .. _build_from_source: -Building from source --------------------- +Building python-flint from source +--------------------------------- .. note:: - Building from source is not the recommended way for most users to install - ``python-flint``. For most users it is recommended to use the binaries from - ``PyPI`` or ``conda-forge`` except in cases where a binary is not available - for the platform. See :ref:`install_pip_conda`. + The instructions here are for building ``python-flint`` from source. For + most users it is recommended to install prebuilt binaries from ``PyPI`` or + ``conda-forge`` instead. The instructions here are only needed if a binary + is not available for the platform. See :ref:`install_pip_conda`. -.. note:: - The instructions here assume that you want to build and install ``python-flint`` - on a platform for which a binary is not available on PyPI or conda-forge. If - you are interested in contributing to ``python-flint`` then see the + Also if you are working on ``python-flint`` itself then it is not + recommended to install the package - - XXX: Add link to separate development page... + - XXX: Add link to separate development page... The source code for ``python-flint`` is available on `GitHub `_ and source distributions can @@ -117,20 +169,17 @@ from PyPI, then build and install it into the active Python environment:: pip install python-flint -This will download, build and install the latest release of ``python-flint`` -from PyPI. +This will try to install a binary first but will otherwise download, build and +install the latest release of ``python-flint`` from PyPI. If you definitely +want to build from source then you can use the ``--no-binary`` option:: -.. note:: - If you have more than one Python environment on your system then you need to - ensure that you are installing ``python-flint`` into the correct one. This - may require using the full path to ``pip`` or something like ``python3 -m - pip`` or by activating the correct environment first. + pip install --no-binary python-flint python-flint -To install a specific version of ``python-flint`` from PyPI use:: +To install a specific version of ``python-flint`` from PyPI use e.g.:: - pip install python-flint==0.6.0 + pip install python-flint==0.7.0a4 -To download and install the latest ``python-flint`` from git master you can +To build and install the latest ``python-flint`` from git master you can use:: pip install git+https://github.com/flintlib/python-flint.git@master @@ -148,33 +197,23 @@ Alternatively if you would like to build a wheel you can use Note that wheels built in this way will not include the dependencies (unlike those distributed on PyPI) and cannot generally be installed on other systems. -For ``python-flint < 0.6.0`` the source distribution did not include -``pyproject.toml`` and did not list the build requirements. Also for -``python-flint < 0.7.0`` there were no version constraints on the build -requirements. If you are building an older version of ``python-flint`` then you -may need to install the build requirements manually and disable build isolation -with ``--no-build-isolation``:: - - pip install Cython==3.0 setuptools numpy - pip install --no-build-isolation . - -To build without build isolation with ``python-flint >= 0.7.0`` the needed -dependencies are different:: - - pip install Cython==3.0 meson meson-python ninja - pip install --no-build-isolation . - Since ``python-flint 0.7.0`` the build system is ``meson`` and the build requirements and version constraints are listed in ``pyproject.toml``. When using build isolation the build requirements are installed in a temporary virtual environment and so it should not be necessary to install them in the active Python environment before running ``pip install``. -The ``meson`` build system will also detect the version of ``FLINT`` installed -in the system and will fail if it is not a version that was known to be -compatible at time of the release of ``python-flint``. To build against new, -untested versions of ``FLINT`` you can pass the ``-Dflint_version_check=false`` -option to the ``meson`` build system:: +To build without build isolation with ``python-flint >= 0.7.0`` the +dependencies should first be installed in the active Python environment:: + + pip install Cython==3.0 meson meson-python ninja + pip install --no-build-isolation . + +The ``meson`` build system will detect the versions of ``FLINT`` and Cython +installed in the system and will fail if they are not versions that were known +to be compatible at the time of the release of ``python-flint``. To build +against new, untested versions of ``FLINT`` or Cython you can pass the +``-Dflint_version_check=false`` option to the ``meson`` build system:: pip install --config-settings=setup-args="-Dflint_version_check=false" . @@ -184,6 +223,27 @@ example if you want to build ``python-flint`` against the latest git version of and Cython that are supported by each version of ``python-flint``. +Building older versions of python-flint +--------------------------------------- + +For ``python-flint < 0.6.0`` the source distribution did not include +``pyproject.toml`` and did not list the build requirements. Also for +``python-flint < 0.7.0`` the build requirements were different and there were +no version constraints listed on the dependencies. An list of the build +requirements for older versions of ``python-flint`` is given above in +:ref:`supported_versions`. + +For ``python-flint < 0.7.0`` you will need to install the build requirements +manually, pin the version of Cython, and disable build isolation:: + + pip install Cython==3.0 setuptools numpy + pip install --no-build-isolation . + +For ``python-flint < 0.4.0`` older versions of Cython are needed (``<= 0.29``). +If the build fails during the Cython step then it is likely that a different +version of Cython is needed. + + .. _install_dependencies: Installing the dependencies @@ -290,9 +350,9 @@ Building on Windows The instructions in :ref:`install_dependencies` above are for Unix-like systems (e.g. Linux or MacOS). On Windows the dependencies can be built in a similar -way using MSYS2 or WSL. It is also possible to build ``python-flint`` and its -dependencies using MSVC but we do not currently provide instructions for this. -The `conda-forge recipe +way using MSYS2 or under WSL. It is also possible to build ``python-flint`` and +its dependencies using MSVC but we do not currently provide instructions for +this. The `conda-forge recipe `_ for ``python-flint`` builds on Windows using MSVC. @@ -331,7 +391,7 @@ Using ``FLINT`` from a non-standard location For most users it is recommended to use the binaries from ``PyPI`` or ``conda-forge``. See :ref:`install_pip_conda`. The instructions here are also not needed if you have installed ``FLINT`` and its dependencies - system-wide. + system-wide (e.g. using a package manager like ``apt-get`` or ``brew``). If you have installed ``FLINT`` in a non-standard location then you will need to instruct the ``python-flint`` build system where to find it and ensure that @@ -353,8 +413,9 @@ location e.g. ``lib64/pkgconfig``. It is also possible to pass the path to the ``pkg-config`` files to the ``meson-python`` build backend. For example if building with ``pip``:: - pip install --config-settings=setup-args="--pkg-config-path=$(pwd)/.local/lib/pkgconfig" \ - python-flint + pip install \ + --config-settings=setup-args="--pkg-config-path=$(pwd)/.local/lib/pkgconfig" \ + python-flint Setting the path to the ``pkg-config`` files in this way will allow the ``python-flint`` build system to find the ``FLINT`` library at build time. At @@ -376,9 +437,10 @@ platforms this is done automatically by the ``meson`` build system but on others it needs to be enabled explicitly. This can be done by passing the ``-Dadd_flint_rpath=true`` option to the ``meson`` build system:: - pip install --config-settings=setup-args="--pkg-config-path=$(pwd)/.local/lib/pkgconfig" \ - --config-settings=setup-args="-Dadd_flint_rpath=true" \ - python-flint + pip install \ + --config-settings=setup-args="--pkg-config-path=$(pwd)/.local/lib/pkgconfig" \ + --config-settings=setup-args="-Dadd_flint_rpath=true" \ + python-flint For versions of ``python-flint`` before ``0.7.0`` the build system is ``setuptools`` (or ``numpy.distutils`` for ``Python < 3.12``). In this case @@ -447,9 +509,10 @@ running ``python`` again or by restarting the Jupyter kernel. If you have installed ``FLINT`` in a non-standard location then you should set the ``pkg-config`` path as described in :ref:`non_standard_location` above:: - pip install --no-build-isolation \ - --config-settings=setup-args="--pkg-config-path=$(pwd)/.local/lib/pkgconfig" \ - --editable . + pip install + --no-build-isolation \ + --config-settings=setup-args="--pkg-config-path=$(pwd)/.local/lib/pkgconfig" \ + --editable . To fully remove the editable install you can run:: @@ -457,4 +520,3 @@ To fully remove the editable install you can run:: and then delete the ``build`` directory that was created in the root of the ``python-flint`` git repo. - diff --git a/doc/source/install.rst b/doc/source/install.rst index d106dd81..f359fde3 100644 --- a/doc/source/install.rst +++ b/doc/source/install.rst @@ -38,21 +38,25 @@ specifying the version number, for example:: pip install python-flint==0.6.0 -At the time of writing the next release of ``python-flint`` will be ``0.7.0`` -and binaries will be provided for Python 3.10, 3.11, 3.12 and 3.13 for the -following platforms: +After installing ``python-flint`` you can test your installation as described +in :ref:`test_installation` below. + + +Fully supported platforms +------------------------- + +Generally each release of python-flint will be compatible with a range of +Python versions as described in `SPEC 0 +`_. At the time of writing, the +current release of ``python-flint`` is ``0.6.0`` and binaries are provided for +Python 3.9, 3.10 3.11 and 3.12 for the following platforms: - Windows 64-bit (``x86_64``) - MacOS 64-bit Intel and 64-bit ARM (i.e. Apple Silicon) - Linux 64-bit (``x86_64``, ``manylinux``) -Notably, at the time of writing the following platforms do *not* have binaries -available: - -- Windows on ARM. -- Linux aarch64 (``conda-forge`` has binaries but ``PyPI`` does not, see - `gh-105 `_). -- non-glibc Linux distros (``musllinux`` rather than ``manylinux``) +A WASM build of ``python-flint`` is also available for use in the browser +via `Pyodide `_. You can see which binaries are provided for the latest release of ``python-flint`` `from PyPI `_ @@ -61,9 +65,43 @@ and which are provided `from conda-forge of Python and other ``python-flint`` dependencies can be found at :ref:`supported_versions`. -If a binary is not available for your platform then you can build from source -as described in :ref:`build_from_source`. +Platforms without binaries +-------------------------- + +There are many other platforms on which ``python-flint`` works fine but for +which binaries are not provided. If a binary is not available for your platform +then you may be able to build from source as described in +:ref:`build_from_source`. + +Notably, at the time of writing the following platforms do *not* have binaries +available but ``python-flint`` should work if built from source: + +- Linux aarch64 (``conda-forge`` has binaries but ``PyPI`` does not, see + `gh-105 `_). +- non-glibc Linux distros (e.g. ``musllinux`` rather than ``manylinux``) +- PyPy + +Binaries for Linux aarch64 will likely be added in future when the platform is +available for testing in CI. + + +Unsupported platforms +--------------------- + +It is *not* known or expected that ``python-flint`` will currently work on: + +- Windows on ARM (has never been tested) +- Any 32-bit platform (previously worked but has not been tested for some + time) +- Other Python implementations besides CPython and PyPy (e.g. GraalPython, + Jython, IronPython) + +Support for Windows on ARM will likely be added in future when the platform is +available for testing in CI. + + +.. _test_installation: Testing your installation ------------------------- From 8c0a2ed03d7740b88fe226a446518217026ae907 Mon Sep 17 00:00:00 2001 From: Oscar Benjamin Date: Sun, 1 Sep 2024 19:16:54 +0100 Subject: [PATCH 4/4] doc: Add a development workflow guide --- doc/source/build.rst | 93 ++++--- doc/source/index.rst | 1 + doc/source/install.rst | 2 +- doc/source/workflow.rst | 522 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 567 insertions(+), 51 deletions(-) create mode 100644 doc/source/workflow.rst diff --git a/doc/source/build.rst b/doc/source/build.rst index 2c843e14..9861d50d 100644 --- a/doc/source/build.rst +++ b/doc/source/build.rst @@ -1,5 +1,5 @@ -Build from source -================= +Install/Build from source +========================= .. note:: The instructions here are for building ``python-flint`` from source. For @@ -8,13 +8,15 @@ Build from source is not available for the platform. See :ref:`install_pip_conda`. +.. _simple_build_instructions: + Simple build instructions ------------------------- The simple explanation of how to build ``python-flint`` from source is that there are two steps: -- Install ``FLINT >= 3.0`` (see :ref:`install_dependencies` below). +- Install ``FLINT >= 3.0`` (see :ref:`installing_the_dependencies` below). - Run ``pip install --no-binary python-flint python-flint``. For example on Ubuntu 24.04 (but not older versions of Ubuntu) and when installing @@ -38,10 +40,15 @@ If you have the source code locally then you can build and install with:: After installing from source it is recommended to run the tests to check that everything is working correctly as described in :ref:`test_installation`. -The remainder of this page provides more detailed instructions for building -``python-flint`` from source including how to install the dependencies, how to -build older versions of ``python-flint`` (``< 0.7.0``), how to install from -git, and other more advanced topics. +The remainder of this page provides more detailed instructions for: + +- :ref:`supported_versions`. +- :ref:`building_from_source`. +- :ref:`building_older_versions`. +- :ref:`installing_the_dependencies`. +- :ref:`building_on_windows`. +- :ref:`non_standard_location`. +- :ref:`editable_install`. .. note:: If you have more than one Python environment in your system then you need to @@ -142,10 +149,10 @@ It is also possible to build and use python-flint for PyPy. Other Python implementations may work but are not tested. -.. _build_from_source: +.. _building_from_source: -Building python-flint from source ---------------------------------- +Installing python-flint from source +----------------------------------- .. note:: The instructions here are for building ``python-flint`` from source. For @@ -154,18 +161,18 @@ Building python-flint from source is not available for the platform. See :ref:`install_pip_conda`. Also if you are working on ``python-flint`` itself then it is not - recommended to install the package - - - XXX: Add link to separate development page... + recommended to install the package as described here. Instead see the + :ref:`development_workflow` page for how to work on ``python-flint``. The source code for ``python-flint`` is available on `GitHub `_ and source distributions can be downloaded from PyPI. To build from source you must first install the dependencies (see -:ref:`install_dependencies` below for instructions). Once the dependencies are -installed the following command will download the ``python-flint`` source code -from PyPI, then build and install it into the active Python environment:: +:ref:`installing_the_dependencies` below for instructions). Once the +dependencies are installed the following command will download the +``python-flint`` source code from PyPI, then build and install it into the +active Python environment:: pip install python-flint @@ -223,8 +230,10 @@ example if you want to build ``python-flint`` against the latest git version of and Cython that are supported by each version of ``python-flint``. -Building older versions of python-flint ---------------------------------------- +.. _building_older_versions: + +Installing older versions from source +------------------------------------- For ``python-flint < 0.6.0`` the source distribution did not include ``pyproject.toml`` and did not list the build requirements. Also for @@ -244,7 +253,7 @@ If the build fails during the Cython step then it is likely that a different version of Cython is needed. -.. _install_dependencies: +.. _installing_the_dependencies: Installing the dependencies --------------------------- @@ -321,26 +330,11 @@ again be able to install ``python-flint`` with:: pip install . -In the ``python-flint`` git repo there is a script -`bin/build_dependencies_unix.sh -`_ -which will download and build GMP, MPFR and FLINT and install them in the -current directory under ``~/.local``. The versions used and the installation -directory can be changed by editing the `bin/build_variables.sh -`_ -script. This script is useful for building ``python-flint`` on systems where -the system-wide ``FLINT`` is too old or if precise control over the versions of -GMP, MPFR and FLINT is needed. This script is used for building the binaries -for PyPI and also takes care of ensuring that ``GMP`` and ``FLINT`` are built -as redistributable shared libraries (this is not the default behaviour of the -``configure`` scripts for these libraries and disables some optimisation -features of ``FLINT`` on some ``x86_64`` micro-architectures). Since this -installation is not system-wide, see :ref:`non_standard_location` below for -instructions on how to build and use ``python-flint`` in this case. - - -Building on Windows -------------------- + +.. _building_on_windows: + +Installing from source on Windows +--------------------------------- .. note:: Building from source is not the recommended way for most users to install @@ -348,7 +342,7 @@ Building on Windows use the binaries from ``PyPI`` or ``conda-forge`` except in cases where a binary is not available for the platform. See :ref:`install_pip_conda`. -The instructions in :ref:`install_dependencies` above are for Unix-like systems +The instructions in :ref:`installing_the_dependencies` above are for Unix-like systems (e.g. Linux or MacOS). On Windows the dependencies can be built in a similar way using MSYS2 or under WSL. It is also possible to build ``python-flint`` and its dependencies using MSVC but we do not currently provide instructions for @@ -464,18 +458,17 @@ perhaps it could be possible to link ``FLINT`` and the other libraries statically into ``python-flint``. -Editable install ----------------- - -.. note:: - For working on ``python-flint`` itself it is not recommended to install the - package into the active Python environment. Instead the development - workflow uses ``spin`` and ``meson`` to manage a local build of - ``python-flint``. See +.. _editable_install: - - XXX: Add link to separate development page... +Installing in editable mode +--------------------------- - for more information on how to develop ``python-flint``. +.. note:: + For working on ``python-flint`` itself it is not recommended to install the + package into the active Python environment. Instead the development workflow + uses ``spin`` and ``meson`` to manage a local build of ``python-flint``. See + the :ref:`development_workflow` page for more information on how to develop + ``python-flint``. If you are building and testing ``python-flint`` while working on another project then it may be useful to install ``python-flint`` in editable mode. diff --git a/doc/source/index.rst b/doc/source/index.rst index 747d6512..410b04ba 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -31,6 +31,7 @@ Introduction install.rst general.rst build.rst + workflow.rst Reference diff --git a/doc/source/install.rst b/doc/source/install.rst index f359fde3..6e4ccef8 100644 --- a/doc/source/install.rst +++ b/doc/source/install.rst @@ -72,7 +72,7 @@ Platforms without binaries There are many other platforms on which ``python-flint`` works fine but for which binaries are not provided. If a binary is not available for your platform then you may be able to build from source as described in -:ref:`build_from_source`. +:ref:`simple_build_instructions`. Notably, at the time of writing the following platforms do *not* have binaries available but ``python-flint`` should work if built from source: diff --git a/doc/source/workflow.rst b/doc/source/workflow.rst new file mode 100644 index 00000000..b9fa2780 --- /dev/null +++ b/doc/source/workflow.rst @@ -0,0 +1,522 @@ +.. _development_workflow: + +Development workflow +==================== + +.. note:: + The instructions here are for developing ``python-flint`` e.g. if you want + to contribute to ``python-flint`` such as by making changes to its code. + + For most users it is recommended to install prebuilt binaries from ``PyPI`` + or ``conda-forge`` as explained in :ref:`install_pip_conda`. If you are just + trying to build and install ``python-flint`` from source, see + :ref:`simple_build_instructions`. + + +Outline of building ``python-flint`` for development +---------------------------------------------------- + +The ``python-flint`` project is a Python wrapper around the `FLINT +`_ library. The FLINT library is a C library that +depends on `GMP `_ and `MPFR `_. +The ``python-flint`` codebase is almost entirely written in `Cython +`_ which is a superset of Python that allows writing C +extensions for Python. The Cython code in ``python-flint`` is used to be able +to call the C functions in the FLINT library from Python. + +To develop ``python-flint`` we need to be able to build it which means that we +need to have the FLINT library installed, and to have a C compiler and also the +Python build dependencies such as Cython installed. Installing the FLINT +library is covered in :ref:`installing_the_dependencies` although you may want +to use a local build of FLINT for development as is explained in +:ref:`local_dependency_install` below. + +Once the dependencies are installed, ``python-flint`` itself needs to be built +which happens in four steps: + +1. The Cython code in ``python-flint`` is compiled to C code using ``cython``. +2. The C code generated by ``cython`` is compiled to Python extension modules + (shared libraries) using a C compiler and these are linked against the FLINT + library and other dependencies. +3. The extension modules and Python files are all assembled into a Python + package directory. +4. The Python package directory is bundled into a wheel (a zip file). + +When building to install the next step would be: + +5. The wheel is installed using a Python package installer such as ``pip``. + +When building for distribution on PyPI the next steps would instead be: + +5. All of the dependencies such as the FLINT library are bundled into the + wheel. +6. The wheel is uploaded to ``PyPI``. +7. Users install the wheel from PyPI (``pip install python-flint``). + +The ``python-flint`` project uses ``meson``, ``meson-python`` and ``spin`` to +manage steps 1-4 and other development tasks like running tests. For local +development, only steps 1-3 are needed and then the package directory is used +directly to run tests or other commands. Using ``meson`` and ``spin`` for +developing ``python-flint`` is explained below. + + +.. _local_dependency_install: + +Building and installing dependencies locally +-------------------------------------------- + +It can be useful to build and install the dependencies of ``python-flint`` +locally rather than system-wide. This is useful for development because it +allows you to have precise control over the versions of the dependencies used +for development without conflicting with system-wide installations. + +In the ``python-flint`` git repo there is a script +`bin/build_dependencies_unix.sh +`_ +which will download and build GMP, MPFR and FLINT and install them under the +current directory in a subdirectory called ``.local``. The versions used and +the installation directory can be changed by editing the +`bin/build_variables.sh +`_ +script. This script is useful for building ``python-flint`` on systems where +the system-wide ``FLINT`` is too old or if precise control over the versions of +GMP, MPFR and FLINT is needed (e.g. when developing ``python-flint`` and +testing different versions). This script is used for building the binaries for +PyPI and also takes care of ensuring that ``GMP`` and ``FLINT`` are built as +redistributable shared libraries (this is not the default behaviour of the +``configure`` scripts for these libraries and disables some optimisation +features of ``FLINT`` on some ``x86_64`` micro-architectures). + +Since this local installation is not system-wide, see +:ref:`non_standard_location` and the instructions below for how to configure +``meson`` to use the locally installed dependencies. + + +Meson and spin +-------------- + +The build system for ``python-flint`` uses `meson `_ +and is configured using ``meson.build`` and ``meson.options`` files. When +installing ``python-flint`` with e.g. ``pip install .`` the `meson-python +`_ PEP 517 `build backend +`_ will instruct ``meson`` to build a +``python-flint`` wheel that ``pip`` will then install. + +For development it is preferred not to install ``python-flint`` into the +active environment but to use ``meson`` commands directly to create a local +build and then run tests or other commands using the local build. The `spin +`_ tool is then used as a frontend +to ``meson`` to simplify common development tasks. We will first explain how to +use ``meson`` directly and then show how to use ``spin`` to simplify the +process. This section gives an overview of how ``meson`` and ``spin`` are used +and in the next section we will see how to get started with using these for +``python-flint`` development. + +To build and install a typical (non-Python) project that uses ``meson`` you +would run:: + + meson setup build + meson compile -C build + meson install -C build + +These three commands create a build directory, compile the project, and install +it and are analogous to the autotools commands:: + + ./configure + make + make install + +What each of these commands does is: + +- ``meson setup build``: Create a build directory called ``build``. Options can + be passed to the ``meson setup`` command to configure how the project will be + built. +- ``meson compile -C build``: Build the project and place the built files in + the ``build`` directory. After the initial build, if some code is changed + then ``meson compile`` performs an incremental build which is faster then + rebuilding from scratch. +- ``meson install -C build``: Transfer the built files to the install directory + (e.g. ``/usr/local`` or somewhere). + +In a Python project that uses ``meson``, the ``meson install`` command is not +usually used like this because the ``meson`` build system is typically used to +build e.g. a wheel that is then installed using a Python package installer such +as ``pip``. + +In the ``spin/meson`` workflow for Python projects, we would instead "install" +the project into a local directory with a command like:: + + meson install --only-changed -C build --destdir ../build-install + +This command installs the project into the local ``build-install`` directory +which is a subdirectory of the project root directory. For common development +tasks like running the tests we need to make it so that Python can import this +local build of ``python-flint`` which can be done by either setting +``PYTHONPATH`` or by changing directory to where the local build is installed:: + + cd build-install/usr/lib/python3.12/site-packages + python -m flint.test + +This will run the tests for ``python-flint`` using the local build of +``python-flint``. The ``spin`` tool simplifies this process by providing a +frontend to ``meson`` that can be used to run common development tasks like +running the tests. To run the tests using ``spin`` you can run:: + + spin test + +This will rebuild the project if necessary and then run the tests using the +local build of ``python-flint``. The ``spin`` tool will show what commands it +is effectively running so you can see what is happening if you want to run the +commands directly. In this case ``spin test`` is roughly equivalent to:: + + meson compile -C build # rebuild + meson install --only-changed -C build --destdir ../build-install + export PYTHONPATH="/path/to/python-flint/build-install/usr/lib/python3.12/site-packages" + python -m pytest --pyargs flint + +After any change to the code, common development tasks such as running the +tests require the project to be built and installed first. With ``spin`` and +``meson`` we emulate this without needing to perform a full rebuild and without +actually installing the project into any Python environment or system-wide +location. + + +Setting up the development environment +-------------------------------------- + +First create a fork of the `python-flint repository on GitHub +`_. Clone your fork to your local +machine using ``git`` and then change directory into the cloned repository: + +.. code-block:: bash + + git clone git@github.com:your-username/python-flint.git + cd python-flint + +Now add the upstream repository as a remote so that you can pull in changes in +future: + +.. code-block:: bash + + git remote add upstream git@github.com:flintlib/python-flint.git + +.. note:: + The git URLs with ``git@`` are for SSH access to the repository. If you do + not use SSH keys with GitHub then use the HTTPS URLs instead. + +It is worth reading the :ref:`simple_build_instructions` instructions first +because they cover the basic dependencies needed to build ``python-flint`` from +source and :ref:`installing_the_dependencies`. For local development, you may +want to install non-Python dependencies such as ``FLINT`` locally rather than +system-wide in which case the instructions in :ref:`local_dependency_install` +and :ref:`non_standard_location` are also useful. + +It is also useful to use a `virtual environment +`_ to manage the Python-level +dependencies for ``python-flint`` so that it is kept separate from other Python +environments on your system. You can create and activate a virtual environment +using e.g.: + +.. code-block:: bash + + python3 -m venv venv + source venv/bin/activate + +Now all commands such as ``pip`` and ``python`` will use this activated virtual +environment. You can install the Python development dependencies using: + +.. code-block:: bash + + pip install -r requirements-dev.txt + +This will install the dependencies such as ``cython``, ``meson``, etc that are +needed for building and developing ``python-flint`` into the activated virtual +environment. + +The first step in developing ``python-flint`` is to build it and the first step +in building it is to configure the build using ``meson setup`` (or ``spin +build``). If you have already installed ``FLINT`` system-wide then you can run: + +.. code-block:: bash + + meson setup build + +This will check the system for the dependencies needed to build +``python-flint`` such as ``FLINT`` and ``GMP`` and ``MPFR``. It will also check +for C compilers and for ``Cython``. If setup was successful then you can now +build ``python-flint`` with: + +.. code-block:: bash + + meson compile -C build + +By default, ``python-flint``'s build configuration will reject newer versions +of ``FLINT`` or ``Cython`` than the ones that are known to work. If you want to +override this behaviour (e.g. because you have ``FLINT`` or ``Cython`` from a +newer version or latest git) then you can pass the +``-Dflint_version_check=false`` option: + +.. code-block:: bash + + meson setup build -Dflint_version_check=false + +If you have installed the dependencies in a non-standard location then you +need to tell ``meson`` where to find them when running ``meson setup``. For +example, if you have installed ``FLINT`` in a directory called +``/some/dir/lib`` then you can run: + +.. code-block:: bash + + meson setup build \ + --pkg-config-path=/some/dir/lib/pkgconfig \ + -Dadd_flint_rpath=true + +This tells ``meson`` to look for the ``pkg-config`` files such as ``flint.pc`` +in the ``/some/dir/lib/pkgconfig`` directory and to add the ``/some/dir/lib`` +directory to the runtime library search path in the ``python-flint`` extension +modules. The ``add_flint_rpath`` option may not be needed depending on your OS. + +Usually it is not necessary to use ``meson`` directly as shown above becuase +the ``spin`` tool provides a frontend to ``meson`` that combines common steps. +The ``spin build`` command can be used to setup and build the project in one +step: + +.. code-block:: bash + + spin build -- -Dflint_version_check=false + + # Equivalent to: + + meson setup build -Dflint_version_check=false + meson compile -C build + meson install --only-changed -C build --destdir ../build-install + +Most ``spin`` commands are primarily a wrapper for some other command (not +necessarily a ``meson`` command) and will pass any additional arguments +through. In this case the ``-Dflint_version_check=false`` option is passed to +the ``meson setup`` command. + +The ``spin build`` command is the one case where it is recommended to use +``meson`` directly instead of using ``spin``. For some reason ``spin build`` +does not always configure the project correctly and so the recommended way is: + +.. code-block:: bash + + meson setup build -Dflint_version_check=false + spin build + +After an initial call to ``meson setup`` all subsequent tasks can use ``spin`` +which will automatically rebuild the project when needed. For example, to run +the tests you can run: + +.. code-block:: bash + + spin run python -m flint.test + +This will build or rebuild ``python-flint`` if necessary and then run the +tests. This is equivalent to installing ``python-flint`` and then running the +same command e.g.: + +.. code-block:: bash + + pip install . + python -m flint.test + +More generally the ``spin run`` command can be used to run any command in the +local build environment as if ``python-flint`` was installed. + + +Common development tasks +------------------------ + +The most common development task is to rebuild the project and run the tests +and there are a few ways to do this. The most straight-forward way is + +.. code-block:: bash + + spin test + +The ``spin test`` command will rebuild the project if necessary and then run +``pytest``. Additional arguments can be passed to ``pytest`` by using the +``--`` separator e.g.: + +.. code-block:: bash + + spin test -- -k test_name # run only tests that match 'test_name' + spin test -- --pdb # drop into the debugger on test failure + +Note though that there are two kinds of tests in ``python-flint``: + +1. The general tests in the ``flint/test`` directory. +2. The doctests in the docstrings throughout the codebase (and also in the + docs). + +The ``spin test`` command only runs the general tests but not the doctests. To +run both you can use ``python -m flint.test`` when ``python-flint`` is +installed but in the development environment you can use: + +.. code-block:: bash + + spin run python -m flint.test # run all tests and doctests + +The two most useful ``spin`` commands are: + +- ``meson setup build``: Configure the project. +- ``spin run python -m flint.test``: Run all tests and doctests. + +Other useful ``spin`` commands are: + +- ``spin build``: Build the project. +- ``spin test``: Run the general tests. +- ``spin run``: Run a command in the local build environment. +- ``spin python``: Start a Python shell in the local build environment. +- ``spin ipython``: Start an IPython shell in the local build environment. +- ``spin shell``: Start a system shell in the local build environment. +- ``spin docs``: Build the documentation. + +One other command is provided but not recommended for general development: + +- ``spin install``: Install the project editably in the active Python + environment. + +Sometimes it is useful to install the project editably but it can conflict with +other ``spin`` commands. The editable install uses the same ``build`` directory +as the ``spin`` install and so the normal ``spin`` way of doing things is not +compatible with the editable install. You can uninstall the editable install +using ``pip uninstall python-flint`` and then wipe the build directory: + +.. code-block:: bash + + rm -r build + +In future perhaps other ``spin`` commands could be added to ``python-flint``'s +``spin`` configuration. + + +Measuring code coverage +----------------------- + +To measure code coverage it is first necessary to build the Cython code with +coverage enabled. This can be done by passing the ``-Dcoverage=true`` option to +``meson setup`` or ``spin build``. Measuring coverage of Cython code does not +currently work with ``spin`` (`issue +`_). However ``python-flint`` +has a local coverage plugin that can be used to measure coverage of the Cython +code in ``python-flint``. There is a script ``bin/coverage.sh`` that can be +used for this. Its contents are: + +.. code-block:: bash + + spin build -Dcoverage=true + spin run -- coverage run -m flint.test + coverage report -m --sort=cover + coverage html + +Note that the setting ``-Dcoverage=true`` enables tracing in the Cython code. +This considerably slows down the build as well as making ``python-flint`` a +lot slower to run. The setting is persistent and so needs to be explicitly +disabled when no longer needed: + +.. code-block:: bash + + meson setup build -Dcoverage=false + +(Note that this is an example where ``spin build`` is not used because it does +not trigger a rebuild correctly for some reason unlike ``meson setup``.) + + +Building in release mode +------------------------ + +Another setting that is worth configuring is the build type. By default ``meson +setup`` configures a debug mode build which means that the C code is not fully +optimised by the compiler. If you want to measure the performance of +``python-flint`` then you should build in release mode: + +.. code-block:: bash + + meson setup build -Dbuildtype=release + +This will build the C code with full optimisations enabled. Note that building +in release mode takes longer than building in debug mode and so it is not +always convenient for development. As for the coverage setting, the build type +is persistent and so needs to be disabled explicitly when no longer needed: + +.. code-block:: bash + + meson setup build -Dbuildtype=debug + +Note that the build type setting here only applies when compiling the C code +that is generated from the Cython code. This has no effect on the optimisation +level that is used for FLINT or GMP or MPFR. Setting the build type to release +only reduces the overhead of the ``python-flint`` wrapper code (which may or +may not be significant depending on what is being timed). + + +Differences between meson and autotools +--------------------------------------- + +Some differences between ``meson`` and autotools are worth noting for the +benefit of those who are familiar with autotools but not ``meson``. Firstly, +the way that ``meson`` is intended to be used is that many different build +directories can be created like: + +.. code-block:: bash + + meson setup build-debug -Dbuildtype=debug + meson setup build-release -Dbuildtype=release + +This allows different configurations and builds to be kept simultaneously. What +this means though is that all subsequent commands must be told which build +directory to use e.g. ``meson compile -C build-debug``. + +The ``meson configure`` command can be used to view or change the configuration +of a build directory: + +.. code-block:: bash + + meson configure build-debug # view the configuration + meson configure build-debug -Dsome_option=true # change the configuration + +It is expected that ``meson setup`` would only be called once per build +directory and that ``meson configure`` would be used to change the +configuration of an existing build directory: + +.. code-block:: bash + + meson setup build + meson configure build -Dsome_option=true -Dsome_other_option=false + +It is still possible to run ``meson setup`` multiple times (and does work) but +``meson`` complains (needlessly) that the directory is already configured: + +.. code-block:: console + + $ meson setup build --pkg-config-path=.local/lib/pkgconfig -Dadd_flint_rpath=true -Dbuildtype=debug + Directory already configured. + + Just run your build command (e.g. ninja) and Meson will regenerate as necessary. + Run "meson setup --reconfigure to force Meson to regenerate. + + If build failures persist, run "meson setup --wipe" to rebuild from scratch + using the same options as passed when configuring the build. + +Unlike an autotools ``./configure`` script the configuration options passed to +``meson setup`` are persistent and are combined in repeated calls: + +.. code-block:: bash + + meson setup build -Dfirst_option=true + meson setup build -Dsecond_option=false # first_option is still true + +With ``meson`` all generated files are placed in the build directory and the +source directory is kept clean. This means that rather than running e.g. ``make +clean`` you can just delete the build directory (``rm -r build``). Note that +the ``meson setup`` command has a ``--wipe`` option that will delete all of the +built files while keeping the configuration options: + +.. code-block:: bash + + meson setup build -Doption=true + ... + meson setup build --wipe # deletes all built files, option is still true