Trac #31386: Move list of scripts and package_data from setup.py to setup.cfg

Release Manager · Release Manager · commit bd2f4b663235 · 2021-12-22T00:16:50.000+01:00
This will allow us to reduce the duplication between `src/setup.py` (which is currently used when `./configure --enable-editable` is in use) and `build/pkgs/sagelib/src/setup.py`. We also update the developer's guide on the topic of `package_data`. URL: https://trac.sagemath.org/31386 Reported by: mkoeppe Ticket author(s): Matthias Koeppe Reviewer(s): John Palmieri
diff --git a/pkgs/sagemath-standard/setup.py b/pkgs/sagemath-standard/setup.py
@@ -1,7 +1,5 @@
 #!/usr/bin/env python
 
-from __future__ import print_function
-
 import os
 import sys
 import time
@@ -102,82 +100,5 @@
 
 code = setup(
       packages = python_packages,
-      package_data = {
-          'sage.libs.gap': ['sage.gaprc'],
-          'sage.interfaces': ['sage-maxima.lisp'],
-          'sage.doctest':  ['tests/*'],
-          'sage': ['ext_data/*',
-                   'ext_data/kenzo/*',
-                   'ext_data/singular/*',
-                   'ext_data/singular/function_field/*',
-                   'ext_data/images/*',
-                   'ext_data/doctest/*',
-                   'ext_data/doctest/invalid/*',
-                   'ext_data/doctest/rich_output/*',
-                   'ext_data/doctest/rich_output/example_wavefront/*',
-                   'ext_data/gap/*',
-                   'ext_data/gap/joyner/*',
-                   'ext_data/mwrank/*',
-                   'ext_data/notebook-ipython/*',
-                   'ext_data/nbconvert/*',
-                   'ext_data/graphs/*',
-                   'ext_data/pari/*',
-                   'ext_data/pari/dokchitser/*',
-                   'ext_data/pari/buzzard/*',
-                   'ext_data/pari/simon/*',
-                   'ext_data/magma/*',
-                   'ext_data/magma/latex/*',
-                   'ext_data/magma/sage/*',
-                   'ext_data/valgrind/*',
-                   'ext_data/threejs/*']
-      },
-      scripts = [## The sage script
-                 'bin/sage',
-                 ## Other scripts that should be in the path also for OS packaging of sage:
-                 'bin/sage-eval',
-                 'bin/sage-runtests',          # because it is useful for doctesting user scripts too
-                 'bin/sage-fixdoctests',       # likewise
-                 'bin/sage-coverage',          # because it is useful for coverage-testing user scripts too
-                 'bin/sage-cython',            # deprecated, might be used in user package install scripts
-                 ## Helper scripts invoked by sage script
-                 ## (they would actually belong to something like libexec)
-                 'bin/sage-cachegrind',
-                 'bin/sage-callgrind',
-                 'bin/sage-massif',
-                 'bin/sage-omega',
-                 'bin/sage-valgrind',
-                 'bin/sage-venv-config',
-                 'bin/sage-version.sh',
-                 'bin/sage-cleaner',
-                 ## Only makes sense in sage-the-distribution. TODO: Move to another installation script.
-                 'bin/sage-list-packages',
-                 'bin/sage-location',
-                 ## Uncategorized scripts in alphabetical order
-                 'bin/math-readline',
-                 'bin/sage-env',
-                 # sage-env-config -- installed by sage_conf
-                 # sage-env-config.in -- not to be installed',
-                 'bin/sage-gdb-commands',
-                 'bin/sage-grep',
-                 'bin/sage-grepdoc',
-                 'bin/sage-inline-fortran',
-                 'bin/sage-ipynb2rst',
-                 'bin/sage-ipython',
-                 'bin/sage-native-execute',
-                 'bin/sage-notebook',
-                 'bin/sage-num-threads.py',
-                 'bin/sage-open',
-                 'bin/sage-preparse',
-                 'bin/sage-python',
-                 'bin/sage-rebase.bat',
-                 'bin/sage-rebase.sh',
-                 'bin/sage-rebaseall.bat',
-                 'bin/sage-rebaseall.sh',
-                 'bin/sage-run',
-                 'bin/sage-run-cython',
-                 'bin/sage-startuptime.py',
-                 'bin/sage-update-src',
-                 'bin/sage-update-version',
-                 ],
       cmdclass = cmdclass,
       ext_modules = cython_modules)
diff --git a/src/doc/en/developer/coding_basics.rst b/src/doc/en/developer/coding_basics.rst
@@ -152,17 +152,31 @@ Then in the file ``SAGE_ROOT/src/sage/all.py``, add a line ::
 
     from sage.measure_theory.all import *
 
-Non-Python Sage source code and supporting files should be placed in
-appropriate subdirectories of ``SAGE_ROOT/src/sage/ext_data/``. They will then be
-automatically copied to the corresponding subdirectories of
-``SAGE_ROOT/local/share/sage/ext/`` during the build process and can be
-accessed at runtime using ``SAGE_EXTCODE``.  For example, if ``file`` is placed
-in ``SAGE_ROOT/src/sage/ext_data/directory/`` it can be accessed with ::
+Non-Python Sage source code and supporting files can be included in one
+of the following places:
+
+- In the directory of the Python code that uses that file.  When the
+  Sage library is installed, the file will be installed in the same
+  location as the Python code. For example,
+  ``SAGE_ROOT/src/sage/interfaces/maxima.py`` needs to use the file
+  ``SAGE_ROOT/src/sage/interfaces/maxima.lisp`` at runtime, so it refers
+  to it as ::
+
+    os.path.join(os.path.dirname(__file__), 'sage-maxima.lisp')
+
+- In an appropriate subdirectory of ``SAGE_ROOT/src/sage/ext_data/``.
+  (At runtime, it is then available in the directory indicated by
+  ``SAGE_EXTCODE``).  For example, if ``file`` is placed in
+  ``SAGE_ROOT/src/sage/ext_data/directory/`` it can be accessed with ::
 
     from sage.env import SAGE_EXTCODE
     file = os.path.join(SAGE_EXTCODE, 'directory', 'file')
 
-``SAGE_EXTCODE`` is used because not all distributions have ``SAGE_ROOT``.
+In both cases, the files must be listed (explicitly or via wildcards) in
+the section ``options.package_data`` of the file
+``SAGE_ROOT/pkgs/sagemath-standard/setup.cfg.m4`` (or the corresponding
+file of another distribution).
+
 
 
 Learn by copy/paste
diff --git a/src/setup.cfg.m4 b/src/setup.cfg.m4
@@ -31,7 +31,7 @@ install_requires =
     sage_conf
     esyscmd(`sage-get-system-packages install-requires \
         six \
-        | sed "2,\$s/^/    /;"')dnl
+        | sed "2,\$s/^/    /;"')dnl'
 dnl From build/pkgs/sagelib/dependencies
     esyscmd(`sage-get-system-packages install-requires \
         cypari         \
@@ -44,12 +44,12 @@ dnl From build/pkgs/sagelib/dependencies
         pkgconfig      \
         pplpy          \
         memory_allocator \
-        | sed "2,\$s/^/    /;"')dnl
+        | sed "2,\$s/^/    /;"')dnl'
 dnl From Makefile.in: SAGERUNTIME
     esyscmd(`sage-get-system-packages install-requires \
         ipython        \
         pexpect        \
-        | sed "2,\$s/^/    /;"')dnl
+        | sed "2,\$s/^/    /;"')dnl'
 dnl From Makefile.in: DOC_DEPENDENCIES
     esyscmd(`sage-get-system-packages install-requires \
         sphinx         \
@@ -62,10 +62,98 @@ dnl From Makefile.in: DOC_DEPENDENCIES
         ipykernel      \
         jupyter_client \
         ipywidgets     \
-        | sed "2,\$s/^/    /;"')dnl
+        | sed "2,\$s/^/    /;"')dnl'
 dnl Other Python packages that are standard spkg, used in doctests
     esyscmd(`sage-get-system-packages install-requires \
         rpy2           \
         fpylll         \
-        | sed "2,\$s/^/    /;"')dnl
+        | sed "2,\$s/^/    /;"')dnl'
 dnl pycryptosat  # Sage distribution installs it as part of cryptominisat. According to its README on https://pypi.org/project/pycryptosat/: "The pycryptosat python package compiles while compiling CryptoMiniSat. It cannot be compiled on its own, it must be compiled at the same time as CryptoMiniSat."
+
+scripts =
+    # The sage script
+    bin/sage
+    # Other scripts that should be in the path also for OS packaging of sage:
+    bin/sage-eval
+    # Included because it is useful for doctesting/coverage testing user scripts too:
+    bin/sage-runtests
+    bin/sage-fixdoctests
+    bin/sage-coverage
+    # The following is deprecated but might still be used in user package install scripts
+    bin/sage-cython
+    # Helper scripts invoked by sage script
+    # (they would actually belong to something like libexec)
+    bin/sage-cachegrind
+    bin/sage-callgrind
+    bin/sage-massif
+    bin/sage-omega
+    bin/sage-valgrind
+    bin/sage-venv-config
+    bin/sage-version.sh
+    bin/sage-cleaner
+    # Only makes sense in sage-the-distribution. TODO: Move to another installation script.
+    bin/sage-list-packages
+    bin/sage-location
+    # Uncategorized scripts in alphabetical order
+    bin/math-readline
+    bin/sage-env
+    # sage-env-config -- installed by sage_conf
+    # sage-env-config.in -- not to be installed
+    bin/sage-gdb-commands
+    bin/sage-grep
+    bin/sage-grepdoc
+    bin/sage-inline-fortran
+    bin/sage-ipynb2rst
+    bin/sage-ipython
+    bin/sage-native-execute
+    bin/sage-notebook
+    bin/sage-num-threads.py
+    bin/sage-open
+    bin/sage-preparse
+    bin/sage-python
+    bin/sage-rebase.bat
+    bin/sage-rebase.sh
+    bin/sage-rebaseall.bat
+    bin/sage-rebaseall.sh
+    bin/sage-run
+    bin/sage-run-cython
+    bin/sage-startuptime.py
+    bin/sage-update-src
+    bin/sage-update-version
+
+[options.package_data]
+
+sage.libs.gap =
+    sage.gaprc
+
+sage.interfaces =
+    sage-maxima.lisp
+
+sage.doctest =
+    tests/*
+
+sage =
+    ext_data/*
+    ext_data/kenzo/*
+    ext_data/singular/*
+    ext_data/singular/function_field/*
+    ext_data/images/*
+    ext_data/doctest/*
+    ext_data/doctest/invalid/*
+    ext_data/doctest/rich_output/*
+    ext_data/doctest/rich_output/example_wavefront/*
+    ext_data/gap/*
+    ext_data/gap/joyner/*
+    ext_data/mwrank/*
+    ext_data/notebook-ipython/*
+    ext_data/nbconvert/*
+    ext_data/graphs/*
+    ext_data/pari/*
+    ext_data/pari/dokchitser/*
+    ext_data/pari/buzzard/*
+    ext_data/pari/simon/*
+    ext_data/magma/*
+    ext_data/magma/latex/*
+    ext_data/magma/sage/*
+    ext_data/valgrind/*
+    ext_data/threejs/*
diff --git a/src/setup.py b/src/setup.py
@@ -124,84 +124,6 @@
 # ########################################################
 code = setup(
     packages=python_packages,
-    package_data={
-          'sage.libs.gap': ['sage.gaprc'],
-          'sage.interfaces': ['sage-maxima.lisp'],
-          'sage.doctest':  ['tests/*'],
-          'sage': ['ext_data/*',
-                   'ext_data/kenzo/*',
-                   'ext_data/singular/*',
-                   'ext_data/singular/function_field/*',
-                   'ext_data/images/*',
-                   'ext_data/doctest/*',
-                   'ext_data/doctest/invalid/*',
-                   'ext_data/doctest/rich_output/*',
-                   'ext_data/doctest/rich_output/example_wavefront/*',
-                   'ext_data/gap/*',
-                   'ext_data/gap/joyner/*',
-                   'ext_data/mwrank/*',
-                   'ext_data/notebook-ipython/*',
-                   'ext_data/nbconvert/*',
-                   'ext_data/graphs/*',
-                   'ext_data/pari/*',
-                   'ext_data/pari/dokchitser/*',
-                   'ext_data/pari/buzzard/*',
-                   'ext_data/pari/simon/*',
-                   'ext_data/magma/*',
-                   'ext_data/magma/latex/*',
-                   'ext_data/magma/sage/*',
-                   'ext_data/valgrind/*',
-                   'ext_data/threejs/*']
-      },
-    scripts=[
-        # The sage script
-        'bin/sage',
-        # Other scripts that should be in the path also for OS packaging of sage:
-        'bin/sage-eval',
-        'bin/sage-runtests',          # because it is useful for doctesting user scripts too
-        'bin/sage-fixdoctests',       # likewise
-        'bin/sage-coverage',          # because it is useful for coverage-testing user scripts too
-        'bin/sage-cython',            # deprecated, might be used in user package install scripts
-        # Helper scripts invoked by sage script
-        # (they would actually belong to something like libexec)
-        'bin/sage-cachegrind',
-        'bin/sage-callgrind',
-        'bin/sage-massif',
-        'bin/sage-omega',
-        'bin/sage-valgrind',
-        'bin/sage-venv-config',
-        'bin/sage-version.sh',
-        'bin/sage-cleaner',
-        # Only makes sense in sage-the-distribution. TODO: Move to another installation script.
-        'bin/sage-list-packages',
-        'bin/sage-location',
-        # Uncategorized scripts in alphabetical order
-        'bin/math-readline',
-        'bin/sage-env',
-        # sage-env-config -- installed by sage_conf
-        # sage-env-config.in -- not to be installed
-        'bin/sage-gdb-commands',
-        'bin/sage-grep',
-        'bin/sage-grepdoc',
-        'bin/sage-inline-fortran',
-        'bin/sage-ipynb2rst',
-        'bin/sage-ipython',
-        'bin/sage-native-execute',
-        'bin/sage-notebook',
-        'bin/sage-num-threads.py',
-        'bin/sage-open',
-        'bin/sage-preparse',
-        'bin/sage-python',
-        'bin/sage-rebase.bat',
-        'bin/sage-rebase.sh',
-        'bin/sage-rebaseall.bat',
-        'bin/sage-rebaseall.sh',
-        'bin/sage-run',
-        'bin/sage-run-cython',
-        'bin/sage-startuptime.py',
-        'bin/sage-update-src',
-        'bin/sage-update-version',
-    ],
     cmdclass={
         "build_ext": sage_build_ext_minimal,
         "install":   sage_install,
