From 904a6c51781b2a1e9ba123ee73667b81e2e22118 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Fri, 21 Feb 2020 09:09:05 +0100 Subject: [PATCH 1/5] [bootstrap] Decode zlib-decompressed data to string on PY3 Fixes #3200 --- easybuild/scripts/bootstrap_eb.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/easybuild/scripts/bootstrap_eb.py b/easybuild/scripts/bootstrap_eb.py index 104790a048..ef30883b9b 100644 --- a/easybuild/scripts/bootstrap_eb.py +++ b/easybuild/scripts/bootstrap_eb.py @@ -1151,8 +1151,10 @@ def main(): 7lwukTHLpfpLDH2GT+yCCf8D2cp1xw== """ if IS_PY3: - DISTRIBUTE_SETUP_PY = DISTRIBUTE_SETUP_PY.encode('ascii') + DISTRIBUTE_SETUP_PY = DISTRIBUTE_SETUP_PY.encode() DISTRIBUTE_SETUP_PY = codecs.decode(codecs.decode(DISTRIBUTE_SETUP_PY, "base64"), "zlib") +if IS_PY3: + DISTRIBUTE_SETUP_PY = DISTRIBUTE_SETUP_PY.decode() # run main function as body of script main() From 18c718b4fadedb86d791005272fb50f6171b44a9 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Fri, 21 Feb 2020 09:35:17 +0100 Subject: [PATCH 2/5] [bootstrap] Move setuptools existance checking to top avoid useless work and errors when setuptools is missing --- easybuild/scripts/bootstrap_eb.py | 41 ++++++++++++++++--------------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/easybuild/scripts/bootstrap_eb.py b/easybuild/scripts/bootstrap_eb.py index ef30883b9b..843de8e918 100644 --- a/easybuild/scripts/bootstrap_eb.py +++ b/easybuild/scripts/bootstrap_eb.py @@ -334,32 +334,33 @@ def check_setuptools(): # So, we'll check things by running commands through os.system rather than importing setuptools directly. cmd_tmpl = "%s -c '%%s' > %s 2>&1" % (sys.executable, outfile) - # check setuptools version - try: - os.system(cmd_tmpl % "import setuptools; print(setuptools.__version__)") - setuptools_ver = LooseVersion(open(outfile).read().strip()) - debug("Found setuptools version %s" % setuptools_ver) - - min_setuptools_ver = '0.6c11' - if setuptools_ver < LooseVersion(min_setuptools_ver): - debug("Minimal setuptools version %s not satisfied, found '%s'" % (min_setuptools_ver, setuptools_ver)) - res = False - except Exception as err: - debug("Failed to check setuptools version: %s" % err) - res = False - os.system(cmd_tmpl % "from setuptools.command import easy_install; print(easy_install.__file__)") out = open(outfile).read().strip() - debug("Location of setuptools' easy_install module: %s" % out) if 'setuptools/command/easy_install' not in out: debug("Module 'setuptools.command.easy_install not found") res = False + else: + debug("Location of setuptools' easy_install module: %s" % out) - if res is None: - os.system(cmd_tmpl % "import setuptools; print(setuptools.__file__)") - setuptools_loc = open(outfile).read().strip() - res = os.path.dirname(os.path.dirname(setuptools_loc)) - debug("Location of setuptools installation: %s" % res) + # check setuptools version + try: + os.system(cmd_tmpl % "import setuptools; print(setuptools.__version__)") + setuptools_ver = LooseVersion(open(outfile).read().strip()) + debug("Found setuptools version %s" % setuptools_ver) + + min_setuptools_ver = '0.6c11' + if setuptools_ver < LooseVersion(min_setuptools_ver): + debug("Minimal setuptools version %s not satisfied, found '%s'" % (min_setuptools_ver, setuptools_ver)) + res = False + except Exception as err: + debug("Failed to check setuptools version: %s" % err) + res = False + + if res is None: + os.system(cmd_tmpl % "import setuptools; print(setuptools.__file__)") + setuptools_loc = open(outfile).read().strip() + res = os.path.dirname(os.path.dirname(setuptools_loc)) + debug("Location of setuptools installation: %s" % res) try: os.remove(outfile) From fff7635edbf44dcea191efc1c41df54fe38ff96d Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Fri, 21 Feb 2020 11:02:56 +0100 Subject: [PATCH 3/5] [bootstrap] Fix for PyPi no longer supporting http:// --- easybuild/scripts/bootstrap_eb.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/easybuild/scripts/bootstrap_eb.py b/easybuild/scripts/bootstrap_eb.py index 843de8e918..b08f7b2c67 100644 --- a/easybuild/scripts/bootstrap_eb.py +++ b/easybuild/scripts/bootstrap_eb.py @@ -376,6 +376,10 @@ def run_easy_install(args): debug("Active setuptools installation: %s" % setuptools.__file__) from setuptools.command import easy_install + # PyPi no longer supports http but setuptools 0.6.x easy_install defaults to that + if LooseVersion(setuptools.__version__) < LooseVersion('0.7.0'): + args.insert(0, '--index-url=https://pypi.python.org/simple') + orig_stdout, orig_stderr = mock_stdout_stderr() try: easy_install.main(args) @@ -433,6 +437,9 @@ def stage0(tmpdir): # silence distribute_setup.py completely by setting high log level threshold txt = re.sub(r'([^\n]*)(# extracting the tarball[^\n]*)', r'\1log.set_verbosity(1000)\n\1\2', txt) + # PyPi no longer supports http + txt = txt.replace('http://', 'https://') + # write distribute_setup.py to file (with correct header) distribute_setup = os.path.join(tmpdir, 'distribute_setup.py') f = open(distribute_setup, "w") From 0cac43f0a05ebfa912dcf7fcc53081c635cddb6d Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Mon, 24 Feb 2020 09:33:51 +0100 Subject: [PATCH 4/5] [Bootstrap] Don't use deprecated dummy toolchain for EB >= 4.0 --- easybuild/scripts/bootstrap_eb.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/easybuild/scripts/bootstrap_eb.py b/easybuild/scripts/bootstrap_eb.py index b08f7b2c67..7fd5dede44 100644 --- a/easybuild/scripts/bootstrap_eb.py +++ b/easybuild/scripts/bootstrap_eb.py @@ -756,6 +756,10 @@ def stage2(tmpdir, templates, install_path, distribute_egg_dir, sourcepath): sources_tmpl = "%(easybuild-framework)s%(easybuild-easyblocks)s%(easybuild-easyconfigs)s" if eb_looseversion < LooseVersion('4.0'): sources_tmpl = "%(vsc-install)s%(vsc-base)s" + sources_tmpl + templates['toolchain'] = EASYBUILD_EASYCONFIG_TOOLCHAIN_PRE4 + else: + templates['toolchain'] = EASYBUILD_EASYCONFIG_TOOLCHAIN + templates.update({ 'source_urls': '\n'.join(["'%s'," % x for x in pkg_urls]), @@ -833,7 +837,7 @@ def stage2(tmpdir, templates, install_path, distribute_egg_dir, sourcepath): eb_spec = { 'name': 'EasyBuild', 'hidden': False, - 'toolchain': {'name': 'dummy', 'version': 'dummy'}, + 'toolchain': templates['toolchain'], 'version': templates['version'], 'versionprefix': '', 'versionsuffix': '', @@ -1000,7 +1004,7 @@ def main(): written in Python that allows you to install software in a structured, repeatable and robust way.\"\"\" -toolchain = {'name': 'dummy', 'version': 'dummy'} +toolchain = %(toolchain)s source_urls = [%(source_urls)s] sources = [%(sources)s] @@ -1018,6 +1022,10 @@ def main(): moduleclass = 'tools' """ +# Toolchain to be used for EasyBuild < 4.0 +EASYBUILD_EASYCONFIG_TOOLCHAIN_PRE4 = {'name': 'dummy', 'version': 'dummy'} +# Toolchain to be used for EasyBuild >= 4.0 (new default) +EASYBUILD_EASYCONFIG_TOOLCHAIN = {'name': 'system', 'version': ''} # check Python version loose_pyver = LooseVersion(python_version()) From 802f600dec30f7ea699a4045c2444998c2a3eb09 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Mon, 24 Feb 2020 10:35:50 +0100 Subject: [PATCH 5/5] [bootstrap] Bump version --- .github/workflows/unit_tests.yml | 2 +- .travis.yml | 2 +- easybuild/scripts/bootstrap_eb.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 855c3254f4..39112ff8f8 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -156,7 +156,7 @@ jobs: EB_BOOTSTRAP_VERSION=$(grep '^EB_BOOTSTRAP_VERSION' easybuild/scripts/bootstrap_eb.py | sed 's/[^0-9.]//g') EB_BOOTSTRAP_SHA256SUM=$(sha256sum easybuild/scripts/bootstrap_eb.py | cut -f1 -d' ') EB_BOOTSTRAP_FOUND="$EB_BOOTSTRAP_VERSION $EB_BOOTSTRAP_SHA256SUM" - EB_BOOTSTRAP_EXPECTED="20200820.01 d490d229a18bd5eaa717bb8d5684d754729143d5e995e35a40c84d03ffb1de50" + EB_BOOTSTRAP_EXPECTED="20200914.01 8b89f828d034f9f0a175a1551f71bc3034db487e781e2f41a0f14b747a72b0ee" test "$EB_BOOTSTRAP_FOUND" = "$EB_BOOTSTRAP_EXPECTED" || (echo "Version check on bootstrap script failed $EB_BOOTSTRAP_FOUND" && exit 1) # test bootstrap script diff --git a/.travis.yml b/.travis.yml index 396fa68c5e..6b9f463fa5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -89,7 +89,7 @@ script: - EB_BOOTSTRAP_VERSION=$(grep '^EB_BOOTSTRAP_VERSION' $TRAVIS_BUILD_DIR/easybuild/scripts/bootstrap_eb.py | sed 's/[^0-9.]//g') - EB_BOOTSTRAP_SHA256SUM=$(sha256sum $TRAVIS_BUILD_DIR/easybuild/scripts/bootstrap_eb.py | cut -f1 -d' ') - EB_BOOTSTRAP_FOUND="$EB_BOOTSTRAP_VERSION $EB_BOOTSTRAP_SHA256SUM" - - EB_BOOTSTRAP_EXPECTED="20200820.01 d490d229a18bd5eaa717bb8d5684d754729143d5e995e35a40c84d03ffb1de50" + - EB_BOOTSTRAP_EXPECTED="20200914.01 8b89f828d034f9f0a175a1551f71bc3034db487e781e2f41a0f14b747a72b0ee" - test "$EB_BOOTSTRAP_FOUND" = "$EB_BOOTSTRAP_EXPECTED" || (echo "Version check on bootstrap script failed $EB_BOOTSTRAP_FOUND" && exit 1) # test bootstrap script - python $TRAVIS_BUILD_DIR/easybuild/scripts/bootstrap_eb.py /tmp/$TRAVIS_JOB_ID/eb_bootstrap diff --git a/easybuild/scripts/bootstrap_eb.py b/easybuild/scripts/bootstrap_eb.py index 7fd5dede44..2954bad851 100644 --- a/easybuild/scripts/bootstrap_eb.py +++ b/easybuild/scripts/bootstrap_eb.py @@ -62,7 +62,7 @@ import urllib.request as std_urllib -EB_BOOTSTRAP_VERSION = '20200820.01' +EB_BOOTSTRAP_VERSION = '20200914.01' # argparse preferrred, optparse deprecated >=2.7 HAVE_ARGPARSE = False