Fix PyInstaller bundling

Our bundled binaries seem to have been not working for some time.

We had two separate issues:
- The `av` dependency (via `aiortc`) wasn't being properly discovered
  by PyInstaller.
  This has been fixed with a hook.
- On Linux stripping of the libraries apparently causes some corruption
  See: pypa/manylinux#119
  Therefore stripping is disabled for now.
- The gevent runtime hook would attempt to start the aio event loop. 
  This caused a warning about an already running loop.

This also fixes:
- Don't include unnecessary av libraries in the bundle
- Remove no longer used GETH_URL and SOLC_URL build args from the
  `bundle-docker` make target.
- On CI: Invoke the built binaries with `--help` as a minimal
  functionality sanity check.

Author: ulope

.circleci/config.yml

@@ -578,6 +578,11 @@ jobs:
             source .circleci/get_archive_tag.sh
             make bundle-docker
             echo ${RELEASE_TYPE}/raiden-${ARCHIVE_TAG}-linux-<<parameters.architecture>>.tar.gz > dist/archive/_LATEST-${RELEASE_TYPE}-linux-<<parameters.architecture>>.txt
+      - run:
+          name: Test if Binary can be launched
+          command: |
+            tar -C /tmp -xf dist/archive/raiden-${ARCHIVE_TAG}-linux-<<parameters.architecture>>.tar.gz
+            /tmp/raiden-${ARCHIVE_TAG}-linux-<<parameters.architecture>> --help
       - persist_to_workspace:
           root: "~"
           paths:
@@ -610,6 +615,10 @@ jobs:
             pyinstaller --noconfirm --clean raiden.spec
             zip -9 --junk-paths dist/archive/raiden-${ARCHIVE_TAG}-macOS-x86_64.zip dist/raiden-${ARCHIVE_TAG}-macOS-x86_64
             echo ${RELEASE_TYPE}/raiden-${ARCHIVE_TAG}-macOS-x86_64.zip > dist/archive/_LATEST-${RELEASE_TYPE}-macOS-x86_64.txt
+      - run:
+          name: Test if Binary can be launched
+          command: |
+            dist/raiden-${ARCHIVE_TAG}-macOS-x86_64 --help
       - persist_to_workspace:
           root: "~"
           paths:

Makefile

@@ -129,7 +129,7 @@ endif
 bundle-docker: ARCHITECTURE_TAG = $(shell docker run --rm python:3.7 uname -m)
 bundle-docker: ARCHIVE_TAG ?= v$(shell python setup.py --version)
 bundle-docker:
-	docker build -t pyinstallerbuilder --build-arg GETH_URL_LINUX=$(GETH_URL_LINUX) --build-arg SOLC_URL_LINUX=$(SOLC_URL_LINUX) --build-arg ARCHITECTURE_TAG=$(ARCHITECTURE_TAG) --build-arg ARCHIVE_TAG=$(ARCHIVE_TAG) $(GITHUB_ACCESS_TOKEN_ARG) -f docker/build.Dockerfile .
+	docker build -t pyinstallerbuilder --build-arg ARCHITECTURE_TAG=$(ARCHITECTURE_TAG) --build-arg ARCHIVE_TAG=$(ARCHIVE_TAG) $(GITHUB_ACCESS_TOKEN_ARG) -f docker/build.Dockerfile .
 	-(docker rm builder)
 	docker create --name builder pyinstallerbuilder
 	mkdir -p dist/archive

docker/build.Dockerfile

@@ -1,25 +1,12 @@
 FROM python:3.7-stretch
 
-# these are defined in .circleci/config.yml and passed here in the makefile
-ARG SOLC_URL_LINUX
-ARG GETH_URL_LINUX
-
 # install dependencies
 RUN apt-get update
 RUN apt-get install -y git-core wget xz-utils build-essential \
     automake pkg-config libtool libffi-dev python3-dev libgmp-dev \
     libavdevice-dev libavfilter-dev libopus-dev libvpx-dev pkg-config \
     libsrtp2-dev
 
-RUN wget -nv -O /usr/bin/solc ${SOLC_URL_LINUX} && \
-    chmod +x /usr/bin/solc
-RUN wget -nv -O /tmp/geth.tar.gz ${GETH_URL_LINUX} && \
-    cd /tmp && \
-    tar xf geth.tar.gz && \
-    mv geth-linux-amd64-*/geth /usr/bin/geth && \
-    rm geth.tar.gz
-
-
 RUN python3 -m venv /venv
 ENV PATH="/venv/bin:$PATH"
 

raiden.spec

@@ -12,18 +12,40 @@ PyInstaller spec file to build single file or dir distributions
 """
 
 # Set to false to produce an exploded single-dir
-ONEFILE = int(os.environ.get('ONEFILE', True))
-
-
-def Entrypoint(dist, group, name, scripts=None, pathex=None, hiddenimports=None, hookspath=None,
-               excludes=None, runtime_hooks=None, datas=None):
+ONEFILE = int(os.environ.get("ONEFILE", True))
+
+# Hack: This is a list of prefixes to be removed from the `binaries`.
+#       We do this to prevent including unnecessary libraries (pyav audio / video dependencies)
+BINARIES_PREFIX_BLOCKLIST = [
+    "libav",
+    "libaom",
+    "libswscale",
+    "libswrescale",
+    "libvorbis",
+    "libx264",
+    "libx265",
+]
+
+
+def Entrypoint(
+    dist,
+    group,
+    name,
+    scripts=None,
+    pathex=None,
+    hiddenimports=None,
+    hookspath=None,
+    excludes=None,
+    runtime_hooks=None,
+    datas=None,
+):
     import pkg_resources
 
     # get toplevel packages of distribution from metadata
     def get_toplevel(dist):
         distribution = pkg_resources.get_distribution(dist)
-        if distribution.has_metadata('top_level.txt'):
-            return list(distribution.get_metadata('top_level.txt').split())
+        if distribution.has_metadata("top_level.txt"):
+            return list(distribution.get_metadata("top_level.txt").split())
         else:
             return []
 
@@ -42,54 +64,75 @@ def Entrypoint(dist, group, name, scripts=None, pathex=None, hiddenimports=None,
     # insert path of the egg at the verify front of the search path
     pathex = [ep.dist.location] + pathex
     # script name must not be a valid module name to avoid name clashes on import
-    script_path = os.path.join(workpath, name + '-script.py')
+    script_path = os.path.join(workpath, name + "-script.py")
     print("creating script for entry point", dist, group, name)
-    with open(script_path, 'w') as fh:
+    with open(script_path, "w") as fh:
         print("import", ep.module_name, file=fh)
-        print("%s.%s()" % (ep.module_name, '.'.join(ep.attrs)), file=fh)
+        print("%s.%s()" % (ep.module_name, ".".join(ep.attrs)), file=fh)
         for package in packages:
             print("import", package, file=fh)
 
-    return Analysis(
+    analysis = Analysis(
         [script_path] + scripts,
         pathex=pathex,
         hiddenimports=hiddenimports,
         hookspath=hookspath,
         excludes=excludes,
         runtime_hooks=runtime_hooks,
-        datas=datas
+        datas=datas,
     )
-
-
-if hasattr(pdb, 'pdb'):
+    # `Analysis.binaries` behaves set-like and matches on the first tuple item (`name`).
+    # Since library names include the version we first build a list of the concrete names
+    # by prefix matching and then subtract that via the set-like behaviour.
+    binaries_to_remove = [
+        (name, None, None)
+        for name, *_ in analysis.binaries
+        if any(name.startswith(blocklist_item) for blocklist_item in BINARIES_PREFIX_BLOCKLIST)
+    ]
+    analysis.binaries -= binaries_to_remove
+    return analysis
+
+
+if hasattr(pdb, "pdb"):
     # pdbpp moves the stdlib pdb to the `pdb` attribute of it's own patched pdb module
     raise RuntimeError(
-        'pdbpp is installed which causes broken PyInstaller builds. Please uninstall it.',
+        "pdbpp is installed which causes broken PyInstaller builds. Please uninstall it.",
     )
 
 
 # We don't need Tk and friends
-sys.modules['FixTk'] = None
+sys.modules["FixTk"] = None
 
-executable_name = 'raiden-{}-{}-{}'.format(
-    os.environ.get('ARCHIVE_TAG', 'v' + get_system_spec()['raiden']),
-    'macOS' if platform.system() == 'Darwin' else platform.system().lower(),
-    platform.machine()
+executable_name = "raiden-{}-{}-{}".format(
+    os.environ.get("ARCHIVE_TAG", "v" + get_system_spec()["raiden"]),
+    "macOS" if platform.system() == "Darwin" else platform.system().lower(),
+    platform.machine(),
 )
 
 a = Entrypoint(
-    'raiden',
-    'console_scripts',
-    'raiden',
-    hookspath=['tools/pyinstaller_hooks'],
+    "raiden",
+    "console_scripts",
+    "raiden",
+    hookspath=["tools/pyinstaller_hooks"],
     runtime_hooks=[
-        'tools/pyinstaller_hooks/runtime_gevent_monkey.py',
-        'tools/pyinstaller_hooks/runtime_encoding.py',
-        'tools/pyinstaller_hooks/runtime_raiden_contracts.py',
+        "tools/pyinstaller_hooks/runtime_gevent_monkey.py",
+        "tools/pyinstaller_hooks/runtime_encoding.py",
+        "tools/pyinstaller_hooks/runtime_raiden_contracts.py",
     ],
-    hiddenimports=['scrypt', 'eth_tester'],
+    hiddenimports=[],
     datas=[],
-    excludes=['FixTk', 'tcl', 'tk', '_tkinter', 'tkinter', 'Tkinter'],
+    excludes=[
+        "_tkinter",
+        "FixTk",
+        "ipython",
+        "jupyter",
+        "jupyter_core",
+        "notebook",
+        "tcl",
+        "tk",
+        "tkinter",
+        "Tkinter",
+    ],
 )
 
 pyz = PYZ(a.pure, a.zipped_data, cipher=None)
@@ -103,10 +146,10 @@ if ONEFILE:
         a.datas,
         name=executable_name,
         debug=False,
-        strip=True,
+        strip=False,
         upx=False,
         runtime_tmpdir=None,
-        console=True
+        console=True,
     )
 else:
     exe = EXE(
@@ -117,7 +160,7 @@ else:
         debug=True,
         strip=False,
         upx=False,
-        console=True
+        console=True,
     )
     coll = COLLECT(
         exe,
@@ -126,5 +169,5 @@ else:
         a.datas,
         strip=False,
         upx=False,
-        name='raiden'
+        name="raiden",
     )

tools/pyinstaller_hooks/hook-av.py

@@ -0,0 +1,4 @@
+from PyInstaller.utils.hooks import collect_submodules
+
+
+hiddenimports = collect_submodules("av")

tools/pyinstaller_hooks/runtime_gevent_monkey.py

@@ -1,7 +1,3 @@
-from gevent.monkey import patch_all  # isort:skip # noqa
+from gevent.monkey import patch_all
 
-patch_all()  # isort:skip # noqa
-
-from raiden.network.transport.matrix.rtc.utils import setup_asyncio_event_loop
-
-setup_asyncio_event_loop()
+patch_all()