diff --git a/.travis.yml b/.travis.yml index c2ec15e..3667bf2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,6 +14,7 @@ env: - EXAMPLE=rss2email - EXAMPLE=scipy - EXAMPLE=tornado + - EXAMPLE=connexion # - EXAMPLE=vulnix matrix: exclude: diff --git a/README.rst b/README.rst index c37455a..5a11a6c 100644 --- a/README.rst +++ b/README.rst @@ -180,6 +180,13 @@ dependency of ``execnet``:: This was you can add or remove any python package. +Including overrides +^^^^^^^^^^^^^^^^^^^ + +Additional to a autogenerated ``requirements_overrides.nix`` file you +can include additional overrides files via the ``-O`` command line +argument. These overrides will be included the same way as your +``requirements_overrides.nix``. Creating default.nix for you project ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/default.nix b/default.nix index 3c92ed6..34179a5 100644 --- a/default.nix +++ b/default.nix @@ -1,4 +1,4 @@ -{ stdenv, fetchurl, zip, makeWrapper, nix, nix-prefetch-scripts +{ stdenv, fetchurl, zip, makeWrapper, nix, nix-prefetch-git, nix-prefetch-hg , pythonPackages , src ? { outPath = ./.; name = "pypi2nix"; } }: @@ -26,7 +26,7 @@ in stdenv.mkDerivation rec { ]; buildInputs = [ pythonPackages.python pythonPackages.flake8 - zip makeWrapper nix.out nix-prefetch-scripts + zip makeWrapper nix.out nix-prefetch-git nix-prefetch-hg ]; doCheck = true; sourceRoot = "."; @@ -48,8 +48,8 @@ in stdenv.mkDerivation rec { patchPhase = '' sed -i -e "s|default='nix-shell',|default='${nix.out}/bin/nix-shell',|" $out/pkgs/pypi2nix/cli.py - sed -i -e "s|nix-prefetch-git|${nix-prefetch-scripts}/bin/nix-prefetch-git|" $out/pkgs/pypi2nix/stage2.py - sed -i -e "s|nix-prefetch-hg|${nix-prefetch-scripts}/bin/nix-prefetch-hg|" $out/pkgs/pypi2nix/stage2.py + sed -i -e "s|nix-prefetch-git|${nix-prefetch-git}/bin/nix-prefetch-git|" $out/pkgs/pypi2nix/stage2.py + sed -i -e "s|nix-prefetch-hg|${nix-prefetch-hg}/bin/nix-prefetch-hg|" $out/pkgs/pypi2nix/stage2.py ''; commonPhase = '' @@ -69,7 +69,9 @@ in stdenv.mkDerivation rec { ''; installPhase = commonPhase + '' - wrapProgram $out/bin/pypi2nix --prefix PYTHONPATH : "$PYTHONPATH" + wrapProgram $out/bin/pypi2nix \ + --prefix PYTHONPATH : "$PYTHONPATH" \ + --prefix PATH : "${nix-prefetch-git}/bin:${nix-prefetch-hg}/bin" ''; shellHook = '' diff --git a/examples/Makefile b/examples/Makefile index bc32b39..d46eded 100644 --- a/examples/Makefile +++ b/examples/Makefile @@ -1,6 +1,8 @@ OS := $(shell uname) PYPI2NIX=./pypi2nix/bin/pypi2nix NIX_PATH=nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/nixpkgs-unstable.tar.gz +NIX_BUILD=nix-build +NIX_SHELL=nix-shell ifeq ($(OS), Darwin) @@ -19,7 +21,8 @@ endif scipy \ pillow \ tornado \ - vulnix + vulnix \ + connexion all: \ awscli_and_requests \ @@ -31,7 +34,8 @@ all: \ scipy \ pillow \ tornado \ - vulnix + vulnix \ + connexion clear: \ awscli_and_requests-clear \ @@ -44,7 +48,8 @@ clear: \ scipy-clear \ pillow-clear \ tornado-clear \ - vulnix-clear + vulnix-clear \ + connexion-clear pypi2nix-clear: @@ -52,13 +57,13 @@ pypi2nix-clear: $(PYPI2NIX): pypi2nix-clear echo "building pypi2nix ..." - nix-build ../release.nix -A build.x86_64-linux -o pypi2nix + $(NIX_BUILD) ../release.nix -A build.x86_64-linux -o pypi2nix pillow: pillow.nix echo "building and testing pillow..." - nix-build pillow.nix -o pillow -A interpreter --show-trace + $(NIX_BUILD) pillow.nix -o pillow -A interpreter --show-trace ./pillow/bin/python -c 'import PIL' - nix-shell pillow.nix -A interpreter --run "python -c 'import PIL'" + $(NIX_SHELL) pillow.nix -A interpreter --run "python -c 'import PIL'" pillow.nix: pillow.txt $(PYPI2NIX) echo "generating pillownix expressions ..." @@ -82,9 +87,9 @@ pillow-clear: empy: empy.nix echo "building and testing empy ..." - nix-build empy.nix -o empy -A interpreter --show-trace + $(NIX_BUILD) empy.nix -o empy -A interpreter --show-trace ./empy/bin/python -c 'import em' - nix-shell empy.nix -A interpreter --run "python -c 'import em'" + $(NIX_SHELL) empy.nix -A interpreter --run "python -c 'import em'" empy.nix: empy.txt $(PYPI2NIX) echo "generating empy nix expressions ..." @@ -107,11 +112,11 @@ empy-clear: lektor: lektor.nix echo "building and testing lektor ..." - nix-build lektor.nix -o lektor -A interpreter --show-trace + $(NIX_BUILD) lektor.nix -o lektor -A interpreter --show-trace ./lektor/bin/python -c 'import lektor' ./lektor/bin/lektor --help - nix-shell lektor.nix -A interpreter --run "python -c 'import lektor'" - nix-shell lektor.nix -A interpreter --run "lektor --help" + $(NIX_SHELL) lektor.nix -A interpreter --run "python -c 'import lektor'" + $(NIX_SHELL) lektor.nix -A interpreter --run "lektor --help" lektor.nix: lektor.txt $(PYPI2NIX) echo "generating lektor nix expressions ..." @@ -136,11 +141,11 @@ lektor-clear: rss2email: rss2email.nix echo "building and testing rss2email ..." - nix-build rss2email.nix -o rss2email -A interpreter --show-trace + $(NIX_BUILD) rss2email.nix -o rss2email -A interpreter --show-trace ./rss2email/bin/python -c 'import rss2email' ./rss2email/bin/r2e --help - nix-build rss2email.nix -A interpreter --run "python -c 'import rss2email'" - nix-build rss2email.nix -A interpreter --run "r2e --help" + $(NIX_BUILD) rss2email.nix -A interpreter --run "python -c 'import rss2email'" + $(NIX_BUILD) rss2email.nix -A interpreter --run "r2e --help" rss2email.nix: rss2email.txt $(PYPI2NIX) echo "generating rss2email nix expressions ..." @@ -163,12 +168,12 @@ rss2email-clear: awscli_and_requests: awscli_and_requests.nix echo "building and testing awscli and requests library ..." - nix-build awscli.nix -o awscli_and_requests -A interpreter --show-trace + $(NIX_BUILD) awscli.nix -o awscli_and_requests -A interpreter --show-trace ./awscli_and_requests/bin/python -c 'import awscli; import requests' PAGER=none ./awscli_and_requests/bin/aws help - nix-shell awscli.nix -A interpreter --run "python -c 'import awscli'" - nix-shell awscli.nix -A interpreter --run "python -c 'import awscli; import requests'" - nix-shell awscli.nix -A interpreter --run "PAGER=none aws help" + $(NIX_SHELL) awscli.nix -A interpreter --run "python -c 'import awscli'" + $(NIX_SHELL) awscli.nix -A interpreter --run "python -c 'import awscli; import requests'" + $(NIX_SHELL) awscli.nix -A interpreter --run "PAGER=none aws help" awscli_and_requests.nix: awscli_and_requests.txt $(PYPI2NIX) echo "generating awscli and requests nix expressions ..." @@ -190,8 +195,8 @@ awscli_and_requests-clear: vulnix: vulnix-clear $(PYPI2NIX) echo "building and testing vulnix..." git clone https://github.com/garbas/vulnix vulnix-src - cd vulnix-src && nix-shell update.nix --argstr pypi2nix "./../$(PYPI2NIX)" - NIX_PATH=$(NIX_PATH) nix-build vulnix-src/default.nix --arg pkgs "import {}" -o vulnix + cd vulnix-src && $(NIX_SHELL) update.nix --argstr pypi2nix "./../$(PYPI2NIX)" + NIX_PATH=$(NIX_PATH) $(NIX_BUILD) vulnix-src/default.nix --arg pkgs "import {}" -o vulnix ./vulnix/bin/vulnix --help vulnix-clear: @@ -201,9 +206,9 @@ vulnix-clear: scipy: scipy.nix echo "building and testing scipy..." - nix-build scipy.nix -o scipy -A interpreter --show-trace + $(NIX_BUILD) scipy.nix -o scipy -A interpreter --show-trace ./scipy/bin/python -c 'import scipy' - nix-shell scipy.nix -A interpreter --run "python -c 'import scipy'" + $(NIX_SHELL) scipy.nix -A interpreter --run "python -c 'import scipy'" scipy.nix: scipy.txt $(PYPI2NIX) echo "generating scipy.nix expressions ..." @@ -221,14 +226,14 @@ scipy-clear: tornado: tornado.nix echo "building and testing tornado..." - nix-build tornado.nix -o tornado -A interpreter --show-trace + $(NIX_BUILD) tornado.nix -o tornado -A interpreter --show-trace --fallback ./tornado/bin/python -c 'import tornado' - nix-shell tornado.nix -A interpreter --run "python -c 'import tornado'" + $(NIX_SHELL) tornado.nix -A interpreter --run "python -c 'import tornado'" if ./string_in_file "69253c820df473407c562a227d0ba36df25018ab" tornado.nix ; then true ; else echo "ERROR: Revision '69253c820df473407c562a227d0ba36df25018ab' not found in tornado.nix!" ; exit 123 ; fi tornado.nix: tornado.txt $(PYPI2NIX) echo "generating tornado.nix expressions ..." - $(PYPI2NIX) -v --basename tornado -r tornado.txt -I $(NIX_PATH) -V "3.5" + $(PYPI2NIX) -v --basename tornado -r tornado.txt -I $(NIX_PATH) -V "3.5" -O 'https://raw.githubusercontent.com/garbas/nixpkgs-python/master/overrides.nix' -O 'git+https://github.com/garbas/nixpkgs-python.git#path=overrides.nix&rev=28ecf4bc8cf5e718862159eb1f324ddcf227dfef' tornado.txt: tornado-clear echo "-e git+git://github.com/tornadoweb/tornado.git@69253c820df473407c562a227d0ba36df25018ab#egg=tornado" > tornado.txt @@ -242,9 +247,9 @@ tornado-clear: ldap: ldap.nix echo "building and testing ldap..." - nix-build ldap.nix -o ldap -A interpreter --show-trace + $(NIX_BUILD) ldap.nix -o ldap -A interpreter --show-trace ./ldap/bin/python -c 'import ldap' - nix-shell ldap.nix -A interpreter --run "python -c 'import ldap'" + $(NIX_SHELL) ldap.nix -A interpreter --run "python -c 'import ldap'" ldap.nix: ldap.txt $(PYPI2NIX) echo "generating ldap.nix expressions ..." @@ -269,9 +274,9 @@ ldap-clear: mercurial: mercurial.nix echo "building and testing mercurial..." - nix-build mercurial.nix -o mercurial -A interpreter --show-trace -j 1 + $(NIX_BUILD) mercurial.nix -o mercurial -A interpreter --show-trace -j 1 ./mercurial/bin/python -c 'import flake8' - nix-shell mercurial.nix -A interpreter --run "python -c 'import flake8'" + $(NIX_SHELL) mercurial.nix -A interpreter --run "python -c 'import flake8'" if ./string_in_file "a209fb6" mercurial.nix ; then true ; else echo "ERROR: Revision 'a209fb6' not found in tornado_generated.nix!"; exit 123; fi mercurial.nix: mercurial.txt $(PYPI2NIX) @@ -289,3 +294,23 @@ mercurial-clear: rm -f mercurial.txt rm -f mercurial.nix rm -f mercurial_frozen.txt + +connexion: connexion.nix + echo "building and testing connexion..." + $(NIX_BUILD) connexion.nix -o connexion -A interpreter --show-trace -j 1 + ./connexion/bin/python -c 'import connexion' + $(NIX_SHELL) connexion.nix -A interpreter --run "python -c 'import connexion'" + +connexion.nix: connexion.txt $(PYPI2NIX) + echo "generating connexion.nix expressions ..." + $(PYPI2NIX) -v --basename connexion -r connexion.txt -I $(NIX_PATH) -V "2.7" -e vcversioner -O "git+https://github.com/garbas/nixpkgs-python.git#path=overrides.nix" + + +connexion.txt: connexion-clear + echo "connexion" > connexion.txt + +connexion-clear: + rm -f connexion + rm -f connexion.txt + rm -f connexion.nix + rm -f connexion_frozen.txt diff --git a/examples/connexion_override.nix b/examples/connexion_override.nix new file mode 100644 index 0000000..9aa93a4 --- /dev/null +++ b/examples/connexion_override.nix @@ -0,0 +1,9 @@ +{ pkgs, python }: + +self: super: { + + "jsonschema" = python.overrideDerivation super."jsonschema" (old: { + propagatedBuildInputs = old.propagatedBuildInputs ++ [self.vcversioner]; + }); + +} diff --git a/src/pypi2nix/cli.py b/src/pypi2nix/cli.py index 6a4dfa7..007a22d 100644 --- a/src/pypi2nix/cli.py +++ b/src/pypi2nix/cli.py @@ -4,6 +4,7 @@ import shutil import tempfile +import pypi2nix.overrides import pypi2nix.stage0 import pypi2nix.stage1 import pypi2nix.stage2 @@ -96,6 +97,13 @@ help=u'Extra Python dependencies needed before the installation' u'to build wheels.' ) +@click.option('-O', '--overrides', + multiple=True, + required=False, + type=pypi2nix.overrides.OVERRIDES_URL, + help=u'Extra expressions that override generated expressions ' + + u'for specific packages', + ) def main(version, verbose, nix_shell, @@ -110,6 +118,7 @@ def main(version, buildout, editable, setup_requires, + overrides ): """SPECIFICATION should be requirements.txt (output of pip freeze). """ @@ -319,6 +328,7 @@ def handle_requirements_file(project_dir, requirements_file): enable_tests=enable_tests, python_version=pypi2nix.utils.PYTHON_VERSIONS[python_version], current_dir=current_dir, + common_overrides=overrides, ) click.echo('') diff --git a/src/pypi2nix/overrides.py b/src/pypi2nix/overrides.py new file mode 100644 index 0000000..37c013a --- /dev/null +++ b/src/pypi2nix/overrides.py @@ -0,0 +1,127 @@ +import json +import subprocess +from urllib.parse import urldefrag, urlparse + +import click + +from .utils import cmd + + +class UnsupportedUrlError(Exception): + pass + + +class OverridesFile(object): + def __init__(self, path): + self.path = path + + expression_template = 'import %(path)s { inherit pkgs python ; }' + + def nix_expression(self): + return self.expression_template % dict(path=self.path) + + +class OverridesUrl(object): + def __init__(self, url): + self.url = url + + expression_template = ( + 'let src = pkgs.fetchurl { ' + + 'url = %(url)s; sha256 = "%(sha_string)s"; ' + + '}; in ' + + 'import "${src}" { inherit pkgs python ; }' + ) + + def nix_expression(self): + command = 'nix-prefetch-url {url}'.format( + url=self.url + ) + + return_code, output = cmd( + command, verbose=False, stderr=subprocess.DEVNULL + ) + sha_sum = output.strip() + if len(sha_sum) != 52 or return_code != 0: + raise click.ClickException( + 'Could not determin hash for url %{url}s' % dict( + url=self.url + ) + ) + return self.expression_template % dict( + url=self.url, + sha_string=sha_sum + ) + + +class OverridesGit(object): + def __init__(self, repo_url, path, rev=None): + self.repo_url = repo_url + self.path = path + self.rev = rev + + expression_template = ( + 'let src = pkgs.fetchgit { ' + + 'url = "%(url)s"; ' + + 'sha256 = "%(sha256)s"; ' + + 'rev = "%(rev)s"; ' + + '} ; in import "${src}/%(path)s" { inherit pkgs python; }' + ) + + def nix_expression(self): + command = 'nix-prefetch-git %(url)s' % dict( + url=self.repo_url + ) + + if self.rev is not None: + command += ' --rev %(rev)s' % dict(rev=self.rev) + + return_code, output = cmd( + command, verbose=False, stderr=subprocess.DEVNULL + ) + + if return_code != 0: + raise click.ClickException( + 'Could not fetch git repository at %(url)s' % dict( + url=self.repo_url + ) + ) + repo_data = json.loads(output) + return self.expression_template % dict( + url=repo_data['url'], + sha256=repo_data['sha256'], + rev=repo_data['rev'], + path=self.path + ) + + +def url_to_overrides(url_string): + url = urlparse(url_string) + if url.scheme == 'file' or '': + return OverridesFile(url.path) + elif url.scheme == 'http' or url.scheme == 'https': + return OverridesUrl(url.geturl()) + elif url.scheme.startswith('git+'): + fragments = dict( + map(lambda x: x.split('='), url.fragment.split('&')) + ) + return OverridesGit( + repo_url=urldefrag(url.geturl()[4:])[0], + path=fragments['path'], + rev=fragments.get('rev', None), + ) + else: + raise UnsupportedUrlError('Cannot handle common overrides url %s' % + url_string) + + +class OverridesUrlParam(click.ParamType): + name = 'url' + + def convert(self, value, param, ctx): + try: + return url_to_overrides(value) + except UnsupportedUrlError as e: + self.fail(str(e), param, ctx) + + +OVERRIDES_URL = OverridesUrlParam() diff --git a/src/pypi2nix/stage3.py b/src/pypi2nix/stage3.py index 42ea67c..9079b45 100644 --- a/src/pypi2nix/stage3.py +++ b/src/pypi2nix/stage3.py @@ -1,5 +1,6 @@ -import sys import os +import sys + import click @@ -74,8 +75,17 @@ %(generated_package_nix)s }; overrides = import %(overrides_file)s { inherit pkgs python; }; - -in python.withPackages (fix' (extends overrides generated)) + commonOverrides = [ +%(common_overrides)s + ]; + +in python.withPackages + (fix' (pkgs.lib.fold + extends + generated + ([overrides] ++ commonOverrides) + ) + ) ''' GENERATED_NIX = '''# generated using pypi2nix tool (version: %s) @@ -123,6 +133,7 @@ def main(packages_metadata, enable_tests, python_version, current_dir, + common_overrides=[], ): '''Create Nix expressions. ''' @@ -199,6 +210,11 @@ def main(packages_metadata, overrides = OVERRIDES_NIX % "" + common_overrides_expressions = [ + ' (' + override.nix_expression() + ')' + for override in common_overrides + ] + default = DEFAULT_NIX % dict( version=version, command_arguments=' '.join(sys.argv[1:]), @@ -211,6 +227,7 @@ def main(packages_metadata, overrides_file='.' + overrides_file[len(current_dir):], enable_tests=str(enable_tests).lower(), generated_package_nix=generated, + common_overrides='\n'.join(common_overrides_expressions), ) if not os.path.exists(overrides_file): diff --git a/src/pypi2nix/utils.py b/src/pypi2nix/utils.py index 133ff93..9403522 100644 --- a/src/pypi2nix/utils.py +++ b/src/pypi2nix/utils.py @@ -38,7 +38,7 @@ def safe(string): return string.replace('"', '\\"') -def cmd(command, verbose=False): +def cmd(command, verbose=False, stderr=subprocess.STDOUT): if isinstance(command, str): command = shlex.split(command) @@ -49,7 +49,7 @@ def cmd(command, verbose=False): p = subprocess.Popen( command, stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, + stderr=stderr, ) out = []