diff --git a/.gitignore b/.gitignore index 0e60aab..e885032 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ /result* /examples/generated* /deploy_rsa +*~ \ No newline at end of file diff --git a/.travis.yml b/.travis.yml index ffddb3d..65d0db8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -28,7 +28,7 @@ before_install: - export SSL_CERT_FILE=$PWD/result-cacert/etc/ssl/certs/ca-bundle.crt - sudo mkdir -p /etc/ssl/certs/ && sudo rm -f /etc/ssl/certs/ca-certificates.crt && sudo ln -s $PWD/result-cacert/etc/ssl/certs/ca-bundle.crt /etc/ssl/certs/ca-certificates.crt - echo 'binary-caches = https://cache.nixos.org/ https://travis.garbas.si/pypi2nix/' | sudo tee -a /etc/nix/nix.conf > /dev/null - - if [ "$TRAVIS_PULL_REQUEST" != "true" -a "$TRAVIS_BRANCH" = "master" ]; then + - if [ "$TRAVIS_PULL_REQUEST" = "false" -a "$TRAVIS_BRANCH" = "master" ]; then openssl aes-256-cbc -K $encrypted_0cfcc1944c73_key -iv $encrypted_0cfcc1944c73_iv -in deploy_rsa.enc -out deploy_rsa -d; eval "$(ssh-agent -s)"; chmod 600 $TRAVIS_BUILD_DIR/deploy_rsa; @@ -36,12 +36,12 @@ before_install: fi script: - cd examples && make $EXAMPLE && cd .. - - if [ "$TRAVIS_PULL_REQUEST" != "true" -a "$TRAVIS_BRANCH" = "master" ]; then + - if [ "$TRAVIS_PULL_REQUEST" = "false" -a "$TRAVIS_BRANCH" = "master" ]; then mkdir nars/; nix-push --dest "$PWD/nars/" --force ./examples/$EXAMPLE; fi after_success: - - if [ "$TRAVIS_PULL_REQUEST" != "true" -a "$TRAVIS_BRANCH" = "master" ]; then + - if [ "$TRAVIS_PULL_REQUEST" = "false" -a "$TRAVIS_BRANCH" = "master" ]; then rsync -avh --ignore-existing $TRAVIS_BUILD_DIR/nars/ travis@garbas.si:/var/travis/pypi2nix/; fi diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..e2d334e --- /dev/null +++ b/Makefile @@ -0,0 +1,5 @@ +flakes: + nix-shell '' -p python3Packages.flake8 --command \ + 'flake8 src/ --ignore E501' + +.PHONY: flakes diff --git a/default.nix b/default.nix index cb5c5cc..3c92ed6 100644 --- a/default.nix +++ b/default.nix @@ -1,4 +1,5 @@ -{ stdenv, fetchurl, python, zip, makeWrapper, nix, nix-prefetch-scripts +{ stdenv, fetchurl, zip, makeWrapper, nix, nix-prefetch-scripts +, pythonPackages , src ? { outPath = ./.; name = "pypi2nix"; } }: @@ -23,7 +24,11 @@ in stdenv.mkDerivation rec { click requests ]; - buildInputs = [ python zip makeWrapper nix.out nix-prefetch-scripts ]; + buildInputs = [ + pythonPackages.python pythonPackages.flake8 + zip makeWrapper nix.out nix-prefetch-scripts + ]; + doCheck = true; sourceRoot = "."; postUnpack = '' @@ -50,7 +55,7 @@ in stdenv.mkDerivation rec { commonPhase = '' mkdir -p $out/bin - echo "#!${python.interpreter}" > $out/bin/pypi2nix + echo "#!${pythonPackages.python.interpreter}" > $out/bin/pypi2nix echo "import pypi2nix.cli" >> $out/bin/pypi2nix echo "pypi2nix.cli.main()" >> $out/bin/pypi2nix @@ -59,6 +64,10 @@ in stdenv.mkDerivation rec { export PYTHONPATH=$out/pkgs:$PYTHONPATH ''; + checkPhase = '' + flake8 ${src}/src + ''; + installPhase = commonPhase + '' wrapProgram $out/bin/pypi2nix --prefix PYTHONPATH : "$PYTHONPATH" ''; diff --git a/release.nix b/release.nix index e94bf52..7f46076 100644 --- a/release.nix +++ b/release.nix @@ -10,8 +10,8 @@ let if builtins.elem system supportedSystems then import ./default.nix { inherit (pkgs) stdenv fetchurl zip makeWrapper nix nix-prefetch-scripts; - python = pkgs.python35; src = pypi2nix; + pythonPackages = pkgs.python3Packages; } else abort "Unsupported system type: ${system}"; diff --git a/shell.nix b/shell.nix index 908ca09..1cfc587 100644 --- a/shell.nix +++ b/shell.nix @@ -5,8 +5,8 @@ let pkgs = import nixpkgs {}; - python = (builtins.getAttr "python${pythonVersion}Packages" pkgs).python; + pythonPackages = builtins.getAttr "python${pythonVersion}Packages" pkgs; in import ./default.nix { - inherit src python; + inherit src pythonPackages; inherit (pkgs) stdenv fetchurl zip makeWrapper nix nix-prefetch-scripts; } diff --git a/src/pypi2nix/cli.py b/src/pypi2nix/cli.py index 66644d2..a2f932c 100644 --- a/src/pypi2nix/cli.py +++ b/src/pypi2nix/cli.py @@ -195,12 +195,13 @@ def handle_requirements_file(project_dir, requirements_file): with open(new_requirements_file, "w+") as f2: for requirements_line in f1.readlines(): if requirements_line.startswith("-e git+") or \ - requirements_line.startswith("-e hg+"): + requirements_line.startswith("-e hg+"): pass elif requirements_line.startswith("-e"): requirements_line = requirements_line.strip()[3:] try: - tmp_path, egg = requirements_line.strip().split('#') + tmp_path, egg = requirements_line.strip() \ + .split('#') tmp_name = egg.split('egg=')[1] _tmp = tmp_path.split('[') if len(_tmp) > 1: @@ -229,7 +230,7 @@ def handle_requirements_file(project_dir, requirements_file): elif requirements_line.startswith("-r ./"): requirements_file2 = os.path.abspath(os.path.join( os.path.dirname(requirements_file), - requirements_line.strip()[3:] + requirements_line.strip()[3:] )) new_requirements_file2 = handle_requirements_file( project_dir, requirements_file2) @@ -251,7 +252,8 @@ def handle_requirements_file(project_dir, requirements_file): click.echo('') if buildout: - click.echo('Stage0: Generating requirements.txt from buildout configuration ...') + click.echo('Stage0: Generating requirements.txt from buildout ' + 'configuration ...') buildout_requirements = pypi2nix.stage0.main( verbose=verbose, buildout_file=buildout, @@ -319,7 +321,6 @@ def handle_requirements_file(project_dir, requirements_file): current_dir=current_dir, ) - click.echo('') click.echo('Nix expressions generated successfully.') click.echo('') diff --git a/src/pypi2nix/stage0.py b/src/pypi2nix/stage0.py index 041a654..10fe403 100644 --- a/src/pypi2nix/stage0.py +++ b/src/pypi2nix/stage0.py @@ -40,5 +40,7 @@ def main(verbose, raise click.ClickException( u'While trying to run the command something went wrong.') - - return buildout_file and os.path.join(project_dir, 'buildout_requirements.txt') or None + return buildout_file and \ + os.path.join(project_dir, + 'buildout_requirements.txt') \ + or None diff --git a/src/pypi2nix/stage1.py b/src/pypi2nix/stage1.py index 386302c..575ef40 100644 --- a/src/pypi2nix/stage1.py +++ b/src/pypi2nix/stage1.py @@ -71,7 +71,7 @@ def main(verbose, returncode, output = pypi2nix.utils.cmd(command, verbose != 0) if returncode != 0 or \ - output.endswith('ERROR: Failed to build one or more wheels'): + output.endswith('ERROR: Failed to build one or more wheels'): if verbose == 0: click.echo(output) @@ -80,9 +80,10 @@ def main(verbose, # trying to recognize the problem and provide more meanigful error # message no_matching_dist = "No matching distribution found for " - no_pg_config = "Error: pg_config executable not found." if no_matching_dist in output: - dist_name = output[output.find(no_matching_dist) + len(no_matching_dist):] + dist_name = output[ + output.find(no_matching_dist) + len(no_matching_dist): + ] dist_name = dist_name[:dist_name.find(' (from')] message = ( "Most likely `%s` package does not have source (zip/tar.bz) " @@ -94,7 +95,8 @@ def main(verbose, if click.confirm('Do you want to report above issue (a browser ' 'will open with prefilled details of issue)?'): title = "Error when running pypi2nix command" - body = "# Description\n\n\n\n" + body = "# Description\n\n\n\n" body += "# Traceback \n\n```bash\n" body += "% pypi2nix --version\n" with open(os.path.join(HERE, 'VERSION')) as f: diff --git a/src/pypi2nix/stage2.py b/src/pypi2nix/stage2.py index 4e4930f..4d583d9 100644 --- a/src/pypi2nix/stage2.py +++ b/src/pypi2nix/stage2.py @@ -1,4 +1,5 @@ """Parse metadata from .dist-info directories in a wheelhouse.""" +# flake8: noqa: E501 import click import hashlib @@ -56,7 +57,6 @@ def extract_deps(metadata): return list(set(deps)) - all_classifiers = { 'License :: Aladdin Free Public License (AFPL)': None, 'License :: CC0 1.0 Universal (CC0 1.0) Public Domain Dedication': None, @@ -226,7 +226,7 @@ def find_release(wheel_cache_dir, wheel, wheel_data): if not _releases: for _version, _releases_tmp in wheel_data['releases'].items(): if pkg_resources.parse_version(wheel['version']) == \ - pkg_resources.parse_version(_version): + pkg_resources.parse_version(_version): _release_version = _version _releases = _releases_tmp break @@ -294,9 +294,8 @@ def process_wheel(wheel_cache_dir, wheel, sources, verbose, index=INDEX_URL, release['hash_type'] = 'sha256' repo_type = sources[wheel['name']]['type'] - if repo_type is None and \ - (release['url'].startswith('http://') or \ - release['url'].startswith('https://')): + if repo_type is None and (release['url'].startswith('http://') or + release['url'].startswith('https://')): release['fetch_type'] = 'fetchurl' @@ -388,14 +387,12 @@ def process_wheel(wheel_cache_dir, wheel, sources, verbose, index=INDEX_URL, else: raise click.ClickException('Source type `{}` not implemented'.format(repo_type)) - else: url = "{}/{}/json".format(index, wheel['name']) r = requests.get(url, timeout=None) r.raise_for_status() # TODO: handle this nicer wheel_data = r.json() - if not wheel_data.get('releases'): raise click.ClickException( "Unable to find releases for packge {name}".format(**wheel)) @@ -427,8 +424,10 @@ def main(verbose, wheels, requirements_files, wheel_cache_dir, index=INDEX_URL, "Source for path `%s` does not exists." % line ) - elif line.startswith('http://') or line.startswith('https://') or \ - line.startswith('git+') or line.startswith('hg+'): + elif (line.startswith('http://') or + line.startswith('https://') or + line.startswith('git+') or + line.startswith('hg+')): try: url, egg = line.split('#') name = egg.split('egg=')[1] diff --git a/src/pypi2nix/stage3.py b/src/pypi2nix/stage3.py index 54f31dd..a731c92 100644 --- a/src/pypi2nix/stage3.py +++ b/src/pypi2nix/stage3.py @@ -18,7 +18,8 @@ inherit (pkgs) makeWrapper; inherit (pkgs.stdenv.lib) fix' extends inNixShell; - pythonPackages = import "${toString pkgs.path}/pkgs/top-level/python-packages.nix" { + pythonPackages = + import "${toString pkgs.path}/pkgs/top-level/python-packages.nix" { inherit pkgs; inherit (pkgs) stdenv; python = pkgs.%(python_version)s; @@ -35,8 +36,10 @@ buildInputs = [ makeWrapper ] ++ (builtins.attrValues pkgs); buildCommand = '' mkdir -p $out/bin - ln -s ${pythonPackages.python.interpreter} $out/bin/${pythonPackages.python.executable} - for dep in ${builtins.concatStringsSep " " (builtins.attrValues pkgs)}; do + ln -s ${pythonPackages.python.interpreter} \ + $out/bin/${pythonPackages.python.executable} + for dep in ${builtins.concatStringsSep " " \ + (builtins.attrValues pkgs)}; do if [ -d "$dep/bin" ]; then for prog in "$dep/bin/"*; do if [ -f $prog ]; then @@ -67,7 +70,8 @@ python = withPackages {}; - generated = import %(generated_file)s { inherit pkgs python commonBuildInputs commonDoCheck; }; + generated = import %(generated_file)s + { inherit pkgs python commonBuildInputs commonDoCheck; }; overrides = import %(overrides_file)s { inherit pkgs python; }; in python.withPackages (fix' (extends overrides generated)) @@ -122,10 +126,18 @@ def main(packages_metadata, '''Create Nix expressions. ''' - default_file = os.path.join(current_dir, '{}.nix'.format(requirements_name)) - generated_file = os.path.join(current_dir, '{}_generated.nix'.format(requirements_name)) - overrides_file = os.path.join(current_dir, '{}_override.nix'.format(requirements_name)) - frozen_file = os.path.join(current_dir, '{}_frozen.txt'.format(requirements_name)) + default_file = os.path.join( + current_dir, '{}.nix'.format(requirements_name) + ) + generated_file = os.path.join( + current_dir, '{}_generated.nix'.format(requirements_name) + ) + overrides_file = os.path.join( + current_dir, '{}_override.nix'.format(requirements_name) + ) + frozen_file = os.path.join( + current_dir, '{}_frozen.txt'.format(requirements_name) + ) version_file = os.path.join(os.path.dirname(__file__), 'VERSION') with open(version_file) as f: @@ -137,36 +149,41 @@ def main(packages_metadata, for item in sorted(packages_metadata, key=lambda x: x['name']): propagatedBuildInputs = '[ ]' if item.get('deps'): - deps = [x for x in item['deps'] if x.lower() in metadata_by_name.keys()] + deps = [x for x in item['deps'] + if x.lower() in metadata_by_name.keys()] if deps: propagatedBuildInputs = "[\n%s\n ]" % ( '\n'.join(sorted( - [' self."%s"' % (metadata_by_name[x.lower()]['name']) - for x in deps if x not in [item['name']] - ]))) + [' self."%s"' % + (metadata_by_name[x.lower()]['name']) + for x in deps if x not in [item['name']]] + ))) fetch_type = item.get('fetch_type', None) if fetch_type == 'path': - fetch_expression = './' + os.path.relpath(item['url'], current_dir) + fetch_expression = './' + os.path.relpath(item['url'], current_dir) elif fetch_type == 'fetchgit': - fetch_expression = 'pkgs.fetchgit { url = "%(url)s"; %(hash_type)s = "%(hash_value)s"; rev = "%(rev)s"; }' % dict( - url=item['url'], - hash_type=item['hash_type'], - hash_value=item['hash_value'], - rev=item['rev'] - ) + fetch_expression = 'pkgs.fetchgit { url = "%(url)s"; '\ + '%(hash_type)s = "%(hash_value)s"; rev = "%(rev)s"; }' % dict( + url=item['url'], + hash_type=item['hash_type'], + hash_value=item['hash_value'], + rev=item['rev'] + ) elif fetch_type == 'fetchhg': - fetch_expression = 'pkgs.fetchhg { url = "%(url)s"; %(hash_type)s = "%(hash_value)s"; rev = "%(rev)s"; }' % dict( - url=item['url'], - hash_type=item['hash_type'], - hash_value=item['hash_value'], - rev=item['rev'] - ) + fetch_expression = 'pkgs.fetchhg { url = "%(url)s"; '\ + '%(hash_type)s = "%(hash_value)s"; rev = "%(rev)s"; }' % dict( + url=item['url'], + hash_type=item['hash_type'], + hash_value=item['hash_value'], + rev=item['rev'] + ) else: - fetch_expression='pkgs.fetchurl { url = "%(url)s"; %(hash_type)s = "%(hash_value)s"; }' % dict( - url=item['url'], - hash_type=item['hash_type'], - hash_value=item['hash_value'], - ) + fetch_expression = 'pkgs.fetchurl { url = "%(url)s"; '\ + '%(hash_type)s = "%(hash_value)s"; }' % dict( + url=item['url'], + hash_type=item['hash_type'], + hash_value=item['hash_value'], + ) generated_packages_metadata.append(dict( name=item.get("name", ""), @@ -188,9 +205,11 @@ def main(packages_metadata, version=version, command_arguments=' '.join(sys.argv[1:]), python_version=python_version, - extra_build_inputs=extra_build_inputs - and "with pkgs; [ %s ]" % (' '.join(extra_build_inputs)) - or "[]", + extra_build_inputs=( + extra_build_inputs and + "with pkgs; [ %s ]" % (' '.join(extra_build_inputs)) or + "[]" + ), generated_file='.' + generated_file[len(current_dir):], overrides_file='.' + overrides_file[len(current_dir):], enable_tests=str(enable_tests).lower(), diff --git a/src/pypi2nix/utils.py b/src/pypi2nix/utils.py index 78fdb27..133ff93 100644 --- a/src/pypi2nix/utils.py +++ b/src/pypi2nix/utils.py @@ -29,9 +29,10 @@ def pretty_option(option): return '' else: return ' [value: {}]'.format( - type(option) in [list, tuple] - and ' '.join(option) - or option) + type(option) in [list, tuple] and + ' '.join(option) or + option) + def safe(string): return string.replace('"', '\\"') @@ -49,7 +50,7 @@ def cmd(command, verbose=False): command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - ) + ) out = [] while True: