Merge branch 'main' into use-asyncio

Author: netomi

.github/workflows/pypi-release.yml

@@ -24,7 +24,7 @@ jobs:
     runs-on: ubuntu-20.04
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
@@ -37,7 +37,7 @@ jobs:
         run: python -m build --sdist --wheel --outdir dist/
 
       - name: Upload built archives
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: pypi_archives
           path: dist/*
@@ -51,7 +51,7 @@ jobs:
 
     steps:
       - name: Download built archives
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: pypi_archives
           path: dist
@@ -71,7 +71,7 @@ jobs:
 
     steps:
       - name: Download built archives
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: pypi_archives
           path: dist

CHANGELOG.rst

@@ -1,6 +1,15 @@
 Changelog
 =========
 
+v0.13.1
+-----------
+
+- Fix ResolutionImpossible for lief==0.15.1 #202
+- Add license expression data from pypi API #208
+- Add python 3.13 in python-inspector #196
+- Update homepage_url and fix CI and tests
+
+
 v0.12.1
 -----------
 

azure-pipelines.yml

@@ -9,46 +9,46 @@ jobs:
     parameters:
       job_name: ubuntu20_cpython
       image_name: ubuntu-20.04
-      python_versions: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"]
+      python_versions: ["3.9", "3.10", "3.11", "3.12"]
       test_suites:
         all: venv/bin/pytest -n 2 -vvs --reruns 2
 
   - template: etc/ci/azure-posix.yml
     parameters:
       job_name: ubuntu22_cpython
       image_name: ubuntu-22.04
-      python_versions: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"]
+      python_versions: ["3.9", "3.10", "3.11", "3.12"]
       test_suites:
         all: venv/bin/pytest -n 2 -vvs --reruns 2
 
   - template: etc/ci/azure-posix.yml
     parameters:
-      job_name: macos12_cpython
-      image_name: macOS-12
-      python_versions: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"]
+      job_name: macos14_cpython
+      image_name: macOS-14
+      python_versions: ["3.9", "3.10", "3.11", "3.12"]
       test_suites:
         all: venv/bin/pytest -n 2 -vvs --reruns 2
 
   - template: etc/ci/azure-posix.yml
     parameters:
       job_name: macos13_cpython
       image_name: macos-13
-      python_versions: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"]
+      python_versions: ["3.9", "3.10", "3.11", "3.12"]
       test_suites:
         all: venv/bin/pytest -n 2 -vvs --reruns 2
 
   - template: etc/ci/azure-win.yml
     parameters:
       job_name: win2019_cpython
       image_name: windows-2019
-      python_versions: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"]
+      python_versions: ["3.9", "3.10", "3.11", "3.12"]
       test_suites:
         all: venv\Scripts\pytest -n 2 -vvs  --reruns 2
 
   - template: etc/ci/azure-win.yml
     parameters:
       job_name: win2022_cpython
       image_name: windows-2022
-      python_versions: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"]
+      python_versions: ["3.9", "3.10", "3.11", "3.12"]
       test_suites:
         all: venv\Scripts\pytest -n 2 -vvs --reruns 2

src/python_inspector/api.py

@@ -66,8 +66,7 @@ def to_dict(self, generic_paths=False):
             # clean file paths
             for file in files:
                 path = file["path"]
-                file["path"] = utils.remove_test_data_dir_variable_prefix(
-                    path=path)
+                file["path"] = utils.remove_test_data_dir_variable_prefix(path=path)
         return {
             "files": files,
             "packages": [package for package in self.packages],
@@ -154,20 +153,16 @@ def resolve_dependencies(
 
     # requirements
     for req_file in requirement_files:
-        deps = dependencies.get_dependencies_from_requirements(
-            requirements_file=req_file)
+        deps = dependencies.get_dependencies_from_requirements(requirements_file=req_file)
         for extra_data in dependencies.get_extra_data_from_requirements(requirements_file=req_file):
-            index_urls = (
-                *index_urls, *tuple(extra_data.get("extra_index_urls") or []))
-            index_urls = (
-                *index_urls, *tuple(extra_data.get("index_url") or []))
+            index_urls = (*index_urls, *tuple(extra_data.get("extra_index_urls") or []))
+            index_urls = (*index_urls, *tuple(extra_data.get("index_url") or []))
         direct_dependencies.extend(deps)
         package_data = [
             pkg_data.to_dict() for pkg_data in PipRequirementsFileHandler.parse(location=req_file)
         ]
         if generic_paths:
-            req_file = utils.remove_test_data_dir_variable_prefix(
-                path=req_file)
+            req_file = utils.remove_test_data_dir_variable_prefix(path=req_file)
 
         files.append(
             dict(
@@ -220,15 +215,13 @@ def resolve_dependencies(
                     files=[setup_py_file],
                     analyze_setup_py_insecurely=analyze_setup_py_insecurely,
                 )
-                setup_py_file_deps = list(
-                    get_dependent_packages_from_reqs(reqs))
+                setup_py_file_deps = list(get_dependent_packages_from_reqs(reqs))
                 direct_dependencies.extend(setup_py_file_deps)
 
         package_data.dependencies = setup_py_file_deps
         file_package_data = [package_data.to_dict()]
         if generic_paths:
-            setup_py_file = utils.remove_test_data_dir_variable_prefix(
-                path=setup_py_file)
+            setup_py_file = utils.remove_test_data_dir_variable_prefix(path=setup_py_file)
         files.append(
             dict(
                 type="file",
@@ -269,11 +262,9 @@ def resolve_dependencies(
             else:
                 credentials = None
                 if parsed_netrc:
-                    login, password = utils.get_netrc_auth(
-                        index_url, parsed_netrc)
+                    login, password = utils.get_netrc_auth(index_url, parsed_netrc)
                     credentials = (
-                        dict(login=login,
-                             password=password) if login and password else None
+                        dict(login=login, password=password) if login and password else None
                     )
                 repo = utils_pypi.PypiSimpleRepository(
                     index_url=index_url,
@@ -451,9 +442,7 @@ async def get_dependencies(requirement: Requirement):
         provider=provider,
         reporter=BaseReporter(),
     )
-    resolver_results = resolver.resolve(
-        requirements=requirements, max_rounds=max_rounds)
-
+    resolver_results = resolver.resolve(requirements=requirements, max_rounds=max_rounds)
     package_list = get_package_list(results=resolver_results)
     if pdt_output:
         return format_pdt_tree(resolver_results), package_list

src/python_inspector/cli_utils.py

@@ -21,8 +21,7 @@ class FileOptionType(click.File):
 
     def convert(self, value, param, ctx):
         known_opts = set(
-            chain.from_iterable(
-                p.opts for p in ctx.command.params if isinstance(p, click.Option))
+            chain.from_iterable(p.opts for p in ctx.command.params if isinstance(p, click.Option))
         )
         if value in known_opts:
             self.fail(

src/python_inspector/package_data.py

@@ -57,15 +57,16 @@ async def get_pypi_data_from_purl(
     project_urls = info.get("project_urls") or {}
     code_view_url = get_pypi_codeview_url(project_urls)
     bug_tracking_url = get_pypi_bugtracker_url(project_urls)
-    python_version = get_python_version_from_env_tag(
-        python_version=environment.python_version)
+    python_version = get_python_version_from_env_tag(python_version=environment.python_version)
     valid_distribution_urls = []
     sdist_url = await get_sdist_download_url(
         purl=parsed_purl, repos=repos, python_version=python_version
     )
     if sdist_url:
         valid_distribution_urls.append(sdist_url)
 
+    valid_distribution_urls = [url for url in valid_distribution_urls if url]
+
     # if prefer_source is True then only source distribution is used
     # in case of no source distribution available then wheel is used
     if not valid_distribution_urls or not prefer_source:
@@ -99,6 +100,7 @@ async def get_pypi_data_from_purl(
             api_data_url=api_url,
             bug_tracking_url=bug_tracking_url,
             code_view_url=code_view_url,
+            license_expression=info.get("license_expression"),
             declared_license=get_declared_license(info),
             download_url=dist_url,
             size=url_data.get("size"),

src/python_inspector/resolution.py

@@ -213,15 +213,13 @@ def get_sdist_file_path_from_filename(sdist):
         sdist_file = sdist.rstrip(".tar.gz")
         with tarfile.open(os.path.join(utils_pypi.CACHE_THIRDPARTY_DIR, sdist)) as file:
             file.extractall(
-                os.path.join(utils_pypi.CACHE_THIRDPARTY_DIR,
-                             "extracted_sdists", sdist_file)
+                os.path.join(utils_pypi.CACHE_THIRDPARTY_DIR, "extracted_sdists", sdist_file)
             )
     elif sdist.endswith(".zip"):
         sdist_file = sdist.rstrip(".zip")
         with ZipFile(os.path.join(utils_pypi.CACHE_THIRDPARTY_DIR, sdist)) as zip:
             zip.extractall(
-                os.path.join(utils_pypi.CACHE_THIRDPARTY_DIR,
-                             "extracted_sdists", sdist_file)
+                os.path.join(utils_pypi.CACHE_THIRDPARTY_DIR, "extracted_sdists", sdist_file)
             )
 
     else:
@@ -364,8 +362,7 @@ def __init__(
         ignore_errors=False,
     ):
         self.environment = environment
-        self.environment_marker = get_environment_marker_from_environment(
-            self.environment)
+        self.environment_marker = get_environment_marker_from_environment(self.environment)
         self.repos = repos or []
         self.versions_by_package: Dict[str, List[Version]] = {}
         self.dependencies_by_purl = {}
@@ -427,11 +424,9 @@ async def _get_versions_for_package_from_repo(
         versions = []
         for version, package in (await repo.get_package_versions(name)).items():
             python_version = parse_version(
-                get_python_version_from_env_tag(
-                    python_version=self.environment.python_version)
+                get_python_version_from_env_tag(python_version=self.environment.python_version)
             )
-            wheels = list(package.get_supported_wheels(
-                environment=self.environment))
+            wheels = list(package.get_supported_wheels(environment=self.environment))
             valid_wheel_present = False
             pypi_valid_python_version = False
             if wheels:
@@ -496,8 +491,7 @@ async def _get_requirements_for_package_from_pypi_simple(
         Return requirements for a package from the simple repositories.
         """
         python_version = parse_version(
-            get_python_version_from_env_tag(
-                python_version=self.environment.python_version)
+            get_python_version_from_env_tag(python_version=self.environment.python_version)
         )
 
         wheels = await utils_pypi.download_wheel(
@@ -510,8 +504,7 @@ async def _get_requirements_for_package_from_pypi_simple(
 
         if wheels:
             for wheel in wheels:
-                wheel_location = os.path.join(
-                    utils_pypi.CACHE_THIRDPARTY_DIR, wheel)
+                wheel_location = os.path.join(utils_pypi.CACHE_THIRDPARTY_DIR, wheel)
                 requirements = get_requirements_from_distribution(
                     handler=PypiWheelHandler,
                     location=wheel_location,
@@ -594,8 +587,7 @@ def get_candidates(
             ):
                 valid_versions.append(parsed_version)
         if not all(version.is_prerelease for version in valid_versions):
-            valid_versions = [
-                version for version in valid_versions if not version.is_prerelease]
+            valid_versions = [version for version in valid_versions if not version.is_prerelease]
         for version in valid_versions:
             yield Candidate(name=name, version=version, extras=extras)
 
@@ -611,7 +603,12 @@ def _iter_matches(
         name = remove_extras(identifier=identifier)
         bad_versions = {c.version for c in incompatibilities[identifier]}
         extras = {e for r in requirements[identifier] for e in r.extras}
-        versions = self.get_versions_for_package(name)
+        versions = []
+        if not self.repos:
+            versions.extend(self.get_versions_for_package(name=name))
+        else:
+            for repo in self.repos:
+                versions.extend(self.get_versions_for_package(name=name, repo=repo))
 
         if not versions:
             if self.ignore_errors:
@@ -702,8 +699,7 @@ def dfs(mapping: Dict, graph: DirectedGraph, src: str):
 
     return dict(
         package=str(src_purl),
-        dependencies=sorted([dfs(mapping, graph, c)
-                            for c in children], key=lambda d: d["package"]),
+        dependencies=sorted([dfs(mapping, graph, c) for c in children], key=lambda d: d["package"]),
     )
 
 
@@ -821,8 +817,7 @@ def get_setup_requirements(sdist_location: str, setup_py_location: str, setup_cf
     """
 
     if not os.path.exists(setup_py_location) and not os.path.exists(setup_cfg_location):
-        raise Exception(
-            f"No setup.py or setup.cfg found in pypi sdist {sdist_location}")
+        raise Exception(f"No setup.py or setup.cfg found in pypi sdist {sdist_location}")
 
     # Some commonon packages like flask may have some dependencies in setup.cfg
     # and some dependencies in setup.py. We are going to check both.

src/python_inspector/resolve_cli.py

@@ -19,7 +19,7 @@
 
 TRACE = False
 
-__version__ = "0.12.0"
+__version__ = "0.13.0"
 
 DEFAULT_PYTHON_VERSION = "38"
 PYPI_SIMPLE_URL = "https://pypi.org/simple"
@@ -84,8 +84,7 @@ def print_version(ctx, param, value):
     metavar="OS",
     show_default=True,
     required=True,
-    help="OS to use for dependency resolution. One of " +
-    ", ".join(utils_pypi.PLATFORMS_BY_OS),
+    help="OS to use for dependency resolution. One of " + ", ".join(utils_pypi.PLATFORMS_BY_OS),
 )
 @click.option(
     "--index-url",
@@ -229,8 +228,7 @@ def resolve_dependencies(
     from python_inspector.api import resolve_dependencies as resolver_api
 
     if not (json_output or pdt_output):
-        click.secho(
-            "No output file specified. Use --json or --json-pdt.", err=True)
+        click.secho("No output file specified. Use --json or --json-pdt.", err=True)
         ctx.exit(1)
 
     if json_output and pdt_output:
@@ -339,8 +337,7 @@ def get_pretty_options(ctx, generic_paths=False):
             value = [value]
 
         for val in value:
-            val = get_pretty_value(param_type=param.type,
-                                   value=val, generic_paths=generic_paths)
+            val = get_pretty_value(param_type=param.type, value=val, generic_paths=generic_paths)
 
             if isinstance(param, click.Argument):
                 args.append(val)

src/python_inspector/utils_pip_compatibility_tags.py.ABOUT

@@ -11,4 +11,4 @@ package_url: pkg:github/pypa/[email protected]#src/pip/_internal/utils/compatibility_ta
 download_url: https://github.com/pypa/pip/blob/20.3.1/src/pip/_internal/utils/compatibility_tags.py
 copyright: Copyright (c) 2008-2020 The pip developers (see AUTHORS.txt file)
 license_expression: mit
-notes: subset copied from pip for tag handling
+notes: subset copied from pip for tag handling. Modified copy of pip compatibility_tags.py.