From 1014743f41d1c789b5769620e16dec0c1b859a5d Mon Sep 17 00:00:00 2001 From: Helio Chissini de Castro Date: Thu, 14 Nov 2024 23:59:19 +0100 Subject: [PATCH 1/4] Create settings class based on pydantic-settings Settings can now be defined through environment variables, .env local file or regular arguments to the script. All variables need to be prefixed by PYTHON_INSPECTOR_ to be recognized. Example: PYTHON_INSPECTOR_INDEX_URL="https://pypy1.org,https://foo.bar" will add this two repositories overriding the public repository. Raise the minimum python version to 3.9 as 3.8 is EOL. Signed-off-by: Helio Chissini de Castro --- requirements.txt | 1 + src/python_inspector/__init__.py | 10 +++++++- src/python_inspector/resolve_cli.py | 8 ++++-- src/python_inspector/settings.py | 40 +++++++++++++++++++++++++++++ 4 files changed, 56 insertions(+), 3 deletions(-) create mode 100644 src/python_inspector/settings.py diff --git a/requirements.txt b/requirements.txt index 610856cb..d72ac07c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,6 +14,7 @@ packaging==21.3 packvers==21.5 pip-requirements-parser==32.0.1 pkginfo2==30.0.0 +pydantic_settings >= 2.6.1 pyparsing==3.0.9 PyYAML==6.0 requests==2.28.1 diff --git a/src/python_inspector/__init__.py b/src/python_inspector/__init__.py index a990b071..862ef2f6 100644 --- a/src/python_inspector/__init__.py +++ b/src/python_inspector/__init__.py @@ -7,4 +7,12 @@ # See https://aboutcode.org for more information about nexB OSS projects. # -DEFAULT_PYTHON_VERSION = "3.8" +from pydantic import ValidationError + +from python_inspector.settings import Settings + +# Initialize global settings +try: + settings = Settings() +except ValidationError as e: + print(e) diff --git a/src/python_inspector/resolve_cli.py b/src/python_inspector/resolve_cli.py index a8670d11..57f85f22 100644 --- a/src/python_inspector/resolve_cli.py +++ b/src/python_inspector/resolve_cli.py @@ -13,6 +13,7 @@ import click +from python_inspector import settings from python_inspector import utils_pypi from python_inspector.cli_utils import FileOptionType from python_inspector.utils import write_output_in_file @@ -92,9 +93,9 @@ def print_version(ctx, param, value): "index_urls", type=str, metavar="INDEX", - show_default=True, - default=tuple([PYPI_SIMPLE_URL]), + show_default=False, multiple=True, + required=False, help="PyPI simple index URL(s) to use in order of preference. " "This option can be used multiple times.", ) @@ -255,6 +256,9 @@ def resolve_dependencies( errors=[], ) + if index_urls: + settings.INDEX_URL = index_urls + try: resolution_result: Dict = resolver_api( requirement_files=requirement_files, diff --git a/src/python_inspector/settings.py b/src/python_inspector/settings.py new file mode 100644 index 00000000..8f29f561 --- /dev/null +++ b/src/python_inspector/settings.py @@ -0,0 +1,40 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# ScanCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/python-inspector for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +from pathlib import Path +from typing import Optional +from typing import Union + +from pydantic import field_validator +from pydantic_settings import BaseSettings +from pydantic_settings import SettingsConfigDict + +# Reference: https://docs.pydantic.dev/latest/concepts/pydantic_settings/ + + +class Settings(BaseSettings): + model_config = SettingsConfigDict( + env_file=".env", + env_file_encoding="utf-8", + env_prefix="PYTHON_INSPECTOR_", + case_sensitive=True, + extra="allow", + ) + DEFAULT_PYTHON_VERSION: str = "39" + INDEX_URL: tuple[str, ...] = ("https://pypi.org/simple",) + CACHE_THIRDPARTY_DIR: Path = Path.home() / ".cache/python_inspector" + + @field_validator("INDEX_URL") + def validate_index_url(cls, value): + if isinstance(value, str): + return (value,) + elif isinstance(value, tuple): + return value + else: + raise ValueError("INDEX_URL must be either a string or a tuple of strings.") From 739545a85a2d29757838acd9307cae169f369516 Mon Sep 17 00:00:00 2001 From: Helio Chissini de Castro Date: Fri, 15 Nov 2024 00:08:32 +0100 Subject: [PATCH 2/4] Adapt base code to use new settings class Signed-off-by: Helio Chissini de Castro --- src/python_inspector/api.py | 33 ++++--- src/python_inspector/package_data.py | 23 +++-- src/python_inspector/resolution.py | 91 ++++++++++------- src/python_inspector/utils.py | 4 +- src/python_inspector/utils_pypi.py | 143 ++++++++++++++++----------- 5 files changed, 175 insertions(+), 119 deletions(-) diff --git a/src/python_inspector/api.py b/src/python_inspector/api.py index a64d0c17..fbd9ae30 100644 --- a/src/python_inspector/api.py +++ b/src/python_inspector/api.py @@ -26,8 +26,8 @@ from _packagedcode.pypi import PipRequirementsFileHandler from _packagedcode.pypi import PythonSetupPyHandler from _packagedcode.pypi import can_process_dependent_package -from python_inspector import DEFAULT_PYTHON_VERSION from python_inspector import dependencies +from python_inspector import settings from python_inspector import utils from python_inspector import utils_pypi from python_inspector.package_data import get_pypi_data_from_purl @@ -40,8 +40,8 @@ from python_inspector.resolution import get_reqs_insecurely from python_inspector.resolution import get_requirements_from_python_manifest from python_inspector.utils_pypi import PLATFORMS_BY_OS -from python_inspector.utils_pypi import PYPI_SIMPLE_URL from python_inspector.utils_pypi import Environment +from python_inspector.utils_pypi import PypiSimpleRepository from python_inspector.utils_pypi import valid_python_versions @@ -79,7 +79,7 @@ def resolve_dependencies( specifiers=tuple(), python_version=None, operating_system=None, - index_urls=tuple([PYPI_SIMPLE_URL]), + index_urls: tuple[str, ...] = settings.INDEX_URL, pdt_output=None, netrc_file=None, max_rounds=200000, @@ -106,7 +106,7 @@ def resolve_dependencies( """ if not operating_system: - raise Exception(f"No operating system provided.") + raise Exception("No operating system provided.") if operating_system not in PLATFORMS_BY_OS: raise ValueError( f"Invalid operating system: {operating_system}. " @@ -114,7 +114,7 @@ def resolve_dependencies( ) if not python_version: - raise Exception(f"No python version provided.") + raise Exception("No python version provided.") if python_version not in valid_python_versions: raise ValueError( f"Invalid python version: {python_version}. " @@ -147,14 +147,13 @@ def resolve_dependencies( files = [] - if PYPI_SIMPLE_URL not in index_urls: - index_urls = tuple([PYPI_SIMPLE_URL]) + tuple(index_urls) - # requirements for req_file in requirement_files: deps = dependencies.get_dependencies_from_requirements( requirements_file=req_file) - for extra_data in dependencies.get_extra_data_from_requirements(requirements_file=req_file): + for extra_data in dependencies.get_extra_data_from_requirements( + requirements_file=req_file + ): index_urls = ( *index_urls, *tuple(extra_data.get("extra_index_urls") or [])) index_urls = ( @@ -260,10 +259,8 @@ def resolve_dependencies( # Collect PyPI repos for index_url in index_urls: index_url = index_url.strip("/") - existing = utils_pypi.DEFAULT_PYPI_REPOS_BY_URL.get(index_url) - if existing: - existing.use_cached_index = use_cached_index - repos.append(existing) + if index_url in settings.INDEX_URL: + repos.append(PypiSimpleRepository(index_url)) else: credentials = None if parsed_netrc: @@ -273,7 +270,7 @@ def resolve_dependencies( dict(login=login, password=password) if login and password else None ) - repo = utils_pypi.PypiSimpleRepository( + repo = PypiSimpleRepository( index_url=index_url, use_cached_index=use_cached_index, credentials=credentials, @@ -366,8 +363,8 @@ def resolve( def get_resolved_dependencies( requirements: List[Requirement], - environment: Environment = None, - repos: Sequence[utils_pypi.PypiSimpleRepository] = tuple(), + environment: Environment, + repos: Sequence[PypiSimpleRepository] = tuple(), as_tree: bool = False, max_rounds: int = 200000, pdt_output: bool = False, @@ -382,6 +379,7 @@ def get_resolved_dependencies( Used the provided ``repos`` list of PypiSimpleRepository. If empty, use instead the PyPI.org JSON API exclusively instead """ + resolver = Resolver( provider=PythonInputProvider( environment=environment, @@ -391,9 +389,12 @@ def get_resolved_dependencies( ), reporter=BaseReporter(), ) + resolver_results = resolver.resolve( requirements=requirements, max_rounds=max_rounds) + package_list = get_package_list(results=resolver_results) + if pdt_output: return (format_pdt_tree(resolver_results), package_list) return ( diff --git a/src/python_inspector/package_data.py b/src/python_inspector/package_data.py index a320bf80..e451ae34 100644 --- a/src/python_inspector/package_data.py +++ b/src/python_inspector/package_data.py @@ -9,6 +9,7 @@ # See https://aboutcode.org for more information about nexB OSS projects. # +from collections.abc import Generator from typing import List from packageurl import PackageURL @@ -25,7 +26,10 @@ def get_pypi_data_from_purl( - purl: str, environment: Environment, repos: List[PypiSimpleRepository], prefer_source: bool + purl: str, + environment: Environment, + repos: list[PypiSimpleRepository], + prefer_source: bool, ) -> PackageData: """ Generate `Package` object from the `purl` string of pypi type @@ -36,9 +40,9 @@ def get_pypi_data_from_purl( ``prefer_source`` is a boolean value to prefer source distribution over wheel, if no source distribution is available then wheel is used """ - purl = PackageURL.from_string(purl) - name = purl.name - version = purl.version + packageurl: PackageURL = PackageURL.from_string(purl) + name = packageurl.name + version = packageurl.version if not version: raise Exception("Version is not specified in the purl") base_path = "https://pypi.org/pypi" @@ -54,12 +58,13 @@ def get_pypi_data_from_purl( code_view_url = get_pypi_codeview_url(project_urls) bug_tracking_url = get_pypi_bugtracker_url(project_urls) python_version = get_python_version_from_env_tag( - python_version=environment.python_version) + python_version=environment.python_version + ) valid_distribution_urls = [] valid_distribution_urls.append( get_sdist_download_url( - purl=purl, + purl=packageurl, repos=repos, python_version=python_version, ) @@ -70,7 +75,7 @@ def get_pypi_data_from_purl( if not valid_distribution_urls or not prefer_source: wheel_urls = list( get_wheel_download_urls( - purl=purl, + purl=packageurl, repos=repos, environment=environment, python_version=python_version, @@ -108,7 +113,7 @@ def get_pypi_data_from_purl( maintainer_key="maintainer", maintainer_email_key="maintainer_email", ), - **purl.to_dict(), + **packageurl.to_dict(), ) @@ -144,7 +149,7 @@ def get_wheel_download_urls( repos: List[PypiSimpleRepository], environment: Environment, python_version: str, -) -> List[str]: +) -> Generator[str, None, None]: """ Return a list of download urls for the given purl. """ diff --git a/src/python_inspector/resolution.py b/src/python_inspector/resolution.py index f160b113..34038652 100644 --- a/src/python_inspector/resolution.py +++ b/src/python_inspector/resolution.py @@ -10,12 +10,13 @@ import ast import operator import os -import re import tarfile +from typing import Any from typing import Dict from typing import Generator from typing import List from typing import NamedTuple +from typing import Optional from typing import Tuple from typing import Union from zipfile import ZipFile @@ -36,6 +37,7 @@ from _packagedcode.pypi import PythonSetupPyHandler from _packagedcode.pypi import SetupCfgHandler from _packagedcode.pypi import can_process_dependent_package +from python_inspector import settings from python_inspector import utils_pypi from python_inspector.error import NoVersionsFound from python_inspector.setup_py_live_eval import iter_requirements @@ -208,23 +210,23 @@ def fetch_and_extract_sdist( def get_sdist_file_path_from_filename(sdist): if sdist.endswith(".tar.gz"): sdist_file = sdist.rstrip(".tar.gz") - with tarfile.open(os.path.join(utils_pypi.CACHE_THIRDPARTY_DIR, sdist)) as file: + with tarfile.open(os.path.join(settings.CACHE_THIRDPARTY_DIR.as_posix(), sdist)) as file: file.extractall( - os.path.join(utils_pypi.CACHE_THIRDPARTY_DIR, + os.path.join(settings.CACHE_THIRDPARTY_DIR.as_posix(), "extracted_sdists", sdist_file) ) elif sdist.endswith(".zip"): sdist_file = sdist.rstrip(".zip") - with ZipFile(os.path.join(utils_pypi.CACHE_THIRDPARTY_DIR, sdist)) as zip: + with ZipFile(os.path.join(settings.CACHE_THIRDPARTY_DIR.as_posix(), sdist)) as zip: zip.extractall( - os.path.join(utils_pypi.CACHE_THIRDPARTY_DIR, + os.path.join(settings.CACHE_THIRDPARTY_DIR.as_posix(), "extracted_sdists", sdist_file) ) else: raise Exception(f"Unable to extract sdist {sdist}") - return os.path.join(utils_pypi.CACHE_THIRDPARTY_DIR, "extracted_sdists", sdist_file, sdist_file) + return os.path.join(settings.CACHE_THIRDPARTY_DIR.as_posix(), "extracted_sdists", sdist_file, sdist_file) def get_requirements_from_dependencies( @@ -337,8 +339,8 @@ def get_requirements_from_python_manifest( ) if len(install_requires) > 1: print( - f"Warning: identified multiple definitions of 'install_requires' in " - "{setup_py_location}, defaulting to the first occurrence" + "Warning: identified multiple definitions of 'install_requires' in " + f"{setup_py_location}, defaulting to the first occurrence" ) install_requires = install_requires[0].elts if len(install_requires) != 0: @@ -392,11 +394,12 @@ def get_preference( return transitive, identifier def get_versions_for_package( - self, name: str, repo: Union[List[PypiSimpleRepository], None] = None + self, name: str, repo: Optional[PypiSimpleRepository] = None ) -> List[Version]: """ Return a list of versions for a package. """ + if repo and self.environment: return self.get_versions_for_package_from_repo(name, repo) else: @@ -409,24 +412,27 @@ def get_versions_for_package_from_repo( Return a list of versions for a package name from a repo """ versions = [] + for version, package in repo.get_package_versions(name).items(): python_version = parse_version( get_python_version_from_env_tag( - python_version=self.environment.python_version) + python_version=self.environment.python_version + ) ) - wheels = list(package.get_supported_wheels( - environment=self.environment)) + wheels = list(package.get_supported_wheels(environment=self.environment)) valid_wheel_present = False pypi_valid_python_version = False if wheels: for wheel in wheels: if utils_pypi.valid_python_version( - python_requires=wheel.python_requires, python_version=python_version + python_requires=wheel.python_requires, + python_version=python_version, ): valid_wheel_present = True if package.sdist: pypi_valid_python_version = utils_pypi.valid_python_version( - python_requires=package.sdist.python_requires, python_version=python_version + python_requires=package.sdist.python_requires, + python_version=python_version, ) if valid_wheel_present or pypi_valid_python_version: versions.append(version) @@ -436,13 +442,17 @@ def get_versions_for_package_from_pypi_json_api(self, name: str) -> List[Version """ Return a list of versions for a package name from the PyPI.org JSON API """ + if name not in self.versions_by_package: api_url = f"https://pypi.org/pypi/{name}/json" - resp = get_response(api_url) - if not resp: - self.versions_by_package[name] = [] - releases = resp.get("releases") or {} - self.versions_by_package[name] = releases.keys() or [] + releases: dict[str, Any] = {} + self.versions_by_package[name] = [] + + response = get_response(api_url) + + if isinstance(response, dict): + releases = response.get("releases", {}) + self.versions_by_package[name] = releases.keys() or [] versions = self.versions_by_package[name] return versions @@ -465,7 +475,8 @@ def get_requirements_for_package_from_pypi_simple( """ python_version = parse_version( get_python_version_from_env_tag( - python_version=self.environment.python_version) + python_version=self.environment.python_version + ) ) wheels = utils_pypi.download_wheel( @@ -479,7 +490,8 @@ def get_requirements_for_package_from_pypi_simple( if wheels: for wheel in wheels: wheel_location = os.path.join( - utils_pypi.CACHE_THIRDPARTY_DIR, wheel) + settings.CACHE_THIRDPARTY_DIR.as_posix(), wheel + ) requirements = get_requirements_from_distribution( handler=PypiWheelHandler, location=wheel_location, @@ -532,19 +544,22 @@ def get_requirements_for_package_from_pypi_simple( def get_requirements_for_package_from_pypi_json_api( self, purl: PackageURL - ) -> List[Requirement]: + ) -> Generator[Requirement, Any, Any]: """ Return requirements for a package from the PyPI.org JSON API """ + self.dependencies_by_purl[str(purl)] = [] + info: dict[str, Any] = {} + # if no repos are provided use the incorrect but fast JSON API if str(purl) not in self.dependencies_by_purl: api_url = f"https://pypi.org/pypi/{purl.name}/{purl.version}/json" - resp = get_response(api_url) - if not resp: - self.dependencies_by_purl[str(purl)] = [] - info = resp.get("info") or {} - requires_dist = info.get("requires_dist") or [] - self.dependencies_by_purl[str(purl)] = requires_dist + + response = get_response(api_url) + if isinstance(response, dict): + info = response.get("info", {}) + requires_dist = info.get("requires_dist", []) + self.dependencies_by_purl[str(purl)] = requires_dist for dependency in self.dependencies_by_purl[str(purl)]: yield Requirement(dependency) @@ -572,7 +587,8 @@ def get_candidates( valid_versions.append(parsed_version) if not all(version.is_prerelease for version in valid_versions): valid_versions = [ - version for version in valid_versions if not version.is_prerelease] + version for version in valid_versions if not version.is_prerelease + ] for version in valid_versions: yield Candidate(name=name, version=version, extras=extras) @@ -685,8 +701,9 @@ def dfs(mapping: Dict, graph: DirectedGraph, src: str): return dict( package=str(src_purl), - dependencies=sorted([dfs(mapping, graph, c) - for c in children], key=lambda d: d["package"]), + dependencies=sorted( + [dfs(mapping, graph, c) for c in children], key=lambda d: d["package"] + ), ) @@ -743,7 +760,10 @@ def pdt_dfs(mapping, graph, src): children = list(graph.iter_children(src)) if not children: return dict( - key=src, package_name=src, installed_version=str(mapping[src].version), dependencies=[] + key=src, + package_name=src, + installed_version=str(mapping[src].version), + dependencies=[], ) # recurse dependencies = [pdt_dfs(mapping, graph, c) for c in children] @@ -794,7 +814,9 @@ def get_package_list(results): return list(sorted(packages)) -def get_setup_requirements(sdist_location: str, setup_py_location: str, setup_cfg_location: str): +def get_setup_requirements( + sdist_location: str, setup_py_location: str, setup_cfg_location: str +): """ Yield Requirement(s) from Pypi in the ``location`` directory that contains a setup.py and/or a setup.cfg and optionally a requirements.txt file if @@ -805,7 +827,8 @@ def get_setup_requirements(sdist_location: str, setup_py_location: str, setup_cf if not os.path.exists(setup_py_location) and not os.path.exists(setup_cfg_location): raise Exception( - f"No setup.py or setup.cfg found in pypi sdist {sdist_location}") + f"No setup.py or setup.cfg found in pypi sdist {sdist_location}" + ) # Some commonon packages like flask may have some dependencies in setup.cfg # and some dependencies in setup.py. We are going to check both. diff --git a/src/python_inspector/utils.py b/src/python_inspector/utils.py index 9f5b0900..00f41808 100644 --- a/src/python_inspector/utils.py +++ b/src/python_inspector/utils.py @@ -11,7 +11,7 @@ import json import os -from typing import Dict +from typing import Any from typing import List from typing import NamedTuple @@ -64,7 +64,7 @@ class Candidate(NamedTuple): extras: str -def get_response(url: str) -> Dict: +def get_response(url: str) -> Any: """ Return a mapping of the JSON response from fetching ``url`` or None if the ``url`` cannot be fetched.. diff --git a/src/python_inspector/utils_pypi.py b/src/python_inspector/utils_pypi.py index 2af5d57f..6d455e14 100644 --- a/src/python_inspector/utils_pypi.py +++ b/src/python_inspector/utils_pypi.py @@ -11,7 +11,6 @@ import email import itertools import os -import pathlib import re import shutil import tempfile @@ -19,6 +18,7 @@ from collections import defaultdict from typing import List from typing import NamedTuple +from typing import Optional from urllib.parse import quote_plus from urllib.parse import unquote from urllib.parse import urlparse @@ -34,8 +34,8 @@ from packvers import version as packaging_version from packvers.specifiers import SpecifierSet -from python_inspector import DEFAULT_PYTHON_VERSION from python_inspector import utils_pip_compatibility_tags +from python_inspector import settings """ Utilities to manage Python thirparty libraries source, binaries and metadata in @@ -116,7 +116,9 @@ } valid_python_versions = list(PYTHON_DOT_VERSIONS_BY_VER.keys()) -valid_python_versions.extend([dot_ver for pyver, dot_ver in PYTHON_DOT_VERSIONS_BY_VER.items()]) +valid_python_versions.extend( + [dot_ver for pyver, dot_ver in PYTHON_DOT_VERSIONS_BY_VER.items()] +) def get_python_dot_version(version): @@ -180,21 +182,12 @@ def get_python_dot_version(version): ], } -CACHE_THIRDPARTY_DIR = os.environ.get("PYTHON_INSPECTOR_CACHE_DIR") -if not CACHE_THIRDPARTY_DIR: - CACHE_THIRDPARTY_DIR = ".cache/python_inspector" - try: - os.makedirs(CACHE_THIRDPARTY_DIR, exist_ok=True) - except Exception: - home = pathlib.Path.home() - CACHE_THIRDPARTY_DIR = str(home / ".cache/python_inspector") - os.makedirs(CACHE_THIRDPARTY_DIR, exist_ok=True) - +try: + settings.CACHE_THIRDPARTY_DIR.mkdir(parents=True, exist_ok=True) +except OSError as e: + print(f"Unable to create the directory {e.filename}") + exit(1) -################################################################################ - -PYPI_SIMPLE_URL = "https://pypi.org/simple" -PYPI_INDEX_URLS = (PYPI_SIMPLE_URL,) ################################################################################ @@ -220,11 +213,11 @@ def download_wheel( name, version, environment, - dest_dir=CACHE_THIRDPARTY_DIR, + dest_dir=settings.CACHE_THIRDPARTY_DIR.as_posix(), repos=tuple(), verbose=False, echo_func=None, - python_version=DEFAULT_PYTHON_VERSION, + python_version=settings.DEFAULT_PYTHON_VERSION, ): """ Download the wheels binary distribution(s) of package ``name`` and @@ -238,7 +231,7 @@ def download_wheel( print(f" download_wheel: {name}=={version} for envt: {environment}") if not repos: - repos = DEFAULT_PYPI_REPOS + repos = get_current_indexes() fetched_wheel_filenames = [] for repo in repos: @@ -265,15 +258,21 @@ def download_wheel( return fetched_wheel_filenames -def get_valid_sdist(repo, name, version, python_version=DEFAULT_PYTHON_VERSION): +def get_valid_sdist( + repo, name, version, python_version=settings.DEFAULT_PYTHON_VERSION +): package = repo.get_package_version(name=name, version=version) if not package: if TRACE_DEEP: print( - print(f" get_valid_sdist: No package in {repo.index_url} for {name}=={version}") + print( + f" get_valid_sdist: No package in {repo.index_url} for {name}=={version}" + ) ) return + sdist = package.sdist + if not sdist: if TRACE_DEEP: print(f" get_valid_sdist: No sdist for {name}=={version}") @@ -283,12 +282,14 @@ def get_valid_sdist(repo, name, version, python_version=DEFAULT_PYTHON_VERSION): ): return if TRACE_DEEP: - print(f" get_valid_sdist: Getting sdist from index (or cache): {sdist.download_url}") + print( + f" get_valid_sdist: Getting sdist from index (or cache): {sdist.download_url}" + ) return sdist def get_supported_and_valid_wheels( - repo, name, version, environment, python_version=DEFAULT_PYTHON_VERSION + repo, name, version, environment, python_version=settings.DEFAULT_PYTHON_VERSION ) -> List: """ Return a list of wheels matching the ``environment`` Environment constraints. @@ -334,11 +335,11 @@ def valid_python_version(python_version, python_requires): def download_sdist( name, version, - dest_dir=CACHE_THIRDPARTY_DIR, + dest_dir=settings.CACHE_THIRDPARTY_DIR.as_posix(), repos=tuple(), verbose=False, echo_func=None, - python_version=DEFAULT_PYTHON_VERSION, + python_version=settings.DEFAULT_PYTHON_VERSION, ): """ Download the sdist source distribution of package ``name`` and ``version`` @@ -351,11 +352,12 @@ def download_sdist( print(f" download_sdist: {name}=={version}") if not repos: - repos = DEFAULT_PYPI_REPOS + repos = get_current_indexes() fetched_sdist_filename = None for repo in repos: + sdist = get_valid_sdist(repo, name, version, python_version=python_version) if not sdist: if TRACE_DEEP: @@ -428,7 +430,6 @@ class Link(NamedTuple): @attr.attributes class Distribution(NameVer): - """ A Distribution is either either a Wheel or Sdist and is identified by and created from its filename as well as its name and version. A Distribution is @@ -620,7 +621,7 @@ def get_best_download_url(self, repos=tuple()): """ if not repos: - repos = DEFAULT_PYPI_REPOS + repos = get_current_indexes() for repo in repos: package = repo.get_package_version(name=self.name, version=self.version) @@ -642,7 +643,7 @@ def get_best_download_url(self, repos=tuple()): def download( self, - dest_dir=CACHE_THIRDPARTY_DIR, + dest_dir=settings.CACHE_THIRDPARTY_DIR.as_posix(), verbose=False, echo_func=None, ): @@ -712,7 +713,7 @@ def to_dict(self): """ return {k: v for k, v in attr.asdict(self).items() if v} - def get_checksums(self, dest_dir=CACHE_THIRDPARTY_DIR): + def get_checksums(self, dest_dir=settings.CACHE_THIRDPARTY_DIR.as_posix()): """ Return a mapping of computed checksums for this dist filename is `dest_dir`. @@ -723,13 +724,13 @@ def get_checksums(self, dest_dir=CACHE_THIRDPARTY_DIR): else: return {} - def set_checksums(self, dest_dir=CACHE_THIRDPARTY_DIR): + def set_checksums(self, dest_dir=settings.CACHE_THIRDPARTY_DIR.as_posix()): """ Update self with checksums computed for this dist filename is `dest_dir`. """ self.update(self.get_checksums(dest_dir), overwrite=True) - def validate_checksums(self, dest_dir=CACHE_THIRDPARTY_DIR): + def validate_checksums(self, dest_dir=settings.CACHE_THIRDPARTY_DIR.as_posix()): """ Return True if all checksums that have a value in this dist match checksums computed for this dist filename is `dest_dir`. @@ -742,7 +743,7 @@ def validate_checksums(self, dest_dir=CACHE_THIRDPARTY_DIR): return False return True - def extract_pkginfo(self, dest_dir=CACHE_THIRDPARTY_DIR): + def extract_pkginfo(self, dest_dir=settings.CACHE_THIRDPARTY_DIR.as_posix()): """ Return the text of the first PKG-INFO or METADATA file found in the archive of this Distribution in `dest_dir`. Return None if not found. @@ -771,7 +772,7 @@ def extract_pkginfo(self, dest_dir=CACHE_THIRDPARTY_DIR): with open(pi) as fi: return fi.read() - def load_pkginfo_data(self, dest_dir=CACHE_THIRDPARTY_DIR): + def load_pkginfo_data(self, dest_dir=settings.CACHE_THIRDPARTY_DIR.as_posix()): """ Update self with data loaded from the PKG-INFO file found in the archive of this Distribution in `dest_dir`. @@ -1000,7 +1001,6 @@ def to_filename(self): @attr.attributes class Wheel(Distribution): - """ Represents a wheel file. @@ -1096,7 +1096,10 @@ def from_filename(cls, filename): # All the tag combinations from this file tags = { - packaging_tags.Tag(x, y, z) for x in python_versions for y in abis for z in platforms + packaging_tags.Tag(x, y, z) + for x in python_versions + for y in abis + for z in platforms } return cls( @@ -1156,7 +1159,11 @@ def is_pure(self): >>> Wheel.from_filename('future-0.16.0-py3-cp36m-any.whl').is_pure() False """ - return "py3" in self.python_versions and "none" in self.abis and "any" in self.platforms + return ( + "py3" in self.python_versions + and "none" in self.abis + and "any" in self.platforms + ) def is_pure_wheel(filename): @@ -1318,7 +1325,9 @@ def dists_from_links(cls, links: List[Link]): dists = [] if TRACE_ULTRA_DEEP: print(" ###paths_or_urls:", links) - installable: List[Link] = [link for link in links if link.url.endswith(EXTENSIONS)] + installable: List[Link] = [ + link for link in links if link.url.endswith(EXTENSIONS) + ] for link in installable: try: dist = Distribution.from_link(link=link) @@ -1462,9 +1471,10 @@ class PypiSimpleRepository: PyPI simple index. It is populated lazily based on requested packages names. """ - index_url = attr.ib( - type=str, - default=PYPI_SIMPLE_URL, + # We use first entry in INDEX_URL setting as default + + index_url: str = attr.ib( + default=settings.INDEX_URL[0], metadata=dict(help="Base PyPI simple URL for this index."), ) @@ -1488,8 +1498,7 @@ class PypiSimpleRepository: repr=False, ) - use_cached_index = attr.ib( - type=bool, + use_cached_index: bool = attr.ib( default=False, metadata=dict( help="If True, use any existing on-disk cached PyPI index files. Otherwise, fetch and cache." @@ -1497,7 +1506,7 @@ class PypiSimpleRepository: repr=False, ) - credentials = attr.ib(type=dict, default=None) + credentials: Optional[dict] = attr.ib(default=None) def _get_package_versions_map( self, @@ -1512,7 +1521,10 @@ def _get_package_versions_map( assert name normalized_name = NameVer.normalize_name(name) versions = self.packages[normalized_name] - if not versions and normalized_name not in self.fetched_package_normalized_names: + if ( + not versions + and normalized_name not in self.fetched_package_normalized_names + ): self.fetched_package_normalized_names.add(normalized_name) try: links = self.fetch_links( @@ -1528,7 +1540,9 @@ def _get_package_versions_map( self.packages[normalized_name] = versions except RemoteNotFetchedException as e: if TRACE: - print(f"failed to fetch package name: {name} from: {self.index_url}:\n{e}") + print( + f"failed to fetch package name: {name} from: {self.index_url}:\n{e}" + ) if not versions and TRACE: print(f"WARNING: package {name} not found in repo: {self.index_url}") @@ -1537,7 +1551,7 @@ def _get_package_versions_map( def get_package_versions( self, - name, + name: str, verbose=False, echo_func=None, ): @@ -1630,20 +1644,24 @@ def resolve_relative_url(package_url, url): url_parts = urlparse(url) # If the relative URL starts with '..', remove the last directory from the base URL if url_parts.path.startswith(".."): - path = base_url_parts.path.rstrip("/").rsplit("/", 1)[0] + url_parts.path[2:] + path = ( + base_url_parts.path.rstrip("/").rsplit("/", 1)[0] + url_parts.path[2:] + ) else: path = urlunparse( - ("", "", url_parts.path, url_parts.params, url_parts.query, url_parts.fragment) + ( + "", + "", + url_parts.path, + url_parts.params, + url_parts.query, + url_parts.fragment, + ) ) resolved_url_parts = base_url_parts._replace(path=path) url = urlunparse(resolved_url_parts) return url - -PYPI_PUBLIC_REPO = PypiSimpleRepository(index_url=PYPI_SIMPLE_URL) -DEFAULT_PYPI_REPOS = (PYPI_PUBLIC_REPO,) -DEFAULT_PYPI_REPOS_BY_URL = {r.index_url: r for r in DEFAULT_PYPI_REPOS} - ################################################################################ # # Basic file and URL-based operations using a persistent file-based Cache @@ -1658,7 +1676,7 @@ class Cache: This is used to avoid impolite fetching from remote locations. """ - directory = attr.ib(type=str, default=CACHE_THIRDPARTY_DIR) + directory = attr.ib(type=str, default=settings.CACHE_THIRDPARTY_DIR.as_posix()) def __attrs_post_init__(self): os.makedirs(self.directory, exist_ok=True) @@ -1718,7 +1736,7 @@ def get_file_content( if path_or_url.startswith("https://"): if TRACE_DEEP: print(f"Fetching: {path_or_url}") - _headers, content = get_remote_file_content( + _, content = get_remote_file_content( url=path_or_url, credentials=credentials, as_text=as_text, @@ -1810,7 +1828,9 @@ def get_remote_file_content( ) else: - raise RemoteNotFetchedException(f"Failed HTTP request from {url} with {status}") + raise RemoteNotFetchedException( + f"Failed HTTP request from {url} with {status}" + ) if headers_only: return response.headers, None @@ -1845,3 +1865,10 @@ def fetch_and_save( with open(output, wmode) as fo: fo.write(content) return content + + +def get_current_indexes() -> list[PypiSimpleRepository]: + """ + Return a lost of PypiSimpleRepository indexes available + """ + return [PypiSimpleRepository(index_url=url) for url in settings.INDEX_URL] From 9fb79340d48ae49db9b985fe05869bb7d9eed389 Mon Sep 17 00:00:00 2001 From: Helio Chissini de Castro Date: Fri, 15 Nov 2024 00:08:55 +0100 Subject: [PATCH 3/4] Adapt tests to use new settings class Signed-off-by: Helio Chissini de Castro --- ...json => azure-devops.req-39-expected.json} | 0 tests/test_cli.py | 12 ++-- tests/test_resolution.py | 69 +++++++++++-------- 3 files changed, 46 insertions(+), 35 deletions(-) rename tests/data/{azure-devops.req-38-expected.json => azure-devops.req-39-expected.json} (100%) diff --git a/tests/data/azure-devops.req-38-expected.json b/tests/data/azure-devops.req-39-expected.json similarity index 100% rename from tests/data/azure-devops.req-38-expected.json rename to tests/data/azure-devops.req-39-expected.json diff --git a/tests/test_cli.py b/tests/test_cli.py index 8491ad36..5d22e646 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -130,7 +130,7 @@ def test_cli_with_single_index_url_except_pypi_simple(): "single-url-except-simple-expected.json", must_exist=False ) # using flask since it's not present in thirdparty - specifier = "flask" + specifier = "requests" extra_options = [ "--index-url", "https://thirdparty.aboutcode.org/pypi/simple/", @@ -219,14 +219,14 @@ def test_cli_with_azure_devops_with_python_312(): @pytest.mark.online -def test_cli_with_azure_devops_with_python_38(): +def test_cli_with_azure_devops_with_python_39(): requirements_file = test_env.get_test_loc("azure-devops.req.txt") - expected_file = test_env.get_test_loc("azure-devops.req-38-expected.json", must_exist=False) + expected_file = test_env.get_test_loc("azure-devops.req-39-expected.json", must_exist=False) extra_options = [ "--operating-system", "linux", "--python-version", - "38", + "39", ] check_requirements_resolution( requirements_file=requirements_file, @@ -399,7 +399,7 @@ def check_specs_resolution( def append_os_and_pyver_options(options): if "--python-version" not in options: - options.extend(["--python-version", "38"]) + options.extend(["--python-version", "39"]) if "--operating-system" not in options: options.extend(["--operating-system", "linux"]) return options @@ -439,7 +439,7 @@ def test_passing_of_no_json_output_flag(): def test_passing_of_no_os(): - options = ["--specifier", "foo", "--json", "-", "--python-version", "38"] + options = ["--specifier", "foo", "--json", "-", "--python-version", "39"] run_cli(options=options, expected_rc=2, get_env=False) diff --git a/tests/test_resolution.py b/tests/test_resolution.py index 5269f7a1..20d9c7c0 100644 --- a/tests/test_resolution.py +++ b/tests/test_resolution.py @@ -18,6 +18,7 @@ from packvers.requirements import Requirement from _packagedcode import models +from python_inspector import settings from python_inspector.api import get_resolved_dependencies from python_inspector.error import NoVersionsFound from python_inspector.resolution import PythonInputProvider @@ -25,9 +26,9 @@ from python_inspector.resolution import get_requirements_from_python_manifest from python_inspector.resolution import is_valid_version from python_inspector.resolution import parse_reqs_from_setup_py_insecurely -from python_inspector.utils_pypi import PYPI_PUBLIC_REPO from python_inspector.utils_pypi import Environment from python_inspector.utils_pypi import PypiSimpleRepository +from python_inspector.utils_pypi import get_current_indexes setup_test_env = FileDrivenTesting() setup_test_env.test_data_dir = os.path.join(os.path.dirname(__file__), "data") @@ -43,16 +44,16 @@ def test_get_resolved_dependencies_with_flask_and_python_310(): python_version="310", operating_system="linux", ), - repos=[PYPI_PUBLIC_REPO], + repos=get_current_indexes(), as_tree=False, ) assert plist == [ "pkg:pypi/click@8.1.7", "pkg:pypi/flask@2.1.2", - "pkg:pypi/itsdangerous@2.1.2", - "pkg:pypi/jinja2@3.1.3", - "pkg:pypi/markupsafe@2.1.5", - "pkg:pypi/werkzeug@3.0.1", + "pkg:pypi/itsdangerous@2.2.0", + "pkg:pypi/jinja2@3.1.4", + "pkg:pypi/markupsafe@3.0.2", + "pkg:pypi/werkzeug@3.1.3", ] @@ -66,17 +67,17 @@ def test_get_resolved_dependencies_with_flask_and_python_310_windows(): python_version="310", operating_system="windows", ), - repos=[PYPI_PUBLIC_REPO], + repos=get_current_indexes(), as_tree=False, ) assert plist == [ "pkg:pypi/click@8.1.7", "pkg:pypi/colorama@0.4.6", "pkg:pypi/flask@2.1.2", - "pkg:pypi/itsdangerous@2.1.2", - "pkg:pypi/jinja2@3.1.3", - "pkg:pypi/markupsafe@2.1.5", - "pkg:pypi/werkzeug@3.0.1", + "pkg:pypi/itsdangerous@2.2.0", + "pkg:pypi/jinja2@3.1.4", + "pkg:pypi/markupsafe@3.0.2", + "pkg:pypi/werkzeug@3.1.3", ] @@ -90,7 +91,7 @@ def test_get_resolved_dependencies_with_flask_and_python_36(): python_version="36", operating_system="linux", ), - repos=[PYPI_PUBLIC_REPO], + repos=get_current_indexes(), as_tree=False, ) @@ -116,19 +117,21 @@ def test_get_resolved_dependencies_with_tilde_requirement_using_json_api(): requirements=[req], as_tree=False, environment=Environment( - python_version="38", + python_version="39", operating_system="linux", ), + repos=get_current_indexes(), ) + assert plist == [ "pkg:pypi/click@8.1.7", "pkg:pypi/flask@2.1.3", - "pkg:pypi/importlib-metadata@7.1.0", - "pkg:pypi/itsdangerous@2.1.2", - "pkg:pypi/jinja2@3.1.3", - "pkg:pypi/markupsafe@2.1.5", - "pkg:pypi/werkzeug@3.0.1", - "pkg:pypi/zipp@3.18.1", + "pkg:pypi/importlib-metadata@8.5.0", + "pkg:pypi/itsdangerous@2.2.0", + "pkg:pypi/jinja2@3.1.4", + "pkg:pypi/markupsafe@3.0.2", + "pkg:pypi/werkzeug@3.1.3", + "pkg:pypi/zipp@3.21.0", ] @@ -144,7 +147,9 @@ def test_get_resolved_dependencies_for_version_containing_local_version_identifi operating_system="linux", ), repos=[ - PypiSimpleRepository(index_url="https://download.pytorch.org/whl/cpu", credentials=None) + PypiSimpleRepository( + index_url="https://download.pytorch.org/whl/cpu", credentials=None + ) ], as_tree=False, ) @@ -168,21 +173,21 @@ def test_without_supported_wheels(): _, plist = get_resolved_dependencies( requirements=[req], as_tree=False, - repos=[PYPI_PUBLIC_REPO], + repos=get_current_indexes(), environment=Environment( - python_version="38", + python_version="39", operating_system="linux", ), ) assert plist == [ "pkg:pypi/autobahn@22.3.2", - "pkg:pypi/cffi@1.16.0", - "pkg:pypi/cryptography@42.0.5", + "pkg:pypi/cffi@1.17.1", + "pkg:pypi/cryptography@43.0.3", "pkg:pypi/hyperlink@21.0.0", - "pkg:pypi/idna@3.6", - "pkg:pypi/pycparser@2.21", - "pkg:pypi/setuptools@69.2.0", + "pkg:pypi/idna@3.10", + "pkg:pypi/pycparser@2.22", + "pkg:pypi/setuptools@75.5.0", "pkg:pypi/txaio@23.1.1", ] @@ -316,13 +321,19 @@ def test_get_requirements_from_python_manifest_securely(): def test_setup_py_parsing_insecure(): setup_py_file = setup_test_env.get_test_loc("insecure-setup/setup.py") - reqs = [str(req) for req in list(parse_reqs_from_setup_py_insecurely(setup_py=setup_py_file))] + reqs = [ + str(req) + for req in list(parse_reqs_from_setup_py_insecurely(setup_py=setup_py_file)) + ] assert reqs == ["isodate", "pyparsing", "six"] def test_setup_py_parsing_insecure_testpkh(): setup_py_file = setup_test_env.get_test_loc("insecure-setup-2/setup.py") - reqs = [str(req) for req in list(parse_reqs_from_setup_py_insecurely(setup_py=setup_py_file))] + reqs = [ + str(req) + for req in list(parse_reqs_from_setup_py_insecurely(setup_py=setup_py_file)) + ] assert reqs == [ "CairoSVG<2.0.0,>=1.0.20", "click>=5.0.0", From 0ef6345c647a7cfb3cac95090b64d806c1958fac Mon Sep 17 00:00:00 2001 From: Helio Chissini de Castro Date: Fri, 15 Nov 2024 00:11:26 +0100 Subject: [PATCH 4/4] Update dev requirements - Downgrade dataclasses to version 0.6, as last supported to python 3.9 - Update mypy to version 1.0.0 Signed-off-by: Helio Chissini de Castro --- requirements-dev.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 6514e9e2..116220eb 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -4,7 +4,7 @@ bleach==4.1.0 boolean.py==4.0 cffi==1.15.0 cryptography==37.0.2 -dataclasses==0.8 +dataclasses==0.6 docutils==0.18.1 et-xmlfile==1.1.0 execnet==1.9.0 @@ -16,7 +16,7 @@ jinja2==3.0.3 keyring==23.4.1 license-expression==30.0.0 markupsafe==2.0.1 -mypy-extensions==0.4.3 +mypy-extensions==1.0.0 openpyxl==3.0.10 pathspec==0.9.0 pkginfo==1.8.3 @@ -38,4 +38,4 @@ tomli==1.2.3 tqdm==4.64.0 twine==3.8.0 typed-ast==1.5.4 -webencodings==0.5.1 \ No newline at end of file +webencodings==0.5.1