New settings module for python inspector

requirements-dev.txt

@@ -4,7 +4,7 @@ bleach==4.1.0
 boolean.py==4.0
 cffi==1.15.0
 cryptography==37.0.2
-dataclasses==0.8
+dataclasses==0.6
 docutils==0.18.1
 et-xmlfile==1.1.0
 execnet==1.9.0
@@ -16,7 +16,7 @@ jinja2==3.0.3
 keyring==23.4.1
 license-expression==30.0.0
 markupsafe==2.0.1
-mypy-extensions==0.4.3
+mypy-extensions==1.0.0
 openpyxl==3.0.10
 pathspec==0.9.0
 pkginfo==1.8.3
@@ -38,4 +38,4 @@ tomli==1.2.3
 tqdm==4.64.0
 twine==3.8.0
 typed-ast==1.5.4
-webencodings==0.5.1
+webencodings==0.5.1

requirements.txt

@@ -14,6 +14,7 @@ packaging==21.3
 packvers==21.5
 pip-requirements-parser==32.0.1
 pkginfo2==30.0.0
+pydantic_settings >= 2.6.1
 pyparsing==3.0.9
 PyYAML==6.0
 requests==2.28.1

src/python_inspector/__init__.py

@@ -7,4 +7,12 @@
 # See https://aboutcode.org for more information about nexB OSS projects.
 #
 
-DEFAULT_PYTHON_VERSION = "3.8"
+from pydantic import ValidationError
+
+from python_inspector.settings import Settings
+
+# Initialize global settings
+try:
+    settings = Settings()
+except ValidationError as e:
+    print(e)

src/python_inspector/api.py

@@ -26,8 +26,8 @@
 from _packagedcode.pypi import PipRequirementsFileHandler
 from _packagedcode.pypi import PythonSetupPyHandler
 from _packagedcode.pypi import can_process_dependent_package
-from python_inspector import DEFAULT_PYTHON_VERSION
 from python_inspector import dependencies
+from python_inspector import settings
 from python_inspector import utils
 from python_inspector import utils_pypi
 from python_inspector.package_data import get_pypi_data_from_purl
@@ -40,8 +40,8 @@
 from python_inspector.resolution import get_reqs_insecurely
 from python_inspector.resolution import get_requirements_from_python_manifest
 from python_inspector.utils_pypi import PLATFORMS_BY_OS
-from python_inspector.utils_pypi import PYPI_SIMPLE_URL
 from python_inspector.utils_pypi import Environment
+from python_inspector.utils_pypi import PypiSimpleRepository
 from python_inspector.utils_pypi import valid_python_versions
 
 
@@ -79,7 +79,7 @@ def resolve_dependencies(
     specifiers=tuple(),
     python_version=None,
     operating_system=None,
-    index_urls=tuple([PYPI_SIMPLE_URL]),
+    index_urls: tuple[str, ...] = settings.INDEX_URL,
     pdt_output=None,
     netrc_file=None,
     max_rounds=200000,
@@ -106,15 +106,15 @@ def resolve_dependencies(
     """
 
     if not operating_system:
-        raise Exception(f"No operating system provided.")
+        raise Exception("No operating system provided.")
     if operating_system not in PLATFORMS_BY_OS:
         raise ValueError(
             f"Invalid operating system: {operating_system}. "
             f"Must be one of: {', '.join(PLATFORMS_BY_OS.keys())}"
         )
 
     if not python_version:
-        raise Exception(f"No python version provided.")
+        raise Exception("No python version provided.")
     if python_version not in valid_python_versions:
         raise ValueError(
             f"Invalid python version: {python_version}. "
@@ -147,14 +147,13 @@ def resolve_dependencies(
 
     files = []
 
-    if PYPI_SIMPLE_URL not in index_urls:
-        index_urls = tuple([PYPI_SIMPLE_URL]) + tuple(index_urls)
-
     # requirements
     for req_file in requirement_files:
         deps = dependencies.get_dependencies_from_requirements(
             requirements_file=req_file)
-        for extra_data in dependencies.get_extra_data_from_requirements(requirements_file=req_file):
+        for extra_data in dependencies.get_extra_data_from_requirements(
+            requirements_file=req_file
+        ):
             index_urls = (
                 *index_urls, *tuple(extra_data.get("extra_index_urls") or []))
             index_urls = (
@@ -260,10 +259,8 @@ def resolve_dependencies(
         # Collect PyPI repos
         for index_url in index_urls:
             index_url = index_url.strip("/")
-            existing = utils_pypi.DEFAULT_PYPI_REPOS_BY_URL.get(index_url)
-            if existing:
-                existing.use_cached_index = use_cached_index
-                repos.append(existing)
+            if index_url in settings.INDEX_URL:
+                repos.append(PypiSimpleRepository(index_url))
             else:
                 credentials = None
                 if parsed_netrc:
@@ -273,7 +270,7 @@ def resolve_dependencies(
                         dict(login=login,
                              password=password) if login and password else None
                     )
-                repo = utils_pypi.PypiSimpleRepository(
+                repo = PypiSimpleRepository(
                     index_url=index_url,
                     use_cached_index=use_cached_index,
                     credentials=credentials,
@@ -366,8 +363,8 @@ def resolve(
 
 def get_resolved_dependencies(
     requirements: List[Requirement],
-    environment: Environment = None,
-    repos: Sequence[utils_pypi.PypiSimpleRepository] = tuple(),
+    environment: Environment,
+    repos: Sequence[PypiSimpleRepository] = tuple(),
     as_tree: bool = False,
     max_rounds: int = 200000,
     pdt_output: bool = False,
@@ -382,6 +379,7 @@ def get_resolved_dependencies(
     Used the provided ``repos`` list of PypiSimpleRepository.
     If empty, use instead the PyPI.org JSON API exclusively instead
     """
+
     resolver = Resolver(
         provider=PythonInputProvider(
             environment=environment,
@@ -391,9 +389,12 @@ def get_resolved_dependencies(
         ),
         reporter=BaseReporter(),
     )
+
     resolver_results = resolver.resolve(
         requirements=requirements, max_rounds=max_rounds)
+
     package_list = get_package_list(results=resolver_results)
+
     if pdt_output:
         return (format_pdt_tree(resolver_results), package_list)
     return (

src/python_inspector/package_data.py

@@ -9,6 +9,7 @@
 # See https://aboutcode.org for more information about nexB OSS projects.
 #
 
+from collections.abc import Generator
 from typing import List
 
 from packageurl import PackageURL
@@ -25,7 +26,10 @@
 
 
 def get_pypi_data_from_purl(
-    purl: str, environment: Environment, repos: List[PypiSimpleRepository], prefer_source: bool
+    purl: str,
+    environment: Environment,
+    repos: list[PypiSimpleRepository],
+    prefer_source: bool,
 ) -> PackageData:
     """
     Generate `Package` object from the `purl` string of pypi type
@@ -36,9 +40,9 @@ def get_pypi_data_from_purl(
     ``prefer_source`` is a boolean value to prefer source distribution over wheel,
     if no source distribution is available then wheel is used
     """
-    purl = PackageURL.from_string(purl)
-    name = purl.name
-    version = purl.version
+    packageurl: PackageURL = PackageURL.from_string(purl)
+    name = packageurl.name
+    version = packageurl.version
     if not version:
         raise Exception("Version is not specified in the purl")
     base_path = "https://pypi.org/pypi"
@@ -54,12 +58,13 @@ def get_pypi_data_from_purl(
     code_view_url = get_pypi_codeview_url(project_urls)
     bug_tracking_url = get_pypi_bugtracker_url(project_urls)
     python_version = get_python_version_from_env_tag(
-        python_version=environment.python_version)
+        python_version=environment.python_version
+    )
     valid_distribution_urls = []
 
     valid_distribution_urls.append(
         get_sdist_download_url(
-            purl=purl,
+            purl=packageurl,
             repos=repos,
             python_version=python_version,
         )
@@ -70,7 +75,7 @@ def get_pypi_data_from_purl(
     if not valid_distribution_urls or not prefer_source:
         wheel_urls = list(
             get_wheel_download_urls(
-                purl=purl,
+                purl=packageurl,
                 repos=repos,
                 environment=environment,
                 python_version=python_version,
@@ -108,7 +113,7 @@ def get_pypi_data_from_purl(
                 maintainer_key="maintainer",
                 maintainer_email_key="maintainer_email",
             ),
-            **purl.to_dict(),
+            **packageurl.to_dict(),
         )
 
 
@@ -144,7 +149,7 @@ def get_wheel_download_urls(
     repos: List[PypiSimpleRepository],
     environment: Environment,
     python_version: str,
-) -> List[str]:
+) -> Generator[str, None, None]:
     """
     Return a list of download urls for the given purl.
     """