WIP: Remove all extra debug statements. Make sure wheels have credentials

Author: calizarr

etc/scripts/fetch_thirdparty.py

@@ -19,7 +19,7 @@
 import utils_thirdparty
 import utils_requirements
 
-TRACE = True
+TRACE = False
 TRACE_DEEP = False
 
 

etc/scripts/utils_thirdparty.py

@@ -110,7 +110,7 @@
 
 """
 
-TRACE = True
+TRACE = False
 TRACE_DEEP = False
 TRACE_ULTRA_DEEP = False
 

src/_packagedcode/pypi.py

@@ -53,7 +53,7 @@
 # TODO: add support for pex, pyz, etc.
 # TODO: Add missing ABOUT file for Pyserial code
 
-TRACE = True
+TRACE = False
 
 
 def logger_debug(*args):

src/python_inspector/api.py

@@ -287,9 +287,6 @@ def resolve_dependencies(
         analyze_setup_py_insecurely=analyze_setup_py_insecurely,
         ignore_errors=ignore_errors,
     )
-    if TRACE:
-        print(f"api.py -> resolve_dependencies() => {purls = }")
-        print(f"api.py -> resolve_dependencies() => {repos = }")
 
     packages = []
 

src/python_inspector/dependencies.py

@@ -21,7 +21,7 @@
 Utilities to resolve dependencies .
 """
 
-TRACE = True
+TRACE = False
 
 
 def get_dependencies_from_requirements(requirements_file="requirements.txt"):

src/python_inspector/resolve_cli.py

@@ -17,7 +17,7 @@
 from python_inspector.cli_utils import FileOptionType
 from python_inspector.utils import write_output_in_file
 
-TRACE = True
+TRACE = False
 
 __version__ = "0.12.0"
 
@@ -257,11 +257,6 @@ def resolve_dependencies(
     )
 
     try:
-        if TRACE:
-            print(f"resolve_cli.py -> resolve_dependencies() => {index_urls = }")
-            print(f"resolve_cli.py -> resolve_dependencies() => {use_pypi_json_api = }")
-            print(f"resolve_cli.py -> resolve_dependencies() => {prefer_source = }")
-            print(f"resolve_cli.py -> resolve_dependencies() => {generic_paths = }")
         resolution_result: Dict = resolver_api(
             requirement_files=requirement_files,
             setup_py_file=setup_py_file,
@@ -293,10 +288,6 @@ def resolve_dependencies(
             output=output,
             location=json_output or pdt_output,
         )
-        if TRACE:
-            print(f"resolve_cli.py -> resolve_dependencies() => {files = }")
-            print(f"resolve_cli.py -> resolve_dependencies() => {resolution_result.packages = }")
-            print(f"resolve_cli.py -> resolve_dependencies() => {resolution_result.resolution = }")
     except Exception:
         import traceback
 

src/python_inspector/utils_pypi.py

@@ -16,7 +16,6 @@
 import shutil
 import tempfile
 import time
-import re
 
 from collections import defaultdict
 from typing import List
@@ -30,8 +29,6 @@
 import packageurl
 import requests
 
-from requests.auth import AuthBase, HTTPBasicAuth
-
 from bs4 import BeautifulSoup
 from commoncode import fileutils
 from commoncode.hash import multi_checksums
@@ -102,8 +99,8 @@
 
 """
 
-TRACE = True
-TRACE_DEEP = True
+TRACE = False
+TRACE_DEEP = False
 TRACE_ULTRA_DEEP = False
 
 # Supported environments
@@ -220,31 +217,6 @@ def get_python_dot_version(version):
 class DistributionNotFound(Exception):
     pass
 
-class HTTPUrlAuth(AuthBase):
-    """Modifies the URL to contain the credentials aka https://user:[email protected]"""
-    def __init__(self, username, password):
-        self.username = username
-        self.password = password
-
-    def __eq__(self, other):
-            return all(
-                [
-                    self.username == getattr(other, "username", None),
-                    self.password == getattr(other, "password", None),
-                ]
-            )
-
-    def __ne__(self, other):
-        return not self == other
-
-    def __call__(self, r):
-        url_re = re.compile("(https?://)(.*)")
-        match = url_re.search(r.url)
-        if match:
-            url = f"{match.group(1)}{self.username}:{self.password}@{match.group(2)}"
-            r.url = url
-        return r
-
 def download_wheel(
     name,
     version,
@@ -281,6 +253,7 @@ def download_wheel(
                 )
             continue
         for wheel in supported_and_valid_wheels:
+            wheel.credentials = repo.credentials
             fetched_wheel_filename = wheel.download(
                 dest_dir=dest_dir,
                 verbose=verbose,
@@ -653,10 +626,6 @@ def get_best_download_url(self, repos=tuple()):
 
         for repo in repos:
             package = repo.get_package_version(name=self.name, version=self.version)
-            if TRACE:
-                print(f"Repo: {repo = }")
-                print(f"get_best_download_url: {self.name = } and {self.version = }")
-                print(f"{repo.index_url = }")
             if not package:
                 if TRACE:
                     print(
@@ -693,9 +662,6 @@ def download(
             )
 
         # FIXME:
-        if TRACE:
-            print(f"utils_pypi.py -> download() => {self.path_or_url = }")
-            print(f"utils_pypi.py -> download() => {dest_dir = }")
         fetch_and_save(
             path_or_url=self.path_or_url,
             dest_dir=dest_dir,
@@ -1153,7 +1119,7 @@ def is_supported_by_tags(self, tags):
         """
         Return True is this wheel is compatible with one of a list of PEP 425 tags.
         """
-        if TRACE_ULTRA_DEEP:
+        if TRACE_DEEP:
             print()
             print("is_supported_by_tags: tags:", tags)
             print("self.tags:", self.tags)
@@ -1169,8 +1135,6 @@ def to_filename(self):
         abis = ".".join(self.abis)
         plats = ".".join(self.platforms)
         name = f"{self.name}-{self.version}{build}-{pyvers}-{abis}-{plats}.whl"
-        if TRACE:
-            print(f"The wheel filename is: {name = }")
         return name
 
     def is_pure(self):
@@ -1634,18 +1598,6 @@ def fetch_links(
         name using the `index_url` of this repository.
         """
         package_url = f"{self.index_url}/{normalized_name}"
-        if TRACE:
-            print(f"utils_pypi.py -> fetch_links() => {package_url = }")
-        # if len(package_url) >= 256:
-        #     base64_re = re.compile(f"https://(.*:.*)@(.*){normalized_name}")
-        #     match = base64_re.search(self.index_url)
-        #     if match:
-        #         auth = match.group(1)
-        #         username = auth.split(":")[0]
-        #         token = auth.split(":")[1]
-        #         remainder = match.group(2)
-        #         new_index_url =  f"https://{username}:auth_token@{remainder}"
-        #         package_url = f"{new_index_url}/{normalized_name}"
         text = CACHE.get(
             path_or_url=package_url,
             credentials=self.credentials,
@@ -1688,7 +1640,10 @@ def resolve_relative_url(package_url, url):
             path = urlunparse(
                 ("", "", url_parts.path, url_parts.params, url_parts.query, url_parts.fragment)
             )
-        resolved_url_parts = base_url_parts._replace(path=path)
+        if base_url_parts.path != "":
+            resolved_url_parts = base_url_parts._replace(path=base_url_parts.path + "/" + path)
+        else:
+            resolved_url_parts = base_url_parts._replace(path=path)
         url = urlunparse(resolved_url_parts)
     return url
 
@@ -1731,15 +1686,13 @@ def get(
         True otherwise as treat as binary. `path_or_url` can be a path or a URL
         to a file.
         """
+
+
         cache_key = quote_plus(path_or_url.strip("/"))
         cached = os.path.join(self.directory, cache_key)
-        if TRACE:
-            print(f"Cache.get() => {path_or_url = }")
-            print(f"Cache.get() => {cache_key = }")
-            print(f"Cache.get() => {cached = }")
 
         if force or not os.path.exists(cached):
-            if TRACE:
+            if TRACE_DEEP:
                 print(f"        FILE CACHE MISS: {path_or_url}")
             content = get_file_content(
                 path_or_url=path_or_url,
@@ -1753,7 +1706,7 @@ def get(
                 fo.write(content)
             return content
         else:
-            if TRACE:
+            if TRACE_DEEP:
                 print(f"        FILE CACHE HIT: {path_or_url}")
             return get_local_file_content(path=cached, as_text=as_text)
 
@@ -1773,7 +1726,7 @@ def get_file_content(
     remote URL. Return the content as bytes is `as_text` is False.
     """
     if path_or_url.startswith("https://"):
-        if TRACE:
+        if TRACE_DEEP:
             print(f"Fetching: {path_or_url}")
         _headers, content = get_remote_file_content(
             url=path_or_url,
@@ -1839,13 +1792,11 @@ def get_remote_file_content(
     if verbose:
         echo_func(f"DOWNLOADING: {url}")
 
-    # if credentials and not change_auth:
-    #     auth = HTTPUrlAuth(credentials.get("login"), credentials.get("password"))
-    # elif credentials and change_auth:
-    #     auth = HTTPBasicAuth(credentials.get("login"), credentials.get("password"))
+    if TRACE:
+        print(f"DOWNLOADING: {url}")
 
     if credentials:
-        auth = HTTPBasicAuth(credentials.get("login"), credentials.get("password"))
+        auth = (credentials.get("login"), credentials.get("password"))
     else:
         auth = None
 
@@ -1859,15 +1810,8 @@ def get_remote_file_content(
 
     with stream as response:
         status = response.status_code
-        if TRACE:
-            print(f"The status is {status}")
-            print(f"The full response is: {response}")
-            print(f"The response url is: {response.url}")
-            # print(f"The original prepared request headers are: {response.request.headers}")
         if status != requests.codes.ok: # NOQA
             if status == 429 and _delay < 20:
-                if TRACE:
-                    print("Delay exponentially...")
                 # too many requests: start some exponential delay
                 increased_delay = (_delay * 2) or 1
 
@@ -1903,8 +1847,6 @@ def fetch_and_save(
     errors. Treats the content as text if as_text is True otherwise as treat as
     binary.
     """
-    if TRACE:
-        print(f"fetch_and_save() => {path_or_url = }")
     content = CACHE.get(
         path_or_url=path_or_url,
         credentials=credentials,