Add tests for live_eval

Signed-off-by: Tushar Goel <[email protected]>

Author: TG1999

src/python_inspector/resolution.py

@@ -26,7 +26,6 @@
 from packaging.version import LegacyVersion
 from packaging.version import Version
 from packaging.version import parse as parse_version
-from requirements_builder.requirements_builder import iter_requirements
 from resolvelib import AbstractProvider
 from resolvelib import Resolver
 from resolvelib.reporters import BaseReporter
@@ -40,6 +39,7 @@
 from _packagedcode.pypi import SetupCfgHandler
 from _packagedcode.pypi import can_process_dependent_package
 from python_inspector import utils_pypi
+from python_inspector.setup_py_live_eval import iter_requirements
 from python_inspector.utils_pypi import Environment
 from python_inspector.utils_pypi import PypiSimpleRepository
 
@@ -97,6 +97,8 @@ def get_requirements_from_distribution(
     Return a list of requirements from a source distribution or wheel at
     ``location`` using the provided ``handler`` DatafileHandler for parsing.
     """
+    if not location:
+        return []
     if not os.path.exists(location):
         return []
     deps = []
@@ -137,9 +139,8 @@ def parse_setup_py_insecurely(setup_py):
     """
     if not os.path.exists(setup_py):
         return []
-    unparsed_requirements = iter_requirements(level="", extras=[], setup_file=setup_py)
-    for requirement in unparsed_requirements:
-        yield Requirement(requirement)
+    for req in iter_requirements(level="", extras=[], setup_file=setup_py):
+        yield Requirement(req)
 
 
 def is_valid_version(
@@ -373,61 +374,7 @@ def get_requirements_for_package_from_pypi_simple(
             )
             if not sdist_location:
                 return
-            setup_py_location = os.path.join(
-                sdist_location,
-                "setup.py",
-            )
-            setup_cfg_location = os.path.join(
-                sdist_location,
-                "setup.cfg",
-            )
-
-            if not os.path.exists(setup_py_location) and not os.path.exists(setup_cfg_location):
-                raise Exception(f"No setup.py or setup.cfg found in pypi sdist {sdist_location}")
-
-            # Some commonon packages like flask may have some dependencies in setup.cfg
-            # and some dependencies in setup.py. We are going to check both.
-            location_by_sdist_parser = {
-                PythonSetupPyHandler: setup_py_location,
-                SetupCfgHandler: setup_cfg_location,
-            }
-
-            # Set to True if we found any dependencies in setup.py or setup.cfg
-            has_deps = False
-
-            for handler, location in location_by_sdist_parser.items():
-                deps = get_requirements_from_distribution(
-                    handler=handler,
-                    location=location,
-                )
-                if deps:
-                    has_deps = True
-                    yield from deps
-
-            if not has_deps and contain_string(
-                string="requirements.txt", files=[setup_py_location, setup_cfg_location]
-            ):
-                # Look in requirements file if and only if thy are refered in setup.py or setup.cfg
-                # And no deps have been yielded by requirements file.
-                requirement_location = os.path.join(
-                    sdist_location,
-                    "requirements.txt",
-                )
-                deps = get_requirements_from_distribution(
-                    handler=PipRequirementsFileHandler,
-                    location=requirement_location,
-                )
-                if deps:
-                    has_deps = True
-                    yield from deps
-
-            if not has_deps and contain_string(
-                string="_require", files=[setup_py_location, setup_cfg_location]
-            ):
-                if self.insecure:
-                    yield from parse_setup_py_insecurely(setup_py=setup_py_location)
-                else:
-                    raise Exception("Unable to collect setup.py dependencies securely")
+            yield from get_setup_dependencies(location=sdist_location, insecure=self.insecure)
 
     def get_requirements_for_package_from_pypi_json_api(
         self, purl: PackageURL
@@ -704,6 +651,70 @@ def format_pdt_tree(results):
     return dependencies
 
 
+def get_setup_dependencies(location, insecure=False, use_requirements=True):
+    """
+    Yield dependencies from the given setup.py and setup.cfg location.
+    """
+
+    setup_py_location = os.path.join(
+        location,
+        "setup.py",
+    )
+    setup_cfg_location = os.path.join(
+        location,
+        "setup.cfg",
+    )
+
+    if not os.path.exists(setup_py_location) and not os.path.exists(setup_cfg_location):
+        raise Exception(f"No setup.py or setup.cfg found in pypi sdist {location}")
+
+    # Some commonon packages like flask may have some dependencies in setup.cfg
+    # and some dependencies in setup.py. We are going to check both.
+    location_by_sdist_parser = {
+        PythonSetupPyHandler: setup_py_location,
+        SetupCfgHandler: setup_cfg_location,
+    }
+
+    # Set to True if we found any dependencies in setup.py or setup.cfg
+    has_deps = False
+
+    for handler, location in location_by_sdist_parser.items():
+        deps = get_requirements_from_distribution(
+            handler=handler,
+            location=location,
+        )
+        if deps:
+            has_deps = True
+            yield from deps
+
+    if (
+        use_requirements
+        and not has_deps
+        and contain_string(string="requirements.txt", files=[setup_py_location, setup_cfg_location])
+    ):
+        # Look in requirements file if and only if thy are refered in setup.py or setup.cfg
+        # And no deps have been yielded by requirements file.
+        requirement_location = os.path.join(
+            location,
+            "requirements.txt",
+        )
+        deps = get_requirements_from_distribution(
+            handler=PipRequirementsFileHandler,
+            location=requirement_location,
+        )
+        if deps:
+            has_deps = True
+            yield from deps
+
+    if not has_deps and contain_string(
+        string="_require", files=[setup_py_location, setup_cfg_location]
+    ):
+        if insecure:
+            yield from parse_setup_py_insecurely(setup_py=setup_py_location)
+        else:
+            raise Exception("Unable to collect setup.py dependencies securely")
+
+
 def get_resolved_dependencies(
     requirements: List[Requirement],
     environment: Environment = None,

src/python_inspector/setup_py_live_eval.py

@@ -9,7 +9,8 @@
 #
 """Generate requirements from `setup.py` and `requirements-devel.txt`."""
 
-from __future__ import absolute_import, print_function
+from __future__ import absolute_import
+from __future__ import print_function
 
 import os
 import re
@@ -37,18 +38,16 @@ def parse_set(string):
 def minver_error(pkg_name):
     """Report error about missing minimum version constraint and exit."""
     print(
-        'ERROR: specify minimal version of "{0}" using '
-        '">=" or "=="'.format(pkg_name),
-        file=sys.stderr
+        'ERROR: specify minimal version of "{0}" using ' '">=" or "=="'.format(pkg_name),
+        file=sys.stderr,
     )
     sys.exit(1)
 
 
 def build_pkg_name(pkg):
     """Build package name, including extras if present."""
     if pkg.extras:
-        return '{0}[{1}]'.format(
-            pkg.project_name, ','.join(sorted(pkg.extras)))
+        return "{0}[{1}]".format(pkg.project_name, ",".join(sorted(pkg.extras)))
     return pkg.project_name
 
 
@@ -66,96 +65,77 @@ def parse_pip_file(path):
                 line = line.strip()
 
                 # see https://pip.readthedocs.io/en/1.1/requirements.html
-                if line.startswith('-e'):
+                if line.startswith("-e"):
                     # devel requirement
-                    splitted = line.split('#egg=')
+                    splitted = line.split("#egg=")
                     rdev[splitted[1].lower()] = line
 
-                elif line.startswith('-r'):
+                elif line.startswith("-r"):
                     # recursive file command
-                    splitted = re.split('-r\\s+', line)
+                    splitted = re.split("-r\\s+", line)
                     subrdev, subrnormal, substuff = parse_pip_file(
                         os.path.join(os.path.dirname(path), splitted[1])
                     )
                     for k, v in subrdev.items():
                         if k not in rdev:
                             rdev[k] = v
                     rnormal.extend(subrnormal)
-                elif line.startswith('-'):
+                elif line.startswith("-"):
                     # another special command we don't recognize
                     stuff.append(line)
                 else:
                     # ordinary requirement, similarly to them used in setup.py
                     rnormal.append(line)
     except IOError:
-        print(
-            'Warning: could not parse requirements file "{0}"!'.format(path),
-            file=sys.stderr
-        )
+        print('Warning: could not parse requirements file "{0}"!'.format(path), file=sys.stderr)
 
     return rdev, rnormal, stuff
 
 
-def iter_requirements(level, extras, pip_file, setup_fp, setup_cfg_fp=None):
+def iter_requirements(level, extras, setup_file):
     """Iterate over requirements."""
+    from pathlib import Path
+
+    setup_file = str(Path(setup_file).absolute())
     result = dict()
     requires = []
     stuff = []
-    if level == 'dev' or setup_fp is None:
-        result, requires, stuff = parse_pip_file(pip_file)
-
+    cd = os.getcwd()
+    os.chdir(os.path.dirname(setup_file))
     install_requires = []
     requires_extras = {}
-    if setup_fp is not None:
-        with mock.patch.object(setuptools, 'setup') as mock_setup:
-            sys.path.append(os.path.dirname(setup_fp.name))
-            g = {'__file__': setup_fp.name, '__name__': '__main__'}
-            exec(setup_fp.read(), g)
-            sys.path.pop()
-            assert g['setup']  # silence warning about unused imports
-
-        # called arguments are in `mock_setup.call_args`
-        mock_args, mock_kwargs = mock_setup.call_args
-        install_requires = mock_kwargs.get(
-            'install_requires', install_requires
-        )
-        requires_extras = mock_kwargs.get('extras_require', requires_extras)
-
-    if setup_cfg_fp is not None:
-        parser = configparser.ConfigParser()
-        parser.read_file(setup_cfg_fp)
-
-        if parser.has_section("options"):
-            value = parser.get("options", "install_requires",
-                               fallback="").strip()
-
-            if value:
-                install_requires = [s.strip() for s in value.splitlines()]
-
-        if parser.has_section("options.extras_require"):
-            for name, value in parser.items("options.extras_require"):
-                requires_extras[name] = [s.strip()
-                                         for s in value.strip().splitlines()]
-
-    install_requires.extend(requires)
+    # change directory to setup.py path
+    with mock.patch.object(setuptools, "setup") as mock_setup:
+        sys.path.append(os.path.dirname(setup_file))
+        g = {"__file__": setup_file, "__name__": "__main__"}
+        with open(setup_file) as sf:
+            exec(sf.read(), g)
+        sys.path.pop()
+        assert g["setup"]  # silence warning about unused imports
+    # called arguments are in `mock_setup.call_args`
+    os.chdir(cd)
+    mock_args, mock_kwargs = mock_setup.call_args
+    install_requires = mock_kwargs.get("install_requires", install_requires)
+
+    requires_extras = mock_kwargs.get("extras_require", requires_extras)
 
     for e, reqs in requires_extras.items():
         # Handle conditions on extras. See pkginfo_to_metadata function
         # in Wheel for details.
-        condition = ''
-        if ':' in e:
-            e, condition = e.split(':', 1)
+        condition = ""
+        if ":" in e:
+            e, condition = e.split(":", 1)
         if not e or e in extras:
             if condition:
-                reqs = ['{0}; {1}'.format(r, condition) for r in reqs]
+                reqs = ["{0}; {1}".format(r, condition) for r in reqs]
             install_requires.extend(reqs)
 
     for pkg in pkg_resources.parse_requirements(install_requires):
         # skip things we already know
         # FIXME be smarter about merging things
 
         # Evaluate environment markers skip if not applicable
-        if hasattr(pkg, 'marker') and pkg.marker is not None:
+        if hasattr(pkg, "marker") and pkg.marker is not None:
             if not pkg.marker.evaluate():
                 continue
             else:
@@ -166,44 +146,37 @@ def iter_requirements(level, extras, pip_file, setup_fp, setup_cfg_fp=None):
             continue
 
         specs = dict(pkg.specs)
-        if (('>=' in specs) and ('>' in specs)) \
-                or (('<=' in specs) and ('<' in specs)):
+        if ((">=" in specs) and (">" in specs)) or (("<=" in specs) and ("<" in specs)):
             print(
-                'ERROR: Do not specify such weird constraints! '
-                '("{0}")'.format(pkg),
-                file=sys.stderr
+                "ERROR: Do not specify such weird constraints! " '("{0}")'.format(pkg),
+                file=sys.stderr,
             )
             sys.exit(1)
 
-        if '==' in specs:
-            result[pkg.key] = '{0}=={1}'.format(
-                build_pkg_name(pkg), specs['=='])
+        if "==" in specs:
+            result[pkg.key] = "{0}=={1}".format(build_pkg_name(pkg), specs["=="])
 
-        elif '>=' in specs:
-            if level == 'min':
-                result[pkg.key] = '{0}=={1}'.format(
-                    build_pkg_name(pkg), specs['>=']
-                )
+        elif ">=" in specs:
+            if level == "min":
+                result[pkg.key] = "{0}=={1}".format(build_pkg_name(pkg), specs[">="])
             else:
                 result[pkg.key] = pkg
 
-        elif '>' in specs:
-            if level == 'min':
+        elif ">" in specs:
+            if level == "min":
                 minver_error(build_pkg_name(pkg))
             else:
                 result[pkg.key] = pkg
 
-        elif '~=' in specs:
-            if level == 'min':
-                result[pkg.key] = '{0}=={1}'.format(
-                    build_pkg_name(pkg), specs['~='])
+        elif "~=" in specs:
+            if level == "min":
+                result[pkg.key] = "{0}=={1}".format(build_pkg_name(pkg), specs["~="])
             else:
-                ver, _ = os.path.splitext(specs['~='])
-                result[pkg.key] = '{0}>={1},=={2}.*'.format(
-                    build_pkg_name(pkg), specs['~='], ver)
+                ver, _ = os.path.splitext(specs["~="])
+                result[pkg.key] = "{0}>={1},=={2}.*".format(build_pkg_name(pkg), specs["~="], ver)
 
         else:
-            if level == 'min':
+            if level == "min":
                 minver_error(build_pkg_name(pkg))
             else:
                 result[pkg.key] = build_pkg_name(pkg)