|
| 1 | +""" |
| 2 | +Dependency version management for testing. |
| 3 | +Generates requirements files for min and default dependency versions. |
| 4 | +For min versions, creates flexible constraints (e.g., >=1.2.5,<1.3.0) to allow |
| 5 | +compatible patch updates instead of pinning exact versions. |
| 6 | +""" |
| 7 | + |
| 8 | +import toml |
| 9 | +import sys |
| 10 | +import argparse |
| 11 | +from packaging.specifiers import SpecifierSet |
| 12 | +from packaging.requirements import Requirement |
| 13 | +from pathlib import Path |
| 14 | + |
| 15 | +class DependencyManager: |
| 16 | + def __init__(self, pyproject_path="pyproject.toml"): |
| 17 | + self.pyproject_path = Path(pyproject_path) |
| 18 | + self.dependencies = self._load_dependencies() |
| 19 | + |
| 20 | + # Map of packages that need specific transitive dependency constraints when downgraded |
| 21 | + self.transitive_dependencies = { |
| 22 | + 'pandas': { |
| 23 | + # When pandas is downgraded to 1.x, ensure numpy compatibility |
| 24 | + 'numpy': { |
| 25 | + 'min_constraint': '>=1.16.5,<2.0.0', # pandas 1.x works with numpy 1.x |
| 26 | + 'applies_when': lambda version: version.startswith('1.') |
| 27 | + } |
| 28 | + } |
| 29 | + } |
| 30 | + |
| 31 | + def _load_dependencies(self): |
| 32 | + """Load dependencies from pyproject.toml""" |
| 33 | + with open(self.pyproject_path, 'r') as f: |
| 34 | + pyproject = toml.load(f) |
| 35 | + return pyproject['tool']['poetry']['dependencies'] |
| 36 | + |
| 37 | + def _parse_constraint(self, name, constraint): |
| 38 | + """Parse a dependency constraint into version info""" |
| 39 | + if isinstance(constraint, str): |
| 40 | + return constraint, False # version_constraint, is_optional |
| 41 | + elif isinstance(constraint, list): |
| 42 | + # Handle complex constraints like pandas/pyarrow |
| 43 | + first_constraint = constraint[0] |
| 44 | + version = first_constraint['version'] |
| 45 | + is_optional = first_constraint.get('optional', False) |
| 46 | + return version, is_optional |
| 47 | + elif isinstance(constraint, dict): |
| 48 | + if 'version' in constraint: |
| 49 | + return constraint['version'], constraint.get('optional', False) |
| 50 | + return None, False |
| 51 | + |
| 52 | + def _extract_versions_from_specifier(self, spec_set_str): |
| 53 | + """Extract minimum version from a specifier set""" |
| 54 | + try: |
| 55 | + # Handle caret (^) and tilde (~) constraints that packaging doesn't support |
| 56 | + if spec_set_str.startswith('^'): |
| 57 | + # ^1.2.3 means >=1.2.3, <2.0.0 |
| 58 | + min_version = spec_set_str[1:] # Remove ^ |
| 59 | + return min_version, None |
| 60 | + elif spec_set_str.startswith('~'): |
| 61 | + # ~1.2.3 means >=1.2.3, <1.3.0 |
| 62 | + min_version = spec_set_str[1:] # Remove ~ |
| 63 | + return min_version, None |
| 64 | + |
| 65 | + spec_set = SpecifierSet(spec_set_str) |
| 66 | + min_version = None |
| 67 | + |
| 68 | + for spec in spec_set: |
| 69 | + if spec.operator in ('>=', '=='): |
| 70 | + min_version = spec.version |
| 71 | + break |
| 72 | + |
| 73 | + return min_version, None |
| 74 | + except Exception as e: |
| 75 | + print(f"Warning: Could not parse constraint '{spec_set_str}': {e}", file=sys.stderr) |
| 76 | + return None, None |
| 77 | + |
| 78 | + def _create_flexible_minimum_constraint(self, package_name, min_version): |
| 79 | + """Create a flexible minimum constraint that allows compatible updates""" |
| 80 | + try: |
| 81 | + # Split version into parts |
| 82 | + version_parts = min_version.split('.') |
| 83 | + |
| 84 | + if len(version_parts) >= 2: |
| 85 | + major = version_parts[0] |
| 86 | + minor = version_parts[1] |
| 87 | + |
| 88 | + # Special handling for packages that commonly have conflicts |
| 89 | + # For these packages, use wider constraints to allow more compatibility |
| 90 | + if package_name in ['requests', 'urllib3', 'pandas']: |
| 91 | + # Use wider constraint: >=min_version,<next_major |
| 92 | + # e.g., 2.18.1 becomes >=2.18.1,<3.0.0 |
| 93 | + next_major = int(major) + 1 |
| 94 | + upper_bound = f"{next_major}.0.0" |
| 95 | + return f"{package_name}>={min_version},<{upper_bound}" |
| 96 | + else: |
| 97 | + # For other packages, use minor version constraint |
| 98 | + # e.g., 1.2.5 becomes >=1.2.5,<1.3.0 |
| 99 | + next_minor = int(minor) + 1 |
| 100 | + upper_bound = f"{major}.{next_minor}.0" |
| 101 | + return f"{package_name}>={min_version},<{upper_bound}" |
| 102 | + else: |
| 103 | + # If version doesn't have minor version, just use exact version |
| 104 | + return f"{package_name}=={min_version}" |
| 105 | + |
| 106 | + except (ValueError, IndexError) as e: |
| 107 | + print(f"Warning: Could not create flexible constraint for {package_name}=={min_version}: {e}", file=sys.stderr) |
| 108 | + # Fallback to exact version |
| 109 | + return f"{package_name}=={min_version}" |
| 110 | + |
| 111 | + def _get_transitive_dependencies(self, package_name, version, version_type): |
| 112 | + """Get transitive dependencies that need specific constraints based on the main package version""" |
| 113 | + transitive_reqs = [] |
| 114 | + |
| 115 | + if package_name in self.transitive_dependencies: |
| 116 | + transitive_deps = self.transitive_dependencies[package_name] |
| 117 | + |
| 118 | + for dep_name, dep_config in transitive_deps.items(): |
| 119 | + # Check if this transitive dependency applies for this version |
| 120 | + if dep_config['applies_when'](version): |
| 121 | + if version_type == "min": |
| 122 | + # Use the predefined constraint for minimum versions |
| 123 | + constraint = dep_config['min_constraint'] |
| 124 | + transitive_reqs.append(f"{dep_name}{constraint}") |
| 125 | + # For default version_type, we don't add transitive deps as Poetry handles them |
| 126 | + |
| 127 | + return transitive_reqs |
| 128 | + |
| 129 | + def generate_requirements(self, version_type="min", include_optional=False): |
| 130 | + """ |
| 131 | + Generate requirements for specified version type. |
| 132 | + |
| 133 | + Args: |
| 134 | + version_type: "min" or "default" |
| 135 | + include_optional: Whether to include optional dependencies |
| 136 | + """ |
| 137 | + requirements = [] |
| 138 | + transitive_requirements = [] |
| 139 | + |
| 140 | + for name, constraint in self.dependencies.items(): |
| 141 | + if name == 'python': |
| 142 | + continue |
| 143 | + |
| 144 | + version_constraint, is_optional = self._parse_constraint(name, constraint) |
| 145 | + if not version_constraint: |
| 146 | + continue |
| 147 | + |
| 148 | + if is_optional and not include_optional: |
| 149 | + continue |
| 150 | + |
| 151 | + if version_type == "default": |
| 152 | + # For default, just use the constraint as-is (let poetry resolve) |
| 153 | + requirements.append(f"{name}{version_constraint}") |
| 154 | + elif version_type == "min": |
| 155 | + min_version, _ = self._extract_versions_from_specifier(version_constraint) |
| 156 | + if min_version: |
| 157 | + # Create flexible constraint that allows patch updates for compatibility |
| 158 | + flexible_constraint = self._create_flexible_minimum_constraint(name, min_version) |
| 159 | + requirements.append(flexible_constraint) |
| 160 | + |
| 161 | + # Check if this package needs specific transitive dependencies |
| 162 | + transitive_deps = self._get_transitive_dependencies(name, min_version, version_type) |
| 163 | + transitive_requirements.extend(transitive_deps) |
| 164 | + |
| 165 | + # Combine main requirements with transitive requirements |
| 166 | + all_requirements = requirements + transitive_requirements |
| 167 | + |
| 168 | + # Remove duplicates (prefer main requirements over transitive ones) |
| 169 | + seen_packages = set() |
| 170 | + final_requirements = [] |
| 171 | + |
| 172 | + # First add main requirements |
| 173 | + for req in requirements: |
| 174 | + package_name = Requirement(req).name |
| 175 | + seen_packages.add(package_name) |
| 176 | + final_requirements.append(req) |
| 177 | + |
| 178 | + # Then add transitive requirements that don't conflict |
| 179 | + for req in transitive_requirements: |
| 180 | + package_name = Requirement(req).name |
| 181 | + if package_name not in seen_packages: |
| 182 | + final_requirements.append(req) |
| 183 | + |
| 184 | + return final_requirements |
| 185 | + |
| 186 | + |
| 187 | + def write_requirements_file(self, filename, version_type="min", include_optional=False): |
| 188 | + """Write requirements to a file""" |
| 189 | + requirements = self.generate_requirements(version_type, include_optional) |
| 190 | + |
| 191 | + with open(filename, 'w') as f: |
| 192 | + if version_type == "min": |
| 193 | + f.write(f"# Minimum compatible dependency versions generated from pyproject.toml\n") |
| 194 | + f.write(f"# Uses flexible constraints to resolve compatibility conflicts:\n") |
| 195 | + f.write(f"# - Common packages (requests, urllib3, pandas): >=min,<next_major\n") |
| 196 | + f.write(f"# - Other packages: >=min,<next_minor\n") |
| 197 | + f.write(f"# - Includes transitive dependencies (e.g., numpy for pandas)\n") |
| 198 | + else: |
| 199 | + f.write(f"# {version_type.title()} dependency versions generated from pyproject.toml\n") |
| 200 | + for req in sorted(requirements): |
| 201 | + f.write(f"{req}\n") |
| 202 | + |
| 203 | + print(f"Generated {filename} with {len(requirements)} dependencies") |
| 204 | + return requirements |
| 205 | + |
| 206 | +def main(): |
| 207 | + parser = argparse.ArgumentParser(description="Manage dependency versions for testing") |
| 208 | + parser.add_argument("version_type", choices=["min", "default"], |
| 209 | + help="Type of versions to generate") |
| 210 | + parser.add_argument("--output", "-o", default=None, |
| 211 | + help="Output requirements file (default: requirements-{version_type}.txt)") |
| 212 | + parser.add_argument("--include-optional", action="store_true", |
| 213 | + help="Include optional dependencies") |
| 214 | + parser.add_argument("--pyproject", default="pyproject.toml", |
| 215 | + help="Path to pyproject.toml file") |
| 216 | + |
| 217 | + args = parser.parse_args() |
| 218 | + |
| 219 | + if args.output is None: |
| 220 | + args.output = f"requirements-{args.version_type}.txt" |
| 221 | + |
| 222 | + manager = DependencyManager(args.pyproject) |
| 223 | + requirements = manager.write_requirements_file( |
| 224 | + args.output, |
| 225 | + args.version_type, |
| 226 | + args.include_optional |
| 227 | + ) |
| 228 | + |
| 229 | + # Also print to stdout for GitHub Actions |
| 230 | + for req in requirements: |
| 231 | + print(req) |
| 232 | + |
| 233 | +if __name__ == "__main__": |
| 234 | + main() |
0 commit comments