feat(data-pipelines): allow pinning of hail in requirements

.github/workflows/deploy-scripts-ci.yml

@@ -19,7 +19,7 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@v2
         with:
-          python-version: 3.7
+          python-version: 3.9
       - name: Use pip cache
         uses: actions/cache@v2
         with:

data-pipeline/requirements.txt

@@ -1,5 +1,5 @@
 elasticsearch~=7.17
-hail
+hail==0.2.127
 tqdm
 loguru
 attrs

deploy/deployctl/subcommands/dataproc_cluster.py

@@ -3,6 +3,7 @@
 import subprocess
 import sys
 import typing
+import importlib.metadata
 
 from deployctl.config import config
 
@@ -24,6 +25,22 @@ def start_cluster(name: str, cluster_args: typing.List[str]) -> None:
         raise RuntimeError("project configuration is required")
 
     with open(os.path.join(DATA_PIPELINE_DIRECTORY, "requirements.txt")) as requirements_file:
+        requirements_hail_version = next(
+            (line.strip().split("==")[1] for line in requirements_file if line.strip().startswith("hail==")), None
+        )
+        try:
+            local_hail_version = importlib.metadata.version("hail")
+        except importlib.metadata.PackageNotFoundError as package_not_found:
+            raise RuntimeError("Hail must be installed locally") from package_not_found
+
+        if not requirements_hail_version:
+            raise RuntimeError("Hail must be pinned in data-pipeline/requirements.txt")
+        if requirements_hail_version != local_hail_version:
+            raise RuntimeError(
+                f"Local hail version differs from version pinned in data-pipeline/requirements.txt\nRequired version {requirements_hail_version}\nLocal version {local_hail_version}"
+            )
+
+        requirements_file.seek(0)
         requirements = [line.strip() for line in requirements_file.readlines()]
 
     subprocess.check_output(

requirements-dev.txt

@@ -1,2 +1,2 @@
 black==22.3.0  # This should be kept in sync with the version in .pre-commit-config.yaml
-pylint==2.6.0
+pylint==2.7.1