tests: Move beam under toxgen (#4759)

- move beam under toxgen
- lower the waiting time between pypi requests

Ref #4506

Author: sentrivana

.github/workflows/test-integrations-tasks.yml

@@ -29,7 +29,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.7","3.8","3.10","3.11","3.12","3.13"]
+        python-version: ["3.7","3.10","3.11","3.12","3.13"]
         # python3.6 reached EOL and is no longer being supported on
         # new versions of hosted runners on Github Actions
         # ubuntu-20.04 is the last version that supported python3.6

scripts/populate_tox/config.py

@@ -36,6 +36,10 @@
             "<=0.23": ["pydantic<2"],
         },
     },
+    "beam": {
+        "package": "apache-beam",
+        "python": ">=3.7",
+    },
     "bottle": {
         "package": "bottle",
         "deps": {

scripts/populate_tox/populate_tox.py

@@ -40,7 +40,7 @@
     lstrip_blocks=True,
 )
 
-PYPI_COOLDOWN = 0.15  # seconds to wait between requests to PyPI
+PYPI_COOLDOWN = 0.1  # seconds to wait between requests to PyPI
 
 PYPI_PROJECT_URL = "https://pypi.python.org/pypi/{project}/json"
 PYPI_VERSION_URL = "https://pypi.python.org/pypi/{project}/{version}/json"
@@ -68,7 +68,6 @@
     # Integrations that can be migrated -- we should eventually remove all
     # of these from the IGNORE list
     "asyncpg",
-    "beam",
     "boto3",
     "chalice",
     "gcp",

scripts/populate_tox/tox.jinja

@@ -46,10 +46,6 @@ envlist =
     # AWS Lambda
     {py3.8,py3.9,py3.11,py3.13}-aws_lambda
 
-    # Beam
-    {py3.7}-beam-v{2.12}
-    {py3.8,py3.11}-beam-latest
-
     # Boto3
     {py3.6,py3.7}-boto3-v{1.12}
     {py3.7,py3.11,py3.12}-boto3-v{1.23}
@@ -177,10 +173,6 @@ deps =
     aws_lambda: requests
     aws_lambda: uvicorn
 
-    # Beam
-    beam-v2.12: apache-beam~=2.12.0
-    beam-latest: apache-beam
-
     # Boto3
     boto3-v1.12: boto3~=1.12.0
     boto3-v1.23: boto3~=1.23.0

tox.ini

@@ -10,7 +10,7 @@
 # The file (and all resulting CI YAMLs) then need to be regenerated via
 # "scripts/generate-test-files.sh".
 #
-# Last generated: 2025-09-04T07:00:53.509946+00:00
+# Last generated: 2025-09-04T10:35:13.756355+00:00
 
 [tox]
 requires =
@@ -46,10 +46,6 @@ envlist =
     # AWS Lambda
     {py3.8,py3.9,py3.11,py3.13}-aws_lambda
 
-    # Beam
-    {py3.7}-beam-v{2.12}
-    {py3.8,py3.11}-beam-latest
-
     # Boto3
     {py3.6,py3.7}-boto3-v{1.12}
     {py3.7,py3.11,py3.12}-boto3-v{1.23}
@@ -228,6 +224,11 @@ envlist =
     {py3.7,py3.10,py3.11}-arq-v0.25.0
     {py3.8,py3.11,py3.12}-arq-v0.26.3
 
+    {py3.7}-beam-v2.14.0
+    {py3.7,py3.8}-beam-v2.32.0
+    {py3.8,py3.10,py3.11}-beam-v2.50.0
+    {py3.9,py3.12,py3.13}-beam-v2.67.0
+
     {py3.6,py3.7,py3.8}-celery-v4.4.7
     {py3.6,py3.7,py3.8}-celery-v5.0.5
     {py3.8,py3.12,py3.13}-celery-v5.5.3
@@ -371,10 +372,6 @@ deps =
     aws_lambda: requests
     aws_lambda: uvicorn
 
-    # Beam
-    beam-v2.12: apache-beam~=2.12.0
-    beam-latest: apache-beam
-
     # Boto3
     boto3-v1.12: boto3~=1.12.0
     boto3-v1.23: boto3~=1.23.0
@@ -633,6 +630,11 @@ deps =
     arq: fakeredis>=2.2.0,<2.8
     arq-v0.23: pydantic<2
 
+    beam-v2.14.0: apache-beam==2.14.0
+    beam-v2.32.0: apache-beam==2.32.0
+    beam-v2.50.0: apache-beam==2.50.0
+    beam-v2.67.0: apache-beam==2.67.0
+
     celery-v4.4.7: celery==4.4.7
     celery-v5.0.5: celery==5.0.5
     celery-v5.5.3: celery==5.5.3