CI/TST: Use CI moto service instead of subprocess (#44612)

Author: mroeschke

.github/workflows/ci.yml

@@ -162,6 +162,14 @@ jobs:
   data_manager:
     name: Test experimental data manager
     runs-on: ubuntu-latest
+    services:
+      moto:
+        image: motoserver/moto
+        env:
+          AWS_ACCESS_KEY_ID: foobar_key
+          AWS_SECRET_ACCESS_KEY: foobar_secret
+        ports:
+          - 5000:5000
     strategy:
       matrix:
         pattern: ["not slow and not network and not clipboard", "slow"]

.github/workflows/database.yml

@@ -63,6 +63,14 @@ jobs:
         ports:
           - 5432:5432
 
+      moto:
+        image: motoserver/moto
+        env:
+          AWS_ACCESS_KEY_ID: foobar_key
+          AWS_SECRET_ACCESS_KEY: foobar_secret
+        ports:
+          - 5000:5000
+
     steps:
     - name: Checkout
       uses: actions/checkout@v2

.github/workflows/posix.yml

@@ -49,6 +49,14 @@ jobs:
       # https://github.community/t/concurrecy-not-work-for-push/183068/7
       group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-${{ matrix.settings[0] }}
       cancel-in-progress: true
+    services:
+      moto:
+        image: motoserver/moto
+        env:
+          AWS_ACCESS_KEY_ID: foobar_key
+          AWS_SECRET_ACCESS_KEY: foobar_secret
+        ports:
+          - 5000:5000
 
     steps:
     - name: Checkout

ci/deps/actions-38-db-min.yaml

@@ -38,8 +38,6 @@ dependencies:
   - xlrd
   - xlsxwriter
   - xlwt
-  - moto
-  - flask
 
   # sql
   - psycopg2=2.8.4

ci/deps/actions-38-db.yaml

@@ -14,6 +14,7 @@ dependencies:
   # pandas dependencies
   - aiobotocore<2.0.0
   - beautifulsoup4
+  - boto3
   - botocore>=1.11
   - dask
   - fastparquet>=0.4.0
@@ -22,8 +23,6 @@ dependencies:
   - geopandas
   - html5lib
   - matplotlib
-  - moto>=1.3.14
-  - flask
   - nomkl
   - numexpr
   - numpy=1.18

ci/deps/actions-38-locale.yaml

@@ -14,14 +14,12 @@ dependencies:
 
   # pandas dependencies
   - beautifulsoup4
-  - flask
   - html5lib
   - ipython
   - jinja2
   - jedi
   - lxml
   - matplotlib<3.3.0
-  - moto
   - nomkl
   - numexpr
   - numpy<1.20  # GH#39541 compat with pyarrow<3
@@ -34,7 +32,6 @@ dependencies:
   - xlrd
   - xlsxwriter
   - xlwt
-  - moto
   - pyarrow=1.0.1
   - pip
   - pip:

ci/deps/actions-38-slow.yaml

@@ -13,6 +13,7 @@ dependencies:
 
   # pandas dependencies
   - beautifulsoup4
+  - boto3
   - fsspec>=0.7.4
   - html5lib
   - lxml
@@ -27,12 +28,9 @@ dependencies:
   - python-dateutil
   - pytz
   - s3fs>=0.4.0
-  - moto>=1.3.14
   - scipy
   - sqlalchemy
   - xlrd
   - xlsxwriter
   - xlwt
-  - moto
-  - flask
   - numba

ci/deps/actions-39-slow.yaml

@@ -15,14 +15,13 @@ dependencies:
   # pandas dependencies
   - beautifulsoup4
   - bottleneck
+  - boto3
   - fsspec>=0.8.0
   - gcsfs
   - html5lib
   - jinja2
   - lxml
   - matplotlib
-  - moto>=1.3.14
-  - flask
   - numba
   - numexpr
   - numpy

ci/deps/actions-39.yaml

@@ -14,14 +14,13 @@ dependencies:
   # pandas dependencies
   - beautifulsoup4
   - bottleneck
+  - boto3
   - fsspec>=0.8.0
   - gcsfs
   - html5lib
   - jinja2
   - lxml
   - matplotlib
-  - moto>=1.3.14
-  - flask
   - numba
   - numexpr
   - numpy

ci/deps/azure-windows-38.yaml

@@ -16,10 +16,8 @@ dependencies:
   - blosc
   - bottleneck
   - fastparquet>=0.4.0
-  - flask
   - fsspec>=0.8.0
   - matplotlib=3.3.2
-  - moto>=1.3.14
   - numba
   - numexpr
   - numpy=1.18

ci/deps/azure-windows-39.yaml

@@ -21,8 +21,6 @@ dependencies:
   - jinja2
   - lxml
   - matplotlib
-  - moto>=1.3.14
-  - flask
   - numba
   - numexpr
   - numpy

pandas/tests/io/conftest.py

@@ -1,11 +1,16 @@
-import logging
 import os
 import shlex
 import subprocess
 import time
 
 import pytest
 
+from pandas.compat import (
+    is_platform_arm,
+    is_platform_mac,
+    is_platform_windows,
+)
+
 import pandas._testing as tm
 
 from pandas.io.parsers import read_csv
@@ -35,60 +40,72 @@ def feather_file(datapath):
 
 
 @pytest.fixture
-def s3so(worker_id):
-    worker_id = "5" if worker_id == "master" else worker_id.lstrip("gw")
-    return {"client_kwargs": {"endpoint_url": f"http://127.0.0.1:555{worker_id}/"}}
+def s3so():
+    return {"client_kwargs": {"endpoint_url": "http://localhost:5000/"}}
 
 
 @pytest.fixture(scope="session")
 def s3_base(worker_id):
     """
     Fixture for mocking S3 interaction.
 
-    Sets up moto server in separate process
+    Sets up moto server in separate process locally
+    Return url for motoserver/moto CI service
     """
     pytest.importorskip("s3fs")
     pytest.importorskip("boto3")
-    requests = pytest.importorskip("requests")
-    logging.getLogger("requests").disabled = True
 
     with tm.ensure_safe_environment_variables():
         # temporary workaround as moto fails for botocore >= 1.11 otherwise,
         # see https://github.com/spulec/moto/issues/1924 & 1952
         os.environ.setdefault("AWS_ACCESS_KEY_ID", "foobar_key")
         os.environ.setdefault("AWS_SECRET_ACCESS_KEY", "foobar_secret")
-
-        pytest.importorskip("moto", minversion="1.3.14")
-        pytest.importorskip("flask")  # server mode needs flask too
-
-        # Launching moto in server mode, i.e., as a separate process
-        # with an S3 endpoint on localhost
-
-        worker_id = "5" if worker_id == "master" else worker_id.lstrip("gw")
-        endpoint_port = f"555{worker_id}"
-        endpoint_uri = f"http://127.0.0.1:{endpoint_port}/"
-
-        # pipe to null to avoid logging in terminal
-        with subprocess.Popen(
-            shlex.split(f"moto_server s3 -p {endpoint_port}"),
-            stdout=subprocess.DEVNULL,
-            stderr=subprocess.DEVNULL,
-        ) as proc:
-
-            timeout = 5
-            while timeout > 0:
-                try:
-                    # OK to go once server is accepting connections
-                    r = requests.get(endpoint_uri)
-                    if r.ok:
-                        break
-                except Exception:
-                    pass
-                timeout -= 0.1
-                time.sleep(0.1)
-            yield endpoint_uri
-
-            proc.terminate()
+        if os.environ.get("PANDAS_CI", "0") == "1":
+            if is_platform_arm() or is_platform_mac() or is_platform_windows():
+                # NOT RUN on Windows/MacOS/ARM, only Ubuntu
+                # - subprocess in CI can cause timeouts
+                # - Azure pipelines/Github Actions do not support
+                #   container services for the above OSs
+                # - CircleCI will probably hit the Docker rate pull limit
+                pytest.skip(
+                    "S3 tests do not have a corresponding service in "
+                    "Windows, MacOS or ARM platforms"
+                )
+            else:
+                yield "http://localhost:5000"
+        else:
+            requests = pytest.importorskip("requests")
+            pytest.importorskip("moto", minversion="1.3.14")
+            pytest.importorskip("flask")  # server mode needs flask too
+
+            # Launching moto in server mode, i.e., as a separate process
+            # with an S3 endpoint on localhost
+
+            worker_id = "5" if worker_id == "master" else worker_id.lstrip("gw")
+            endpoint_port = f"555{worker_id}"
+            endpoint_uri = f"http://127.0.0.1:{endpoint_port}/"
+
+            # pipe to null to avoid logging in terminal
+            with subprocess.Popen(
+                shlex.split(f"moto_server s3 -p {endpoint_port}"),
+                stdout=subprocess.DEVNULL,
+                stderr=subprocess.DEVNULL,
+            ) as proc:
+
+                timeout = 5
+                while timeout > 0:
+                    try:
+                        # OK to go once server is accepting connections
+                        r = requests.get(endpoint_uri)
+                        if r.ok:
+                            break
+                    except Exception:
+                        pass
+                    timeout -= 0.1
+                    time.sleep(0.1)
+                yield endpoint_uri
+
+                proc.terminate()
 
 
 @pytest.fixture()