diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index a37ea3eb89167..d05caa7d881f0 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -45,7 +45,7 @@ jobs:
         /opt/python/cp38-cp38/bin/python -m venv ~/virtualenvs/pandas-dev && \
         . ~/virtualenvs/pandas-dev/bin/activate && \
         python -m pip install --no-deps -U pip wheel 'setuptools<60.0.0' && \
-        pip install cython numpy python-dateutil pytz pytest pytest-xdist hypothesis pytest-azurepipelines && \
+        pip install cython numpy python-dateutil pytz pytest pytest-xdist hypothesis && \
         python setup.py build_ext -q -j2 && \
         python -m pip install --no-build-isolation -e . && \
         pytest -m 'not slow and not network and not clipboard' pandas --junitxml=test-data.xml"
diff --git a/pandas/tests/extension/arrow/test_bool.py b/pandas/tests/extension/arrow/test_bool.py
index a73684868e3ae..7f4766bac0329 100644
--- a/pandas/tests/extension/arrow/test_bool.py
+++ b/pandas/tests/extension/arrow/test_bool.py
@@ -1,6 +1,11 @@
 import numpy as np
 import pytest
 
+from pandas.compat import (
+    is_ci_environment,
+    is_platform_windows,
+)
+
 import pandas as pd
 import pandas._testing as tm
 from pandas.api.types import is_bool_dtype
@@ -91,6 +96,10 @@ def test_reduce_series_boolean(self):
         pass
 
 
+@pytest.mark.skipif(
+    is_ci_environment() and is_platform_windows(),
+    reason="Causes stack overflow on Windows CI",
+)
 class TestReduceBoolean(base.BaseBooleanReduceTests):
     pass
 
diff --git a/pandas/tests/extension/json/test_json.py b/pandas/tests/extension/json/test_json.py
index d530a75b74c8f..f13ef65267304 100644
--- a/pandas/tests/extension/json/test_json.py
+++ b/pandas/tests/extension/json/test_json.py
@@ -1,5 +1,6 @@
 import collections
 import operator
+import sys
 
 import pytest
 
@@ -163,12 +164,24 @@ def test_from_dtype(self, data):
     @pytest.mark.xfail(reason="RecursionError, GH-33900")
     def test_series_constructor_no_data_with_index(self, dtype, na_value):
         # RecursionError: maximum recursion depth exceeded in comparison
-        super().test_series_constructor_no_data_with_index(dtype, na_value)
+        rec_limit = sys.getrecursionlimit()
+        try:
+            # Limit to avoid stack overflow on Windows CI
+            sys.setrecursionlimit(100)
+            super().test_series_constructor_no_data_with_index(dtype, na_value)
+        finally:
+            sys.setrecursionlimit(rec_limit)
 
     @pytest.mark.xfail(reason="RecursionError, GH-33900")
     def test_series_constructor_scalar_na_with_index(self, dtype, na_value):
         # RecursionError: maximum recursion depth exceeded in comparison
-        super().test_series_constructor_scalar_na_with_index(dtype, na_value)
+        rec_limit = sys.getrecursionlimit()
+        try:
+            # Limit to avoid stack overflow on Windows CI
+            sys.setrecursionlimit(100)
+            super().test_series_constructor_scalar_na_with_index(dtype, na_value)
+        finally:
+            sys.setrecursionlimit(rec_limit)
 
     @pytest.mark.xfail(reason="collection as scalar, GH-33901")
     def test_series_constructor_scalar_with_index(self, data, dtype):