diff --git a/.travis.yml b/.travis.yml
index 0a5af6d4348c0..b692ec2b26b81 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -31,34 +31,17 @@ matrix:
 
     include:
     - env:
-        - JOB="3.8" ENV_FILE="ci/deps/travis-38.yaml" PATTERN="(not slow and not network)"
-
-    - env:
-        - JOB="3.7" ENV_FILE="ci/deps/travis-37.yaml" PATTERN="(not slow and not network)"
-
-    - env:
-        - JOB="3.6, locale" ENV_FILE="ci/deps/travis-36-locale.yaml" PATTERN="((not slow and not network) or (single and db))" LOCALE_OVERRIDE="zh_CN.UTF-8" SQL="1"
-      services:
-        - mysql
-        - postgresql
-
-    - env:
-        - JOB="3.6, coverage" ENV_FILE="ci/deps/travis-36-cov.yaml" PATTERN="((not slow and not network) or (single and db))" PANDAS_TESTING_MODE="deprecate" COVERAGE=true SQL="1"
+        - JOB="extended" ENV_FILE="ci/deps/extended.yml" PATTERN="(not slow and not network)" SQL="1"
       services:
         - mysql
         - postgresql
 
-    # In allow_failures
     - env:
-        - JOB="3.6, slow" ENV_FILE="ci/deps/travis-36-slow.yaml" PATTERN="slow" SQL="1"
+        - JOB="extended_slow" ENV_FILE="ci/deps/extended.yml" PATTERN="slow" SQL="1"
       services:
         - mysql
         - postgresql
 
-    allow_failures:
-      - env:
-          - JOB="3.6, slow" ENV_FILE="ci/deps/travis-36-slow.yaml" PATTERN="slow" SQL="1"
-
 before_install:
   - echo "before_install"
   # set non-blocking IO on travis
diff --git a/ci/azure/posix.yml b/ci/azure/posix.yml
index cb0b17e3553a4..48dcf27a5d220 100644
--- a/ci/azure/posix.yml
+++ b/ci/azure/posix.yml
@@ -9,48 +9,39 @@ jobs:
   strategy:
     matrix:
       ${{ if eq(parameters.name, 'macOS') }}:
-        py36_macos:
-          ENV_FILE: ci/deps/azure-macos-36.yaml
-          CONDA_PY: "36"
+        latest:
+          ENV_FILE: ci/deps/latest.yml
           PATTERN: "not slow and not network"
 
       ${{ if eq(parameters.name, 'Linux') }}:
-        py36_minimum_versions:
-          ENV_FILE: ci/deps/azure-36-minimum_versions.yaml
-          CONDA_PY: "36"
+        latest:
+          ENV_FILE: ci/deps/latest.yml
           PATTERN: "not slow and not network"
-        py36_locale_slow_old_np:
-          ENV_FILE: ci/deps/azure-36-locale_slow.yaml
-          CONDA_PY: "36"
-          PATTERN: "slow"
           LOCALE_OVERRIDE: "zh_CN.UTF-8"
-          EXTRA_APT: "language-pack-zh-hans"
+          EXTRA_APT: "language-pack-zh-hans xsel"
 
-        py36_locale:
-          ENV_FILE: ci/deps/azure-36-locale.yaml
-          CONDA_PY: "36"
+        oldest:
+          ENV_FILE: ci/deps/oldest.yml
           PATTERN: "not slow and not network"
-          LOCALE_OVERRIDE: "it_IT.UTF-8"
 
-        py36_32bit:
-          ENV_FILE: ci/deps/azure-36-32bit.yaml
-          CONDA_PY: "36"
-          PATTERN: "not slow and not network"
-          BITS32: "yes"
+        slow:
+          ENV_FILE: ci/deps/latest.yml
+          PATTERN: "slow"
 
-        py37_locale:
-          ENV_FILE: ci/deps/azure-37-locale.yaml
-          CONDA_PY: "37"
+        performance:
+          ENV_FILE: ci/deps/performance.yml
           PATTERN: "not slow and not network"
-          LOCALE_OVERRIDE: "zh_CN.UTF-8"
 
-        py37_np_dev:
-          ENV_FILE: ci/deps/azure-37-numpydev.yaml
-          CONDA_PY: "37"
+        32bit:
+          ENV_FILE: ci/deps/32bits.yml
+          PATTERN: "not slow and not network and not clipboard"
+          BITS32: "yes"
+
+        development:
+          ENV_FILE: ci/deps/unstable.yml
           PATTERN: "not slow and not network"
           TEST_ARGS: "-W error"
           PANDAS_TESTING_MODE: "deprecate"
-          EXTRA_APT: "xsel"
 
   steps:
     - script: |
diff --git a/ci/azure/windows.yml b/ci/azure/windows.yml
index 03529bd6569c6..1ef1c5bd9e931 100644
--- a/ci/azure/windows.yml
+++ b/ci/azure/windows.yml
@@ -8,14 +8,8 @@ jobs:
     vmImage: ${{ parameters.vmImage }}
   strategy:
     matrix:
-      py36_np15:
-        ENV_FILE: ci/deps/azure-windows-36.yaml
-        CONDA_PY: "36"
-        PATTERN: "not slow and not network"
-
-      py37_np141:
-        ENV_FILE: ci/deps/azure-windows-37.yaml
-        CONDA_PY: "37"
+      latest:
+        ENV_FILE: ci\\deps\\latest.yml
         PATTERN: "not slow and not network"
 
   steps:
@@ -28,7 +22,7 @@ jobs:
       displayName: 'Update conda'
 
     - bash: |
-        conda env create -q --file ci\\deps\\azure-windows-$(CONDA_PY).yaml
+        conda env create -q --file $(ENV_FILE)
       displayName: 'Create anaconda environment'
 
     - bash: |
@@ -48,7 +42,7 @@ jobs:
       inputs:
         failTaskOnFailedTests: true
         testResultsFiles: 'test-data.xml'
-        testRunTitle: ${{ format('{0}-$(CONDA_PY)', parameters.name) }}
+        testRunTitle: $(build)
       displayName: 'Publish test results'
 
     - bash: |
diff --git a/ci/deps/azure-36-32bit.yaml b/ci/deps/32bits.yml
similarity index 88%
rename from ci/deps/azure-36-32bit.yaml
rename to ci/deps/32bits.yml
index cf3fca307481f..05d9bc7bfe844 100644
--- a/ci/deps/azure-36-32bit.yaml
+++ b/ci/deps/32bits.yml
@@ -1,3 +1,5 @@
+# Testing the core of pandas (no optional dependencies) in 32bits
+
 name: pandas-dev
 channels:
   - defaults
diff --git a/ci/deps/azure-36-locale.yaml b/ci/deps/azure-36-locale.yaml
deleted file mode 100644
index 4f4c4524cb4dd..0000000000000
--- a/ci/deps/azure-36-locale.yaml
+++ /dev/null
@@ -1,39 +0,0 @@
-name: pandas-dev
-channels:
-  - defaults
-  - conda-forge
-dependencies:
-  - python=3.6.*
-
-  # tools
-  - cython>=0.29.13
-  - pytest>=5.0.1
-  - pytest-xdist>=1.21
-  - hypothesis>=3.58.0
-  - pytest-azurepipelines
-
-  # pandas dependencies
-  - beautifulsoup4
-  - gcsfs
-  - html5lib
-  - ipython
-  - jinja2
-  - lxml
-  - matplotlib=3.0.*
-  - nomkl
-  - numexpr
-  - numpy=1.15.*
-  - openpyxl
-  # lowest supported version of pyarrow (putting it here instead of in
-  # azure-36-minimum_versions because it needs numpy >= 1.14)
-  - pyarrow=0.12
-  - pytables
-  - python-dateutil
-  - pytz
-  - s3fs
-  - scipy
-  - xarray
-  - xlrd
-  - xlsxwriter
-  - xlwt
-  - moto
diff --git a/ci/deps/azure-36-locale_slow.yaml b/ci/deps/azure-36-locale_slow.yaml
deleted file mode 100644
index 2bb2b00319382..0000000000000
--- a/ci/deps/azure-36-locale_slow.yaml
+++ /dev/null
@@ -1,32 +0,0 @@
-name: pandas-dev
-channels:
-  - defaults
-  - conda-forge
-dependencies:
-  - python=3.6.*
-
-  # tools
-  - cython>=0.29.13
-  - pytest>=5.0.1
-  - pytest-xdist>=1.21
-  - hypothesis>=3.58.0
-  - pytest-azurepipelines
-
-  # pandas dependencies
-  - beautifulsoup4==4.6.0
-  - bottleneck=1.2.*
-  - lxml
-  - matplotlib=2.2.2
-  - numpy=1.14.*
-  - openpyxl=2.5.7
-  - python-dateutil
-  - python-blosc
-  - pytz=2017.2
-  - scipy
-  - sqlalchemy=1.1.4
-  - xlrd=1.1.0
-  - xlsxwriter=0.9.8
-  - xlwt=1.2.0
-  - pip
-  - pip:
-    - html5lib==1.0b2
diff --git a/ci/deps/azure-37-locale.yaml b/ci/deps/azure-37-locale.yaml
deleted file mode 100644
index a10fa0904a451..0000000000000
--- a/ci/deps/azure-37-locale.yaml
+++ /dev/null
@@ -1,35 +0,0 @@
-name: pandas-dev
-channels:
-  - conda-forge
-dependencies:
-  - python=3.7.*
-
-  # tools
-  - cython>=0.29.13
-  - pytest>=5.0.1
-  - pytest-xdist>=1.21
-  - hypothesis>=3.58.0
-  - pytest-azurepipelines
-
-  # pandas dependencies
-  - beautifulsoup4
-  - html5lib
-  - ipython
-  - jinja2
-  - lxml
-  - matplotlib
-  - moto
-  - nomkl
-  - numexpr
-  - numpy
-  - openpyxl
-  - pytables
-  - python-dateutil
-  - pytz
-  - s3fs
-  - scipy
-  - xarray
-  - xlrd
-  - xlsxwriter
-  - xlwt
-  - pyarrow>=0.15
diff --git a/ci/deps/azure-macos-36.yaml b/ci/deps/azure-macos-36.yaml
deleted file mode 100644
index f393ed84ecf63..0000000000000
--- a/ci/deps/azure-macos-36.yaml
+++ /dev/null
@@ -1,35 +0,0 @@
-name: pandas-dev
-channels:
-  - defaults
-dependencies:
-  - python=3.6.*
-
-  # tools
-  - cython>=0.29.13
-  - pytest>=5.0.1
-  - pytest-xdist>=1.21
-  - hypothesis>=3.58.0
-  - pytest-azurepipelines
-
-  # pandas dependencies
-  - beautifulsoup4
-  - bottleneck
-  - html5lib
-  - jinja2
-  - lxml
-  - matplotlib=2.2.3
-  - nomkl
-  - numexpr
-  - numpy=1.14
-  - openpyxl
-  - pyarrow>=0.12.0
-  - pytables
-  - python-dateutil==2.6.1
-  - pytz
-  - xarray
-  - xlrd
-  - xlsxwriter
-  - xlwt
-  - pip
-  - pip:
-    - pyreadstat
diff --git a/ci/deps/azure-windows-37.yaml b/ci/deps/azure-windows-37.yaml
deleted file mode 100644
index 928896efd5fc4..0000000000000
--- a/ci/deps/azure-windows-37.yaml
+++ /dev/null
@@ -1,36 +0,0 @@
-name: pandas-dev
-channels:
-  - defaults
-  - conda-forge
-dependencies:
-  - python=3.7.*
-
-  # tools
-  - cython>=0.29.13
-  - pytest>=5.0.1
-  - pytest-xdist>=1.21
-  - hypothesis>=3.58.0
-  - pytest-azurepipelines
-
-  # pandas dependencies
-  - beautifulsoup4
-  - bottleneck
-  - gcsfs
-  - html5lib
-  - jinja2
-  - lxml
-  - matplotlib=2.2.*
-  - moto
-  - numexpr
-  - numpy=1.14.*
-  - openpyxl
-  - pytables
-  - python-dateutil
-  - pytz
-  - s3fs
-  - scipy
-  - sqlalchemy
-  - xlrd
-  - xlsxwriter
-  - xlwt
-  - pyreadstat
diff --git a/ci/deps/extended.yml b/ci/deps/extended.yml
new file mode 100644
index 0000000000000..1cad581a0abf8
--- /dev/null
+++ b/ci/deps/extended.yml
@@ -0,0 +1,42 @@
+# Testing downstream packages and tricky IO methods (like network and db)
+
+name: pandas-dev
+channels:
+  - defaults
+  - conda-forge
+dependencies:
+  - python=3.*
+
+  # tools
+  - cython>=0.29.13
+  - pytest>=5.0.1
+  - pytest-xdist>=1.21
+  - hypothesis>=3.58.0
+  - pytest-cov  # this is only needed in the coverage build
+
+  # pandas dependencies
+  - numpy
+  - python-dateutil
+  - pytz
+
+  - dask
+  - geopandas
+  - scikit-learn
+  - xarray
+  - statsmodels
+
+  - sqlalchemy
+  - psycopg2
+  - pymysql
+
+  - pandas-gbq
+  # https://github.com/pydata/pandas-gbq/issues/271
+  - google-cloud-bigquery<=1.11
+  - gcsfs
+
+  - s3fs
+  - botocore>=1.11
+  - moto
+
+  - pip:
+    - pandas-datareader
diff --git a/ci/deps/travis-36-cov.yaml b/ci/deps/latest.yml
similarity index 58%
rename from ci/deps/travis-36-cov.yaml
rename to ci/deps/latest.yml
index c1403f8eb8409..4a239c1384b33 100644
--- a/ci/deps/travis-36-cov.yaml
+++ b/ci/deps/latest.yml
@@ -1,9 +1,11 @@
+# Testing pandas with all optional dependencies, with the latest versions obtained from conda
+
 name: pandas-dev
 channels:
   - defaults
   - conda-forge
 dependencies:
-  - python=3.6.*
+  - python=3.*
 
   # tools
   - cython>=0.29.13
@@ -14,42 +16,25 @@ dependencies:
 
   # pandas dependencies
   - beautifulsoup4
-  - botocore>=1.11
-  - cython>=0.29.13
-  - dask
   - fastparquet>=0.3.2
-  - gcsfs
-  - geopandas
   - html5lib
+  - jinja2
+  - lxml
   - matplotlib
-  - moto
-  - nomkl
-  - numexpr
-  - numpy=1.15.*
+  # - nomkl  # not available in Windows
+  - numpy
   - odfpy
   - openpyxl<=3.0.1
   # https://github.com/pandas-dev/pandas/pull/30009 openpyxl 3.0.2 broke
-  - pandas-gbq
-  # https://github.com/pydata/pandas-gbq/issues/271
-  - google-cloud-bigquery<=1.11
-  - psycopg2
   - pyarrow>=0.12.0
-  - pymysql
   - pytables
   - python-snappy
+  - python-dateutil
   - pytz
-  - s3fs
-  - scikit-learn
   - scipy
-  - sqlalchemy
-  - statsmodels
-  - xarray
   - xlrd
   - xlsxwriter
   - xlwt
   - pip
   - pip:
-    - brotlipy
     - coverage
-    - pandas-datareader
-    - python-dateutil
diff --git a/ci/deps/azure-36-minimum_versions.yaml b/ci/deps/oldest.yml
similarity index 77%
rename from ci/deps/azure-36-minimum_versions.yaml
rename to ci/deps/oldest.yml
index de7e011d9c7ca..f01075ba7e9a9 100644
--- a/ci/deps/azure-36-minimum_versions.yaml
+++ b/ci/deps/oldest.yml
@@ -1,3 +1,7 @@
+# Testing pandas with the minimum supported versions of dependencies
+#
+# TODO: The included optional dependencies seem arbitrary, should we add all them?
+
 name: pandas-dev
 channels:
   - defaults
diff --git a/ci/deps/azure-windows-36.yaml b/ci/deps/performance.yml
similarity index 51%
rename from ci/deps/azure-windows-36.yaml
rename to ci/deps/performance.yml
index 7fa9dee7445a6..455a357b42442 100644
--- a/ci/deps/azure-windows-36.yaml
+++ b/ci/deps/performance.yml
@@ -1,32 +1,25 @@
+# Testing pandas with libraries to optimize code
+
 name: pandas-dev
 channels:
-  - conda-forge
   - defaults
+  - conda-forge
 dependencies:
-  - python=3.6.*
+  - python=3.*
 
   # tools
   - cython>=0.29.13
   - pytest>=5.0.1
   - pytest-xdist>=1.21
   - hypothesis>=3.58.0
-  - pytest-azurepipelines
+  - pytest-cov  # this is only needed in the coverage build
 
   # pandas dependencies
-  - blosc
-  - bottleneck
-  - fastparquet>=0.3.2
-  - matplotlib=3.0.2
-  - numba
-  - numexpr
-  - numpy=1.15.*
-  - openpyxl
-  - jinja2
-  - pyarrow>=0.12.0
-  - pytables
+  - numpy
   - python-dateutil
   - pytz
-  - scipy
-  - xlrd
-  - xlsxwriter
-  - xlwt
+  # mkl will be used, since `nomkl` is not being installed
+  - blosc=1.14.3
+  - bottleneck
+  - numexpr
+  - python-blosc
diff --git a/ci/deps/travis-36-locale.yaml b/ci/deps/travis-36-locale.yaml
deleted file mode 100644
index d0bc046575953..0000000000000
--- a/ci/deps/travis-36-locale.yaml
+++ /dev/null
@@ -1,42 +0,0 @@
-name: pandas-dev
-channels:
-  - defaults
-  - conda-forge
-dependencies:
-  - python=3.6.*
-
-  # tools
-  - cython>=0.29.13
-  - pytest>=5.0.1
-  - pytest-xdist>=1.21
-  - hypothesis>=3.58.0
-
-  # pandas dependencies
-  - beautifulsoup4
-  - blosc=1.14.3
-  - python-blosc
-  - fastparquet=0.3.2
-  - gcsfs=0.2.2
-  - html5lib
-  - ipython
-  - jinja2
-  - lxml=3.8.0
-  - matplotlib=3.0.*
-  - moto
-  - nomkl
-  - numexpr
-  - numpy
-  - openpyxl
-  - pandas-gbq=0.8.0
-  - psycopg2=2.6.2
-  - pymysql=0.7.11
-  - pytables
-  - python-dateutil
-  - pytz
-  - s3fs=0.3.0
-  - scipy
-  - sqlalchemy=1.1.4
-  - xarray=0.10
-  - xlrd
-  - xlsxwriter
-  - xlwt
diff --git a/ci/deps/travis-36-slow.yaml b/ci/deps/travis-36-slow.yaml
deleted file mode 100644
index 1dfd90d0904ac..0000000000000
--- a/ci/deps/travis-36-slow.yaml
+++ /dev/null
@@ -1,34 +0,0 @@
-name: pandas-dev
-channels:
-  - defaults
-  - conda-forge
-dependencies:
-  - python=3.6.*
-
-  # tools
-  - cython>=0.29.13
-  - pytest>=5.0.1
-  - pytest-xdist>=1.21
-  - hypothesis>=3.58.0
-
-  # pandas dependencies
-  - beautifulsoup4
-  - html5lib
-  - lxml
-  - matplotlib
-  - numexpr
-  - numpy
-  - openpyxl
-  - patsy
-  - psycopg2
-  - pymysql
-  - pytables
-  - python-dateutil
-  - pytz
-  - s3fs
-  - scipy
-  - sqlalchemy
-  - xlrd
-  - xlsxwriter
-  - xlwt
-  - moto
diff --git a/ci/deps/travis-37.yaml b/ci/deps/travis-37.yaml
deleted file mode 100644
index 73e2c20b31438..0000000000000
--- a/ci/deps/travis-37.yaml
+++ /dev/null
@@ -1,27 +0,0 @@
-name: pandas-dev
-channels:
-  - defaults
-  - conda-forge
-  - c3i_test
-dependencies:
-  - python=3.7.*
-
-  # tools
-  - cython>=0.29.13
-  - pytest>=5.0.1
-  - pytest-xdist>=1.21
-  - hypothesis>=3.58.0
-
-  # pandas dependencies
-  - botocore>=1.11
-  - numpy
-  - python-dateutil
-  - nomkl
-  - pyarrow
-  - pytz
-  - s3fs
-  - tabulate
-  - pyreadstat
-  - pip
-  - pip:
-    - moto
diff --git a/ci/deps/travis-38.yaml b/ci/deps/travis-38.yaml
deleted file mode 100644
index a627b7edc175f..0000000000000
--- a/ci/deps/travis-38.yaml
+++ /dev/null
@@ -1,20 +0,0 @@
-name: pandas-dev
-channels:
-  - defaults
-  - conda-forge
-dependencies:
-  - python=3.8.*
-
-  # tools
-  - cython>=0.29.13
-  - pytest>=5.0.1
-  - pytest-xdist>=1.21
-  - hypothesis>=3.58.0
-
-  # pandas dependencies
-  - numpy
-  - python-dateutil
-  - nomkl
-  - pytz
-  - pip
-  - tabulate==0.8.3
diff --git a/ci/deps/azure-37-numpydev.yaml b/ci/deps/unstable.yml
similarity index 76%
rename from ci/deps/azure-37-numpydev.yaml
rename to ci/deps/unstable.yml
index a04bdc2448bce..71088534b47e1 100644
--- a/ci/deps/azure-37-numpydev.yaml
+++ b/ci/deps/unstable.yml
@@ -1,8 +1,10 @@
+# Testing the core of pandas (no optional dependencies) with the development versions of NumPy and dateutil
+
 name: pandas-dev
 channels:
   - defaults
 dependencies:
-  - python=3.7.*
+  - python=3.*
 
   # tools
   - cython>=0.29.13