diff --git a/.ci/docker/common/install_conda.sh b/.ci/docker/common/install_conda.sh
index 2d6dd4cf2f4..5dd5582267d 100755
--- a/.ci/docker/common/install_conda.sh
+++ b/.ci/docker/common/install_conda.sh
@@ -27,7 +27,7 @@ install_miniconda() {
 install_python() {
   pushd /opt/conda
   # Install the correct Python version
-  as_ci_user conda create -n "py_${PYTHON_VERSION}" -y python="${PYTHON_VERSION}"
+  as_ci_user conda create -n "py_${PYTHON_VERSION}" -y --file /opt/conda/conda-env-ci.txt python="${PYTHON_VERSION}"
   popd
 }
 
diff --git a/.ci/docker/conda-env-ci.txt b/.ci/docker/conda-env-ci.txt
new file mode 100644
index 00000000000..467bac5e422
--- /dev/null
+++ b/.ci/docker/conda-env-ci.txt
@@ -0,0 +1,3 @@
+# TODO: We might need to update this to install flatbuffers from the pinned commit
+# in fbcode/executorch/third-party/flatbuffers.submodule.txt
+flatbuffers=2.0.0
diff --git a/.ci/docker/requirements-ci.txt b/.ci/docker/requirements-ci.txt
index 9b35a5621b9..23479ed2bd4 100644
--- a/.ci/docker/requirements-ci.txt
+++ b/.ci/docker/requirements-ci.txt
@@ -1,4 +1,3 @@
-flatbuffers==2.0
 mpmath==1.3.0
 PyYAML==6.0.1
 ruamel.yaml==0.17.32
diff --git a/.ci/docker/ubuntu/Dockerfile b/.ci/docker/ubuntu/Dockerfile
index 117c4fd1a15..bfd4b9aa3f7 100644
--- a/.ci/docker/ubuntu/Dockerfile
+++ b/.ci/docker/ubuntu/Dockerfile
@@ -29,9 +29,10 @@ ARG PYTHON_VERSION
 ENV PYTHON_VERSION=$PYTHON_VERSION
 ENV PATH /opt/conda/envs/py_$PYTHON_VERSION/bin:/opt/conda/bin:$PATH
 COPY requirements-ci.txt /opt/conda/
+COPY conda-env-ci.txt /opt/conda/
 COPY ./common/install_conda.sh install_conda.sh
 COPY ./common/utils.sh utils.sh
-RUN bash ./install_conda.sh && rm install_conda.sh utils.sh /opt/conda/requirements-ci.txt
+RUN bash ./install_conda.sh && rm install_conda.sh utils.sh /opt/conda/requirements-ci.txt /opt/conda/conda-env-ci.txt
 
 USER ci-user
 CMD ["bash"]