Skip to content

Commit 6cbf9b6

Browse files
potiukpotiuk
authored
Allow to specify pip version via github URL in Docker/CI workflow (#28697)
We've only supported to install `pip` from released packages with a version number, but since `pip` does not support RC candidates (as extensively discussed in pypa/pip#10882) we cannot use the release versions to do that. We still want to help `pip` maintainers and be able to test the versions they release as early as possible, so we add support to install `pip` in our toolchain from a GitHub URL. That will allow us to test new `pip` version as soon as designated branch of `pip` will contain something resembling a release candidate ready for testing.
1 parent e8533d2 commit 6cbf9b6

10 files changed

+71
-75
lines changed

‎Dockerfile

Lines changed: 22 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -325,20 +325,12 @@ COPY <<"EOF" /install_pip_version.sh
325325

326326
: "${AIRFLOW_PIP_VERSION:?Should be set}"
327327

328-
function install_pip_version() {
329-
echo
330-
echo "${COLOR_BLUE}Installing pip version ${AIRFLOW_PIP_VERSION}${COLOR_RESET}"
331-
echo
332-
pip install --disable-pip-version-check --no-cache-dir --upgrade "pip==${AIRFLOW_PIP_VERSION}" &&
333-
mkdir -p ${HOME}/.local/bin
334-
}
335-
336328
common::get_colors
337329
common::get_airflow_version_specification
338330
common::override_pip_version_if_needed
339331
common::show_pip_version_and_location
340332

341-
install_pip_version
333+
common::install_pip_version
342334
EOF
343335

344336
# The content below is automatically copied from scripts/docker/install_airflow_dependencies_from_branch_tip.sh
@@ -369,8 +361,7 @@ function install_airflow_dependencies_from_branch_tip() {
369361
${ADDITIONAL_PIP_INSTALL_FLAGS} \
370362
"https://github.com/${AIRFLOW_REPO}/archive/${AIRFLOW_BRANCH}.tar.gz#egg=apache-airflow[${AIRFLOW_EXTRAS}]" \
371363
--constraint "${AIRFLOW_CONSTRAINTS_LOCATION}" || true
372-
# make sure correct PIP version is used
373-
pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" 2>/dev/null
364+
common::install_pip_version
374365
pip freeze | grep apache-airflow-providers | xargs pip uninstall --yes 2>/dev/null || true
375366
set +x
376367
echo
@@ -445,6 +436,18 @@ function common::show_pip_version_and_location() {
445436
echo "pip on path: $(which pip)"
446437
echo "Using pip: $(pip --version)"
447438
}
439+
440+
function common::install_pip_version() {
441+
echo
442+
echo "${COLOR_BLUE}Installing pip version ${AIRFLOW_PIP_VERSION}${COLOR_RESET}"
443+
echo
444+
if [[ ${AIRFLOW_PIP_VERSION} =~ .*https.* ]]; then
445+
pip install --disable-pip-version-check --no-cache-dir "pip @ ${AIRFLOW_PIP_VERSION}"
446+
else
447+
pip install --disable-pip-version-check --no-cache-dir "pip==${AIRFLOW_PIP_VERSION}"
448+
fi
449+
mkdir -p "${HOME}/.local/bin"
450+
}
448451
EOF
449452

450453
# The content below is automatically copied from scripts/docker/pip
@@ -531,8 +534,7 @@ function install_airflow_and_providers_from_docker_context_files(){
531534
${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS}
532535
set +x
533536

534-
# make sure correct PIP version is left installed
535-
pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" 2>/dev/null
537+
common::install_pip_version
536538
pip check
537539
}
538540

@@ -549,9 +551,8 @@ function install_all_other_packages_from_docker_context_files() {
549551
set -x
550552
pip install ${ADDITIONAL_PIP_INSTALL_FLAGS} \
551553
--root-user-action ignore --force-reinstall --no-deps --no-index ${reinstalling_other_packages}
552-
# make sure correct PIP version is used
553-
pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" 2>/dev/null
554-
set -x
554+
common::install_pip_version
555+
set +x
555556
fi
556557
}
557558

@@ -613,8 +614,7 @@ function install_airflow() {
613614
set +x
614615
fi
615616

616-
# make sure correct PIP version is used
617-
pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" 2>/dev/null
617+
common::install_pip_version
618618
echo
619619
echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}"
620620
echo
@@ -628,15 +628,13 @@ function install_airflow() {
628628
${ADDITIONAL_PIP_INSTALL_FLAGS} \
629629
"${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}" \
630630
--constraint "${AIRFLOW_CONSTRAINTS_LOCATION}"
631-
# make sure correct PIP version is used
632-
pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" 2>/dev/null
631+
common::install_pip_version
633632
# then upgrade if needed without using constraints to account for new limits in setup.py
634633
pip install --root-user-action ignore --upgrade --upgrade-strategy only-if-needed \
635634
${ADDITIONAL_PIP_INSTALL_FLAGS} \
636635
${AIRFLOW_INSTALL_EDITABLE_FLAG} \
637636
"${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}"
638-
# make sure correct PIP version is used
639-
pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" 2>/dev/null
637+
common::install_pip_version
640638
set +x
641639
echo
642640
echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}"
@@ -675,8 +673,7 @@ function install_additional_dependencies() {
675673
pip install --root-user-action ignore --upgrade --upgrade-strategy eager \
676674
${ADDITIONAL_PIP_INSTALL_FLAGS} \
677675
${ADDITIONAL_PYTHON_DEPS} ${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS}
678-
# make sure correct PIP version is used
679-
pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" 2>/dev/null
676+
common::install_pip_version
680677
set +x
681678
echo
682679
echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}"
@@ -690,8 +687,7 @@ function install_additional_dependencies() {
690687
pip install --root-user-action ignore --upgrade --upgrade-strategy only-if-needed \
691688
${ADDITIONAL_PIP_INSTALL_FLAGS} \
692689
${ADDITIONAL_PYTHON_DEPS}
693-
# make sure correct PIP version is used
694-
pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" 2>/dev/null
690+
common::install_pip_version
695691
set +x
696692
echo
697693
echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}"

‎Dockerfile.ci

Lines changed: 19 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -285,20 +285,12 @@ COPY <<"EOF" /install_pip_version.sh
285285

286286
: "${AIRFLOW_PIP_VERSION:?Should be set}"
287287

288-
function install_pip_version() {
289-
echo
290-
echo "${COLOR_BLUE}Installing pip version ${AIRFLOW_PIP_VERSION}${COLOR_RESET}"
291-
echo
292-
pip install --disable-pip-version-check --no-cache-dir --upgrade "pip==${AIRFLOW_PIP_VERSION}" &&
293-
mkdir -p ${HOME}/.local/bin
294-
}
295-
296288
common::get_colors
297289
common::get_airflow_version_specification
298290
common::override_pip_version_if_needed
299291
common::show_pip_version_and_location
300292

301-
install_pip_version
293+
common::install_pip_version
302294
EOF
303295

304296
# The content below is automatically copied from scripts/docker/install_airflow_dependencies_from_branch_tip.sh
@@ -329,8 +321,7 @@ function install_airflow_dependencies_from_branch_tip() {
329321
${ADDITIONAL_PIP_INSTALL_FLAGS} \
330322
"https://github.com/${AIRFLOW_REPO}/archive/${AIRFLOW_BRANCH}.tar.gz#egg=apache-airflow[${AIRFLOW_EXTRAS}]" \
331323
--constraint "${AIRFLOW_CONSTRAINTS_LOCATION}" || true
332-
# make sure correct PIP version is used
333-
pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" 2>/dev/null
324+
common::install_pip_version
334325
pip freeze | grep apache-airflow-providers | xargs pip uninstall --yes 2>/dev/null || true
335326
set +x
336327
echo
@@ -405,6 +396,18 @@ function common::show_pip_version_and_location() {
405396
echo "pip on path: $(which pip)"
406397
echo "Using pip: $(pip --version)"
407398
}
399+
400+
function common::install_pip_version() {
401+
echo
402+
echo "${COLOR_BLUE}Installing pip version ${AIRFLOW_PIP_VERSION}${COLOR_RESET}"
403+
echo
404+
if [[ ${AIRFLOW_PIP_VERSION} =~ .*https.* ]]; then
405+
pip install --disable-pip-version-check --no-cache-dir "pip @ ${AIRFLOW_PIP_VERSION}"
406+
else
407+
pip install --disable-pip-version-check --no-cache-dir "pip==${AIRFLOW_PIP_VERSION}"
408+
fi
409+
mkdir -p "${HOME}/.local/bin"
410+
}
408411
EOF
409412

410413
# The content below is automatically copied from scripts/docker/install_pipx_tools.sh
@@ -480,8 +483,7 @@ function install_airflow() {
480483
set +x
481484
fi
482485

483-
# make sure correct PIP version is used
484-
pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" 2>/dev/null
486+
common::install_pip_version
485487
echo
486488
echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}"
487489
echo
@@ -495,15 +497,13 @@ function install_airflow() {
495497
${ADDITIONAL_PIP_INSTALL_FLAGS} \
496498
"${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}" \
497499
--constraint "${AIRFLOW_CONSTRAINTS_LOCATION}"
498-
# make sure correct PIP version is used
499-
pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" 2>/dev/null
500+
common::install_pip_version
500501
# then upgrade if needed without using constraints to account for new limits in setup.py
501502
pip install --root-user-action ignore --upgrade --upgrade-strategy only-if-needed \
502503
${ADDITIONAL_PIP_INSTALL_FLAGS} \
503504
${AIRFLOW_INSTALL_EDITABLE_FLAG} \
504505
"${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}"
505-
# make sure correct PIP version is used
506-
pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" 2>/dev/null
506+
common::install_pip_version
507507
set +x
508508
echo
509509
echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}"
@@ -542,8 +542,7 @@ function install_additional_dependencies() {
542542
pip install --root-user-action ignore --upgrade --upgrade-strategy eager \
543543
${ADDITIONAL_PIP_INSTALL_FLAGS} \
544544
${ADDITIONAL_PYTHON_DEPS} ${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS}
545-
# make sure correct PIP version is used
546-
pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" 2>/dev/null
545+
common::install_pip_version
547546
set +x
548547
echo
549548
echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}"
@@ -557,8 +556,7 @@ function install_additional_dependencies() {
557556
pip install --root-user-action ignore --upgrade --upgrade-strategy only-if-needed \
558557
${ADDITIONAL_PIP_INSTALL_FLAGS} \
559558
${ADDITIONAL_PYTHON_DEPS}
560-
# make sure correct PIP version is used
561-
pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" 2>/dev/null
559+
common::install_pip_version
562560
set +x
563561
echo
564562
echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}"

‎scripts/docker/common.sh

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,3 +70,15 @@ function common::show_pip_version_and_location() {
7070
echo "pip on path: $(which pip)"
7171
echo "Using pip: $(pip --version)"
7272
}
73+
74+
function common::install_pip_version() {
75+
echo
76+
echo "${COLOR_BLUE}Installing pip version ${AIRFLOW_PIP_VERSION}${COLOR_RESET}"
77+
echo
78+
if [[ ${AIRFLOW_PIP_VERSION} =~ .*https.* ]]; then
79+
pip install --disable-pip-version-check --no-cache-dir "pip @ ${AIRFLOW_PIP_VERSION}"
80+
else
81+
pip install --disable-pip-version-check --no-cache-dir "pip==${AIRFLOW_PIP_VERSION}"
82+
fi
83+
mkdir -p "${HOME}/.local/bin"
84+
}

‎scripts/docker/install_additional_dependencies.sh

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -35,8 +35,7 @@ function install_additional_dependencies() {
3535
pip install --root-user-action ignore --upgrade --upgrade-strategy eager \
3636
${ADDITIONAL_PIP_INSTALL_FLAGS} \
3737
${ADDITIONAL_PYTHON_DEPS} ${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS}
38-
# make sure correct PIP version is used
39-
pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" 2>/dev/null
38+
common::install_pip_version
4039
set +x
4140
echo
4241
echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}"
@@ -50,8 +49,7 @@ function install_additional_dependencies() {
5049
pip install --root-user-action ignore --upgrade --upgrade-strategy only-if-needed \
5150
${ADDITIONAL_PIP_INSTALL_FLAGS} \
5251
${ADDITIONAL_PYTHON_DEPS}
53-
# make sure correct PIP version is used
54-
pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" 2>/dev/null
52+
common::install_pip_version
5553
set +x
5654
echo
5755
echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}"

‎scripts/docker/install_airflow.sh

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -69,8 +69,7 @@ function install_airflow() {
6969
set +x
7070
fi
7171

72-
# make sure correct PIP version is used
73-
pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" 2>/dev/null
72+
common::install_pip_version
7473
echo
7574
echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}"
7675
echo
@@ -84,15 +83,13 @@ function install_airflow() {
8483
${ADDITIONAL_PIP_INSTALL_FLAGS} \
8584
"${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}" \
8685
--constraint "${AIRFLOW_CONSTRAINTS_LOCATION}"
87-
# make sure correct PIP version is used
88-
pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" 2>/dev/null
86+
common::install_pip_version
8987
# then upgrade if needed without using constraints to account for new limits in setup.py
9088
pip install --root-user-action ignore --upgrade --upgrade-strategy only-if-needed \
9189
${ADDITIONAL_PIP_INSTALL_FLAGS} \
9290
${AIRFLOW_INSTALL_EDITABLE_FLAG} \
9391
"${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}"
94-
# make sure correct PIP version is used
95-
pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" 2>/dev/null
92+
common::install_pip_version
9693
set +x
9794
echo
9895
echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}"

‎scripts/docker/install_airflow_dependencies_from_branch_tip.sh

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -52,8 +52,7 @@ function install_airflow_dependencies_from_branch_tip() {
5252
${ADDITIONAL_PIP_INSTALL_FLAGS} \
5353
"https://github.com/${AIRFLOW_REPO}/archive/${AIRFLOW_BRANCH}.tar.gz#egg=apache-airflow[${AIRFLOW_EXTRAS}]" \
5454
--constraint "${AIRFLOW_CONSTRAINTS_LOCATION}" || true
55-
# make sure correct PIP version is used
56-
pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" 2>/dev/null
55+
common::install_pip_version
5756
pip freeze | grep apache-airflow-providers | xargs pip uninstall --yes 2>/dev/null || true
5857
set +x
5958
echo

‎scripts/docker/install_from_docker_context_files.sh

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -85,8 +85,7 @@ function install_airflow_and_providers_from_docker_context_files(){
8585
${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS}
8686
set +x
8787

88-
# make sure correct PIP version is left installed
89-
pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" 2>/dev/null
88+
common::install_pip_version
9089
pip check
9190
}
9291

@@ -107,9 +106,8 @@ function install_all_other_packages_from_docker_context_files() {
107106
set -x
108107
pip install ${ADDITIONAL_PIP_INSTALL_FLAGS} \
109108
--root-user-action ignore --force-reinstall --no-deps --no-index ${reinstalling_other_packages}
110-
# make sure correct PIP version is used
111-
pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" 2>/dev/null
112-
set -x
109+
common::install_pip_version
110+
set +x
113111
fi
114112
}
115113

‎scripts/docker/install_pip_version.sh

Lines changed: 1 addition & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -20,17 +20,9 @@
2020

2121
: "${AIRFLOW_PIP_VERSION:?Should be set}"
2222

23-
function install_pip_version() {
24-
echo
25-
echo "${COLOR_BLUE}Installing pip version ${AIRFLOW_PIP_VERSION}${COLOR_RESET}"
26-
echo
27-
pip install --disable-pip-version-check --no-cache-dir --upgrade "pip==${AIRFLOW_PIP_VERSION}" &&
28-
mkdir -p ${HOME}/.local/bin
29-
}
30-
3123
common::get_colors
3224
common::get_airflow_version_specification
3325
common::override_pip_version_if_needed
3426
common::show_pip_version_and_location
3527

36-
install_pip_version
28+
common::install_pip_version

‎scripts/in_container/_in_container_utils.sh

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -353,7 +353,12 @@ function setup_provider_packages() {
353353

354354

355355
function install_supported_pip_version() {
356-
pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}"
356+
if [[ ${AIRFLOW_PIP_VERSION} =~ .*https.* ]]; then
357+
pip install --disable-pip-version-check "pip @ ${AIRFLOW_PIP_VERSION}"
358+
else
359+
pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}"
360+
fi
361+
357362
}
358363

359364
function filename_to_python_module() {

‎scripts/in_container/run_prepare_airflow_packages.sh

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,8 @@ function prepare_airflow_packages() {
3434
rm -rf -- *egg-info*
3535
rm -rf -- build
3636

37-
pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" "wheel==${WHEEL_VERSION}"
37+
install_supported_pip_version
38+
pip install "wheel==${WHEEL_VERSION}"
3839

3940
local packages=()
4041

0 commit comments

Comments
 (0)