Skip to content
This repository was archived by the owner on Aug 15, 2025. It is now read-only.

Commit adcb783

Browse files
soumithsoumith
authored
Merge pull request #60 from pjh5/ldd_nm
ldd for manywheel/
2 parents 19325a0 + fce77ff commit adcb783

File tree

2 files changed

+110
-87
lines changed

2 files changed

+110
-87
lines changed

‎manywheel/build.sh‎

Lines changed: 56 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,10 @@
33
set -ex
44

55
if [[ -z "$PYTORCH_BUILD_VERSION" ]]; then
6-
export PYTORCH_BUILD_VERSION=0.4.1
6+
export PYTORCH_BUILD_VERSION=0.4.1
77
fi
88
if [[ -z "$PYTORCH_BUILD_NUMBER" ]]; then
9-
export PYTORCH_BUILD_NUMBER=2
9+
export PYTORCH_BUILD_NUMBER=2
1010
fi
1111
export CMAKE_LIBRARY_PATH="/opt/intel/lib:/lib:$CMAKE_LIBRARY_PATH"
1212
export CMAKE_INCLUDE_PATH="/opt/intel:$CMAKE_INCLUDE_PATH"
@@ -46,22 +46,21 @@ fi
4646

4747
# Build for given Python versions, or for all in /opt/python if none given
4848
if [[ -z "$DESIRED_PYTHON" ]]; then
49-
DESIRED_PYTHON=($(ls -d /opt/python/*/))
49+
pushd /opt/python
50+
DESIRED_PYTHON=(*/)
51+
popd
5052
fi
51-
for (( i=0; i<"${#DESIRED_PYTHON[@]}"; i++ )); do
52-
# Convert eg. cp27-cp27m to /opt/python/cp27-cp27m
53-
if [[ ! -d "${DESIRED_PYTHON[$i]}" ]]; then
54-
if [[ -d "/opt/python/${DESIRED_PYTHON[$i]}" ]]; then
55-
DESIRED_PYTHON[$i]="/opt/python/${DESIRED_PYTHON[$i]}"
56-
else
57-
echo "Error: Given Python ${DESIRED_PYTHON[$i]} is not in /opt/python"
58-
echo "All array elements of env variable DESIRED_PYTHON must be"
59-
echo "valid Python installations under /opt/python"
60-
exit 1
53+
python_installations=()
54+
for desired_py in "${DESIRED_PYTHON[@]}"; do
55+
python_installations+=("/opt/python/$desired_py")
56+
if [[ ! -d "/opt/python/$desired_py" ]]; then
57+
echo "Error: Given Python $desired_py is not in /opt/python"
58+
echo "All array elements of env variable DESIRED_PYTHON must be"
59+
echo "valid Python installations under /opt/python"
60+
exit 1
6161
fi
62-
fi
6362
done
64-
echo "Will build for all Pythons: ${DESIRED_PYTHON[@]}"
63+
echo "Will build for all Pythons versions: ${DESIRED_PYTHON[@]}"
6564

6665

6766
# ########################################################
@@ -70,26 +69,26 @@ echo "Will build for all Pythons: ${DESIRED_PYTHON[@]}"
7069
# clone pytorch source code
7170
PYTORCH_DIR="/pytorch"
7271
if [[ ! -d "$PYTORCH_DIR" ]]; then
73-
git clone https://github.com/pytorch/pytorch $PYTORCH_DIR
74-
pushd $PYTORCH_DIR
75-
if ! git checkout v${PYTORCH_BUILD_VERSION}; then
76-
git checkout tags/v${PYTORCH_BUILD_VERSION}
77-
fi
72+
git clone https://github.com/pytorch/pytorch $PYTORCH_DIR
73+
pushd $PYTORCH_DIR
74+
if ! git checkout v${PYTORCH_BUILD_VERSION}; then
75+
git checkout tags/v${PYTORCH_BUILD_VERSION}
76+
fi
7877
else
79-
# the pytorch dir will already be cloned and checked-out on jenkins jobs
80-
pushd $PYTORCH_DIR
78+
# the pytorch dir will already be cloned and checked-out on jenkins jobs
79+
pushd $PYTORCH_DIR
8180
fi
8281
git submodule update --init --recursive
8382

8483
OLD_PATH=$PATH
85-
for PYDIR in "${DESIRED_PYTHON[@]}"; do
84+
for PYDIR in "${python_installations[@]}"; do
8685
export PATH=$PYDIR/bin:$OLD_PATH
8786
python setup.py clean
8887
pip install -r requirements.txt
8988
if [[ $PYDIR == "/opt/python/cp37-cp37m" ]]; then
90-
pip install numpy==1.15
89+
pip install numpy==1.15
9190
else
92-
pip install numpy==1.11
91+
pip install numpy==1.11
9392
fi
9493
time python setup.py bdist_wheel -d $WHEELHOUSE_DIR
9594
done
@@ -110,11 +109,11 @@ fname_with_sha256() {
110109
DIRNAME=$(dirname $1)
111110
BASENAME=$(basename $1)
112111
if [[ $BASENAME == "libnvrtc-builtins.so" ]]; then
113-
echo $1
112+
echo $1
114113
else
115-
INITNAME=$(echo $BASENAME | cut -f1 -d".")
116-
ENDNAME=$(echo $BASENAME | cut -f 2- -d".")
117-
echo "$DIRNAME/$INITNAME-$HASH.$ENDNAME"
114+
INITNAME=$(echo $BASENAME | cut -f1 -d".")
115+
ENDNAME=$(echo $BASENAME | cut -f 2- -d".")
116+
echo "$DIRNAME/$INITNAME-$HASH.$ENDNAME"
118117
fi
119118
}
120119

@@ -274,9 +273,13 @@ for whl in /$WHEELHOUSE_DIR/torch*linux*.whl; do
274273
rm -rf tmp
275274
done
276275

277-
# Print out sizes of all wheels created
278-
echo "Succesfulle made wheels of size:"
279-
du -h /$WHEELHOUSE_DIR/torch*.whl
276+
# Take the actual package name. Note how this always works because pip converts
277+
# - to _ in names.
278+
pushd /$WHEELHOUSE_DIR
279+
built_wheels=(torch*.whl)
280+
IFS='-' read -r package_name some_unused_variable <<< "${built_wheels[0]}"
281+
echo "Expecting the built wheels to all be called '$package_name'"
282+
popd
280283

281284
# Copy wheels to host machine for persistence after the docker
282285
mkdir -p /remote/$WHEELHOUSE_DIR
@@ -286,22 +289,31 @@ cp /$WHEELHOUSE_DIR/torch*.whl /remote/$WHEELHOUSE_DIR/
286289
rm -rf /usr/local/cuda*
287290
rm -rf /opt/rh
288291

289-
# The package's name that we made could be torch-nightly
290-
if [[ -n "$TORCH_PACKAGE_NAME" ]]; then
291-
package_name="$TORCH_PACKAGE_NAME"
292-
else
293-
package_name='torch'
294-
fi
295-
echo "Expecting the built wheels to be packages for '$package_name'"
296-
297292

298293
# Test that all the wheels work
299294
export OMP_NUM_THREADS=4 # on NUMA machines this takes too long
300295
pushd $PYTORCH_DIR/test
301-
for PYDIR in "${DESIRED_PYTHON[@]}"; do
302-
"${PYDIR}/bin/pip" uninstall -y "$package_name"
303-
"${PYDIR}/bin/pip" install "$package_name" --no-index -f /$WHEELHOUSE_DIR
304-
LD_LIBRARY_PATH="/usr/local/nvidia/lib64" PYCMD=$PYDIR/bin/python $PYDIR/bin/python run_test.py --exclude distributed
296+
for (( i=0; i<"${#DESIRED_PYTHON[@]}"; i++ )); do
297+
# This assumes that there is a 1:1 correspondence between python versions
298+
# and wheels, and that the python version is in the name of the wheel,
299+
# and that the python version matches the regex "cp\d\d-cp\d\dmu?"
300+
pydir="${python_installations[i]}"
301+
pyver="${DESIRED_PYTHON[i]}"
302+
pyver_short="${pyver:2:1}.${pyver:3:1}"
303+
304+
# Install the wheel for this Python version
305+
"${pydir}/bin/pip" uninstall -y "$package_name"
306+
"${pydir}/bin/pip" install "$package_name" --no-index -f /$WHEELHOUSE_DIR --no-dependencies
307+
308+
# Print info on the libraries installed in this wheel
309+
installed_libraries=($(find "$pydir/lib/python$pyver_short/site-packages/torch/" -name '*.so*'))
310+
echo "The wheel installed all of the libraries: ${installed_libraries[@]}"
311+
for installed_lib in "${installed_libraries[@]}"; do
312+
ldd "$installed_lib"
313+
done
314+
315+
# Test that the wheel works
316+
LD_LIBRARY_PATH="/usr/local/nvidia/lib64" PYCMD=$pydir/bin/python $pydir/bin/python run_test.py --exclude distributed
305317

306318
# Distributed tests are not expected to work on shared GPU machines (as of
307319
# 8/06/2018) so the errors from test_distributed are ignored. Expected

‎manywheel/build_cpu.sh‎

Lines changed: 54 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,10 @@
33
set -ex
44

55
if [[ -z "$PYTORCH_BUILD_VERSION" ]]; then
6-
export PYTORCH_BUILD_VERSION=0.4.1
6+
export PYTORCH_BUILD_VERSION=0.4.1
77
fi
88
if [[ -z "$PYTORCH_BUILD_NUMBER" ]]; then
9-
export PYTORCH_BUILD_NUMBER=2
9+
export PYTORCH_BUILD_NUMBER=2
1010
fi
1111
export TH_BINARY_BUILD=1
1212
export NO_CUDA=1
@@ -19,49 +19,48 @@ rm -rf /usr/local/cuda*
1919

2020
# Build for given Python versions, or for all in /opt/python if none given
2121
if [[ -z "$DESIRED_PYTHON" ]]; then
22-
DESIRED_PYTHON=($(ls -d /opt/python/*/))
22+
pushd /opt/python
23+
DESIRED_PYTHON=(*/)
24+
popd
2325
fi
24-
for (( i=0; i<"${#DESIRED_PYTHON[@]}"; i++ )); do
25-
# Convert eg. cp27-cp27m to /opt/python/cp27-cp27m
26-
if [[ ! -d "${DESIRED_PYTHON[$i]}" ]]; then
27-
if [[ -d "/opt/python/${DESIRED_PYTHON[$i]}" ]]; then
28-
DESIRED_PYTHON[$i]="/opt/python/${DESIRED_PYTHON[$i]}"
29-
else
30-
echo "Error: Given Python ${DESIRED_PYTHON[$i]} is not in /opt/python"
31-
echo "All array elements of env variable DESIRED_PYTHON must be"
32-
echo "valid Python installations under /opt/python"
33-
exit 1
26+
python_installations=()
27+
for desired_py in "${DESIRED_PYTHON[@]}"; do
28+
python_installations+=("/opt/python/$desired_py")
29+
if [[ ! -d "/opt/python/$desired_py" ]]; then
30+
echo "Error: Given Python $desired_py is not in /opt/python"
31+
echo "All array elements of env variable DESIRED_PYTHON must be"
32+
echo "valid Python installations under /opt/python"
33+
exit 1
3434
fi
35-
fi
3635
done
37-
echo "Will build for all Pythons: ${DESIRED_PYTHON[@]}"
36+
echo "Will build for all Pythons versions: ${DESIRED_PYTHON[@]}"
3837

3938
# ########################################################
4039
# # Compile wheels
4140
# #######################################################
4241
# clone pytorch source code
4342
PYTORCH_DIR="/pytorch"
4443
if [[ ! -d "$PYTORCH_DIR" ]]; then
45-
git clone https://github.com/pytorch/pytorch $PYTORCH_DIR
46-
pushd $PYTORCH_DIR
47-
if ! git checkout v${PYTORCH_BUILD_VERSION}; then
48-
git checkout tags/v${PYTORCH_BUILD_VERSION}
49-
fi
44+
git clone https://github.com/pytorch/pytorch $PYTORCH_DIR
45+
pushd $PYTORCH_DIR
46+
if ! git checkout v${PYTORCH_BUILD_VERSION}; then
47+
git checkout tags/v${PYTORCH_BUILD_VERSION}
48+
fi
5049
else
51-
# the pytorch dir will already be cloned and checked-out on jenkins jobs
52-
pushd $PYTORCH_DIR
50+
# the pytorch dir will already be cloned and checked-out on jenkins jobs
51+
pushd $PYTORCH_DIR
5352
fi
5453
git submodule update --init --recursive
5554

5655
OLD_PATH=$PATH
57-
for PYDIR in "${DESIRED_PYTHON[@]}"; do
56+
for PYDIR in "${python_installations[@]}"; do
5857
export PATH=$PYDIR/bin:$OLD_PATH
5958
python setup.py clean
6059
pip install -r requirements.txt
6160
if [[ $PYDIR == "/opt/python/cp37-cp37m" ]]; then
62-
pip install numpy==1.15
61+
pip install numpy==1.15
6362
else
64-
pip install numpy==1.11
63+
pip install numpy==1.11
6564
fi
6665
time python setup.py bdist_wheel -d $WHEELHOUSE_DIR
6766
done
@@ -82,11 +81,11 @@ fname_with_sha256() {
8281
DIRNAME=$(dirname $1)
8382
BASENAME=$(basename $1)
8483
if [[ $BASENAME == "libnvrtc-builtins.so" ]]; then
85-
echo $1
84+
echo $1
8685
else
87-
INITNAME=$(echo $BASENAME | cut -f1 -d".")
88-
ENDNAME=$(echo $BASENAME | cut -f 2- -d".")
89-
echo "$DIRNAME/$INITNAME-$HASH.$ENDNAME"
86+
INITNAME=$(echo $BASENAME | cut -f1 -d".")
87+
ENDNAME=$(echo $BASENAME | cut -f 2- -d".")
88+
echo "$DIRNAME/$INITNAME-$HASH.$ENDNAME"
9089
fi
9190
}
9291

@@ -200,32 +199,44 @@ for whl in /$WHEELHOUSE_DIR/torch*linux*.whl; do
200199
rm -rf tmp
201200
done
202201

203-
# Print out sizes of all wheels created
204-
echo "Succesfulle made wheels of size:"
205-
du -h /$WHEELHOUSE_DIR/torch*.whl
202+
# Take the actual package name. Note how this always works because pip converts
203+
# - to _ in names.
204+
pushd /$WHEELHOUSE_DIR
205+
built_wheels=(torch*.whl)
206+
IFS='-' read -r package_name some_unused_variable <<< "${built_wheels[0]}"
207+
echo "Expecting the built wheels to all be called '$package_name'"
208+
popd
206209

207210
# Copy wheels to host machine for persistence after the docker
208-
209211
mkdir -p /remote/$WHEELHOUSE_DIR
210212
cp /$WHEELHOUSE_DIR/torch*.whl /remote/$WHEELHOUSE_DIR/
211213

212214
# remove stuff before testing
213215
rm -rf /opt/rh
214216

215-
# The package's name that we made could be torch-nightly
216-
if [[ -n "$TORCH_PACKAGE_NAME" ]]; then
217-
package_name="$TORCH_PACKAGE_NAME"
218-
else
219-
package_name='torch'
220-
fi
221-
echo "Expecting the built wheels to be packages for '$package_name'"
222-
223217

224218
# Test that all the wheels work
225219
export OMP_NUM_THREADS=4 # on NUMA machines this takes too long
226220
pushd $PYTORCH_DIR/test
227-
for PYDIR in "${DESIRED_PYTHON[@]}"; do
221+
for (( i=0; i<"${#DESIRED_PYTHON[@]}"; i++ )); do
222+
# This assumes that there is a 1:1 correspondence between python versions
223+
# and wheels, and that the python version is in the name of the wheel,
224+
# and that the python version matches the regex "cp\d\d-cp\d\dmu?"
225+
pydir="${python_installations[i]}"
226+
pyver="${DESIRED_PYTHON[i]}"
227+
pyver_short="${pyver:2:1}.${pyver:3:1}"
228+
229+
# Install the wheel for this Python version
228230
"${PYDIR}/bin/pip" uninstall -y "$package_name"
229-
"${PYDIR}/bin/pip" install "$package_name" --no-index -f /$WHEELHOUSE_DIR
231+
"${PYDIR}/bin/pip" install "$package_name" --no-index -f /$WHEELHOUSE_DIR --no-dependencies
232+
233+
# Print info on the libraries installed in this wheel
234+
installed_libraries=($(find "$pydir/lib/python$pyver_short/site-packages/torch/" -name '*.so*'))
235+
echo "The wheel installed all of the libraries: ${installed_libraries[@]}"
236+
for installed_lib in "${installed_libraries[@]}"; do
237+
ldd "$installed_lib"
238+
done
239+
240+
# Test that the wheel works
230241
LD_LIBRARY_PATH="/usr/local/nvidia/lib64" PYCMD=$PYDIR/bin/python $PYDIR/bin/python run_test.py
231242
done

0 commit comments

Comments
 (0)