Skip to content

Commit 2ea3ecc

Browse files
Bordalexierule
Borda
authored and
lexierule
committed
remake nvidia docker (#6686)
* use latest * remake * examples
1 parent aa92c4c commit 2ea3ecc

File tree

6 files changed

+91
-29
lines changed

6 files changed

+91
-29
lines changed

‎azure-pipelines.yml‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -98,8 +98,9 @@ jobs:
9898
- script: |
9999
set -e
100100
python -m pytest pl_examples -v --maxfail=2 --durations=0
101-
python setup.py install --user --quiet
102-
bash pl_examples/run_ddp-example.sh
101+
pip install . --user --quiet
102+
bash pl_examples/run_examples-args.sh --gpus 1 --max_epochs 1 --batch_size 64 --limit_train_batches 5 --limit_val_batches 3
103+
bash pl_examples/run_ddp-examples.sh --max_epochs 1 --batch_size 32 --limit_train_batches 2 --limit_val_batches 2
103104
# cd pl_examples/basic_examples
104105
# bash submit_ddp_job.sh
105106
# bash submit_ddp2_job.sh

‎dockers/nvidia/Dockerfile‎

Lines changed: 55 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -12,26 +12,69 @@
1212
# See the License for the specific language governing permissions and
1313
# limitations under the License.
1414

15-
FROM nvcr.io/nvidia/pytorch:20.12-py3
15+
FROM nvcr.io/nvidia/cuda:11.1.1-runtime-ubuntu20.04
1616

1717
MAINTAINER PyTorchLightning <https://github.com/PyTorchLightning>
1818

1919
ARG LIGHTNING_VERSION=""
2020

21-
COPY ./ ./pytorch-lightning/
21+
SHELL ["/bin/bash", "-c"]
22+
# https://techoverflow.net/2019/05/18/how-to-fix-configuring-tzdata-interactive-input-when-building-docker-images/
23+
ENV \
24+
DEBIAN_FRONTEND=noninteractive \
25+
TZ=Europe/Prague \
26+
PATH="$PATH:/root/.local/bin" \
27+
CUDA_TOOLKIT_ROOT_DIR="/usr/local/cuda" \
28+
MKL_THREADING_LAYER=GNU
29+
30+
RUN apt-get update -qq && \
31+
apt-get install -y --no-install-recommends \
32+
build-essential \
33+
python3 \
34+
python3-distutils \
35+
python3-dev \
36+
pkg-config \
37+
cmake \
38+
git \
39+
wget \
40+
unzip \
41+
ca-certificates \
42+
&& \
43+
44+
# Cleaning
45+
apt-get autoremove -y && \
46+
apt-get clean && \
47+
rm -rf /root/.cache && \
48+
rm -rf /var/lib/apt/lists/* && \
49+
50+
# Setup PIP
51+
update-alternatives --install /usr/bin/python python /usr/bin/python3 1 && \
52+
wget https://bootstrap.pypa.io/get-pip.py --progress=bar:force:noscroll --no-check-certificate && \
53+
python get-pip.py && \
54+
rm get-pip.py && \
55+
pip --version
56+
57+
COPY ./ /home/pytorch-lightning/
2258

23-
# install dependencies
2459
RUN \
25-
# Disable cache
26-
#conda install "pip>20.1" && \
27-
#pip config set global.cache-dir false && \
28-
if [ -z $LIGHTNING_VERSION ] ; then \
29-
pip install ./pytorch-lightning --no-cache-dir ; \
30-
rm -rf pytorch-lightning ; \
31-
else \
60+
cd /home && \
61+
mv pytorch-lightning/notebooks . && \
62+
mv pytorch-lightning/pl_examples . && \
63+
# replace by specific version if asked
64+
if [ ! -z "$LIGHTNING_VERSION" ] ; then \
3265
rm -rf pytorch-lightning ; \
33-
pip install https://github.com/PyTorchLightning/pytorch-lightning/archive/${LIGHTNING_VERSION}.zip --no-cache-dir ; \
34-
fi
66+
wget https://github.com/PyTorchLightning/pytorch-lightning/archive/${LIGHTNING_VERSION}.zip --progress=bar:force:noscroll ; \
67+
unzip ${LIGHTNING_VERSION}.zip ; \
68+
mv pytorch-lightning-*/ pytorch-lightning ; \
69+
rm *.zip ; \
70+
fi && \
71+
72+
# Installations
73+
python -c "fname = './pytorch-lightning/requirements/extra.txt' ; lines = [line for line in open(fname).readlines() if not line.startswith('horovod')] ; open(fname, 'w').writelines(lines)" && \
74+
pip install -r ./pytorch-lightning/requirements/extra.txt -U --no-cache-dir && \
75+
pip install -r ./pytorch-lightning/requirements/examples.txt -U --no-cache-dir && \
76+
pip install ./pytorch-lightning --no-cache-dir && \
77+
rm -rf pytorch-lightning
3578

3679
RUN python --version && \
3780
pip --version && \

‎dockers/release/Dockerfile‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,12 +21,14 @@ MAINTAINER PyTorchLightning <https://github.com/PyTorchLightning>
2121

2222
ARG LIGHTNING_VERSION=""
2323

24-
COPY ./ ./pytorch-lightning/
24+
COPY ./ /home/pytorch-lightning/
2525

2626
# install dependencies
2727
RUN \
28-
# Disable cache
29-
#conda install "pip>20.1" && \
28+
cd /home && \
29+
mv pytorch-lightning/notebooks . && \
30+
mv pytorch-lightning/pl_examples . && \
31+
# replace by specific version if asked
3032
if [ ! -z "$LIGHTNING_VERSION" ] ; then \
3133
rm -rf pytorch-lightning ; \
3234
wget https://github.com/PyTorchLightning/pytorch-lightning/archive/${LIGHTNING_VERSION}.zip --progress=bar:force:noscroll ; \

‎pl_examples/run_ddp-example.sh‎

Lines changed: 0 additions & 12 deletions
This file was deleted.

‎pl_examples/run_ddp-examples.sh‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
#!/bin/bash
2+
3+
ARGS_EXTRA_DDP=" --gpus 2 --accelerator ddp"
4+
ARGS_EXTRA_AMP=" --precision 16"
5+
6+
python pl_examples/basic_examples/simple_image_classifier.py $@ ${ARGS_EXTRA_DDP}
7+
python pl_examples/basic_examples/simple_image_classifier.py $@ ${ARGS_EXTRA_DDP} ${ARGS_EXTRA_AMP}
8+
9+
python pl_examples/basic_examples/backbone_image_classifier.py $@ ${ARGS_EXTRA_DDP}
10+
python pl_examples/basic_examples/backbone_image_classifier.py $@ ${ARGS_EXTRA_DDP} ${ARGS_EXTRA_AMP}
11+
12+
python pl_examples/basic_examples/autoencoder.py $@ ${ARGS_EXTRA_DDP}
13+
python pl_examples/basic_examples/autoencoder.py $@ ${ARGS_EXTRA_DDP} ${ARGS_EXTRA_AMP}

‎pl_examples/run_examples-args.sh‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
#!/bin/bash
2+
3+
echo $@
4+
5+
full_path=$(realpath $0)
6+
echo $full_path
7+
8+
dir_path=$(dirname $full_path)
9+
echo $dir_path
10+
11+
python ${dir_path}/basic_examples/simple_image_classifier.py $@
12+
13+
python ${dir_path}/basic_examples/backbone_image_classifier.py $@
14+
15+
python ${dir_path}/basic_examples/autoencoder.py $@

0 commit comments

Comments
 (0)