From 11b79e37701caf743998da68d5ef26062d6b9d99 Mon Sep 17 00:00:00 2001 From: Shahriar Date: Sat, 10 Aug 2019 15:37:19 +0430 Subject: [PATCH 01/35] Added PNG reading functions --- CMakeLists.txt | 10 ++++ torchvision/csrc/image/image.h | 6 +++ torchvision/csrc/image/readpng.cpp | 83 ++++++++++++++++++++++++++++++ torchvision/csrc/image/readpng.h | 19 +++++++ 4 files changed, 118 insertions(+) create mode 100644 torchvision/csrc/image/image.h create mode 100644 torchvision/csrc/image/readpng.cpp create mode 100644 torchvision/csrc/image/readpng.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 1f04d51131e..9f7c87ffd6d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -10,6 +10,7 @@ if(WITH_CUDA) add_definitions(-D__CUDA_NO_HALF_OPERATORS__) endif() +find_package(PNG REQUIRED) find_package(Torch REQUIRED) find_package(pybind11 REQUIRED) @@ -21,7 +22,15 @@ endif() file(GLOB MODELS_HEADERS torchvision/csrc/models/*.h) file(GLOB MODELS_SOURCES torchvision/csrc/models/*.h torchvision/csrc/models/*.cpp) +file(GLOB IMAGE_HEADERS torchvision/csrc/image/*.h) +file(GLOB IMAGE_SOURCES torchvision/csrc/image/*.h torchvision/csrc/image/*.cpp) + +add_library(${PROJECT_NAME} SHARED ${MODELS_SOURCES} ${IMAGE_SOURCES}) +target_link_libraries(${PROJECT_NAME} PUBLIC "${PNG_LIBRARY}") +target_link_libraries(${PROJECT_NAME} PUBLIC "${TORCH_LIBRARIES}") + add_library(${PROJECT_NAME} SHARED ${MODELS_SOURCES} ${OPERATOR_SOURCES}) + target_link_libraries(${PROJECT_NAME} PRIVATE ${TORCH_LIBRARIES} pybind11::pybind11) set_target_properties(${PROJECT_NAME} PROPERTIES EXPORT_NAME TorchVision) @@ -63,3 +72,4 @@ if(WITH_CUDA) DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/${PROJECT_NAME}/cuda) endif() install(FILES ${MODELS_HEADERS} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/${PROJECT_NAME}/models) +include_directories(${PNG_INCLUDE_DIR}) diff --git a/torchvision/csrc/image/image.h b/torchvision/csrc/image/image.h new file mode 100644 index 00000000000..f3b6973ecf7 --- /dev/null +++ b/torchvision/csrc/image/image.h @@ -0,0 +1,6 @@ +#ifndef IMAGE_H +#define IMAGE_H + +#include "readpng.h" + +#endif // IMAGE_H diff --git a/torchvision/csrc/image/readpng.cpp b/torchvision/csrc/image/readpng.cpp new file mode 100644 index 00000000000..d589d60912e --- /dev/null +++ b/torchvision/csrc/image/readpng.cpp @@ -0,0 +1,83 @@ +#include "readpng.h" + +#include + +#include + +namespace torch { +namespace vision { +namespace image { +namespace impl { +bool is_png(const void* data) { + return png_sig_cmp(png_const_bytep(data), 0, 8) == 0; +} + +torch::Tensor readpng(const void* data) { + struct Reader { + png_const_bytep ptr; + } reader; + + reader.ptr = png_const_bytep(data) + 8; + auto read_callback = + [](png_structp png_ptr, png_bytep output, png_size_t bytes) { + auto reader = static_cast(png_get_io_ptr(png_ptr)); + std::copy(reader->ptr, reader->ptr + bytes, output); + reader->ptr += bytes; + }; + + auto png_ptr = + png_create_read_struct(PNG_LIBPNG_VER_STRING, nullptr, nullptr, nullptr); + if (!png_ptr) + return torch::tensor({0}); + + auto info_ptr = png_create_info_struct(png_ptr); + if (!info_ptr) { + png_destroy_read_struct(&png_ptr, nullptr, nullptr); + return torch::tensor({0}); + } + + if (setjmp(png_jmpbuf(png_ptr)) != 0) { + png_destroy_read_struct(&png_ptr, &info_ptr, nullptr); + return torch::tensor({0}); + } + + png_set_read_fn(png_ptr, &reader, read_callback); + png_set_sig_bytes(png_ptr, 8); + png_read_info(png_ptr, info_ptr); + + png_uint_32 width, height; + int bit_depth, color_type; + auto retval = png_get_IHDR( + png_ptr, + info_ptr, + &width, + &height, + &bit_depth, + &color_type, + nullptr, + nullptr, + nullptr); + + if (retval != 1 || color_type != PNG_COLOR_TYPE_RGB) { + png_destroy_read_struct(&png_ptr, &info_ptr, nullptr); + return torch::tensor({0}); + } + + auto tensor = + torch::empty({int64_t(height), int64_t(width), int64_t(3)}, torch::kU8); + auto ptr = tensor.data(); + auto bytes = png_get_rowbytes(png_ptr, info_ptr); + + for (decltype(height) i = 0; i < height; ++i) { + png_read_row(png_ptr, ptr, nullptr); + ptr += bytes; + } + + png_destroy_read_struct(&png_ptr, &info_ptr, nullptr); + return tensor; +} + +} // namespace impl +} // namespace image +} // namespace vision +} // namespace torch diff --git a/torchvision/csrc/image/readpng.h b/torchvision/csrc/image/readpng.h new file mode 100644 index 00000000000..135f07e3f47 --- /dev/null +++ b/torchvision/csrc/image/readpng.h @@ -0,0 +1,19 @@ +#ifndef READPNG_H +#define READPNG_H + +#include + +namespace torch { +namespace vision { +namespace image { +namespace impl { + +bool is_png(const void* data); +torch::Tensor readpng(const void* data); + +} // namespace impl +} // namespace image +} // namespace vision +} // namespace torch + +#endif // READPNG_H From 9633a4974b945dfb1500457a703da4fd0b88a0d1 Mon Sep 17 00:00:00 2001 From: Ryad ZENINE Date: Sat, 7 Dec 2019 13:37:08 +0100 Subject: [PATCH 02/35] Adds readpng features --- .circleci/config.yml | 6 +- .circleci/config.yml.in | 6 +- .travis.yml | 1 + CMakeLists.txt | 4 +- packaging/build_conda.sh | 1 + packaging/build_wheel.sh | 1 + packaging/build_wheel_macos.sh | 15 ++++ packaging/torchvision/meta.yaml | 2 + packaging/wheel/linux_manywheel.sh | 2 +- setup.py | 4 +- test/test_image.py | 37 ++++++++++ .../readpng.cpp => cpu/image/readpng_cpu.cpp} | 73 +++++++++---------- torchvision/csrc/cpu/image/readpng_cpu.h | 6 ++ torchvision/csrc/image.h | 3 + torchvision/csrc/image/image.h | 6 -- torchvision/csrc/image/readpng.h | 19 ----- torchvision/csrc/vision.cpp | 2 + torchvision/io/image.py | 47 ++++++++++++ 18 files changed, 162 insertions(+), 73 deletions(-) create mode 100755 packaging/build_wheel_macos.sh create mode 100644 test/test_image.py rename torchvision/csrc/{image/readpng.cpp => cpu/image/readpng_cpu.cpp} (59%) create mode 100644 torchvision/csrc/cpu/image/readpng_cpu.h create mode 100644 torchvision/csrc/image.h delete mode 100644 torchvision/csrc/image/image.h delete mode 100644 torchvision/csrc/image/readpng.h create mode 100644 torchvision/io/image.py diff --git a/.circleci/config.yml b/.circleci/config.yml index 461bd6f9cb7..4986e9a6687 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -128,7 +128,7 @@ jobs: ca-certificates \ curl \ gnupg-agent \ - software-properties-common + software-properties-common curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - @@ -219,7 +219,8 @@ jobs: curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh sh conda.sh -b source $HOME/miniconda3/bin/activate - packaging/build_wheel.sh + conda install -yq libpng + packaging/build_wheel_macos.sh - store_artifacts: path: dist - persist_to_workspace: @@ -239,6 +240,7 @@ jobs: sh conda.sh -b source $HOME/miniconda3/bin/activate conda install -yq conda-build + conda install -yq libpng packaging/build_conda.sh - store_artifacts: path: /Users/distiller/miniconda3/conda-bld/osx-64 diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index e3747134c6f..7a14720dbff 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -128,7 +128,7 @@ jobs: ca-certificates \ curl \ gnupg-agent \ - software-properties-common + software-properties-common curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - @@ -219,7 +219,8 @@ jobs: curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh sh conda.sh -b source $HOME/miniconda3/bin/activate - packaging/build_wheel.sh + conda install -yq libpng + packaging/build_wheel_macos.sh - store_artifacts: path: dist - persist_to_workspace: @@ -239,6 +240,7 @@ jobs: sh conda.sh -b source $HOME/miniconda3/bin/activate conda install -yq conda-build + conda install -yq libpng packaging/build_conda.sh - store_artifacts: path: /Users/distiller/miniconda3/conda-bld/osx-64 diff --git a/.travis.yml b/.travis.yml index 1b6ecb7a65b..4960ece27da 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,6 +26,7 @@ matrix: before_install: - sudo apt-get update + - sudo apt-get install -y libpng16-16 libpng16-dev - wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh; - bash miniconda.sh -b -p $HOME/miniconda - export PATH="$HOME/miniconda/bin:$PATH" diff --git a/CMakeLists.txt b/CMakeLists.txt index 9f7c87ffd6d..a3ad8fe6afd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -22,8 +22,8 @@ endif() file(GLOB MODELS_HEADERS torchvision/csrc/models/*.h) file(GLOB MODELS_SOURCES torchvision/csrc/models/*.h torchvision/csrc/models/*.cpp) -file(GLOB IMAGE_HEADERS torchvision/csrc/image/*.h) -file(GLOB IMAGE_SOURCES torchvision/csrc/image/*.h torchvision/csrc/image/*.cpp) +file(GLOB IMAGE_HEADERS torchvision/csrc/image.h) +file(GLOB IMAGE_SOURCES torchvision/csrc/cpu/image/*.h torchvision/csrc/cpu/image/*.cpp) add_library(${PROJECT_NAME} SHARED ${MODELS_SOURCES} ${IMAGE_SOURCES}) target_link_libraries(${PROJECT_NAME} PUBLIC "${PNG_LIBRARY}") diff --git a/packaging/build_conda.sh b/packaging/build_conda.sh index 61022f3c5fa..ed7725326c9 100755 --- a/packaging/build_conda.sh +++ b/packaging/build_conda.sh @@ -10,4 +10,5 @@ export SOURCE_ROOT_DIR="$PWD" setup_conda_pytorch_constraint setup_conda_cudatoolkit_constraint setup_visual_studio_constraint +conda install -yq libpng conda build $CONDA_CHANNEL_FLAGS -c defaults -c conda-forge --no-anaconda-upload --python "$PYTHON_VERSION" packaging/torchvision diff --git a/packaging/build_wheel.sh b/packaging/build_wheel.sh index 5d073d9c104..ebcc318c0aa 100755 --- a/packaging/build_wheel.sh +++ b/packaging/build_wheel.sh @@ -7,6 +7,7 @@ script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" export BUILD_TYPE=wheel setup_env 0.6.0 setup_wheel_python +yum install -y libpng libpng-devel pip_install numpy pyyaml future ninja setup_pip_pytorch_version python setup.py clean diff --git a/packaging/build_wheel_macos.sh b/packaging/build_wheel_macos.sh new file mode 100755 index 00000000000..7d37239563d --- /dev/null +++ b/packaging/build_wheel_macos.sh @@ -0,0 +1,15 @@ +#!/bin/bash +set -ex + +script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +. "$script_dir/pkg_helpers.bash" + +export BUILD_TYPE=wheel +setup_env 0.5.0 +setup_wheel_python +pip_install numpy pyyaml future ninja +# TODO remove after https://github.com/pytorch/pytorch/pull/27282 gets merged +pip_install six +setup_pip_pytorch_version +python setup.py clean +IS_WHEEL=1 python setup.py bdist_wheel diff --git a/packaging/torchvision/meta.yaml b/packaging/torchvision/meta.yaml index 1bc199e437b..c40df3f5b2c 100644 --- a/packaging/torchvision/meta.yaml +++ b/packaging/torchvision/meta.yaml @@ -12,6 +12,7 @@ requirements: host: - python - setuptools + - libpng >=1.6.37 {{ environ.get('CONDA_PYTORCH_BUILD_CONSTRAINT') }} {{ environ.get('CONDA_CUDATOOLKIT_CONSTRAINT') }} {{ environ.get('CONDA_CPUONLY_FEATURE') }} @@ -20,6 +21,7 @@ requirements: - python - pillow >=4.1.1 - numpy >=1.11 + - libpng >=1.6.37 - six {{ environ.get('CONDA_PYTORCH_CONSTRAINT') }} {{ environ.get('CONDA_CUDATOOLKIT_CONSTRAINT') }} diff --git a/packaging/wheel/linux_manywheel.sh b/packaging/wheel/linux_manywheel.sh index d04e334d237..a3bb22d7f88 100644 --- a/packaging/wheel/linux_manywheel.sh +++ b/packaging/wheel/linux_manywheel.sh @@ -32,7 +32,7 @@ rm -rf vision git clone https://github.com/pytorch/vision cd /tmp/vision - +yum install -y libpng libpng-devel for PYDIR in "${python_installations[@]}"; do export PATH=$PYDIR/bin:$OLD_PATH pip install --upgrade pip diff --git a/setup.py b/setup.py index 71d420573ed..4050b098fcb 100644 --- a/setup.py +++ b/setup.py @@ -83,9 +83,10 @@ def get_extensions(): main_file = glob.glob(os.path.join(extensions_dir, '*.cpp')) source_cpu = glob.glob(os.path.join(extensions_dir, 'cpu', '*.cpp')) + source_image_cpu = glob.glob(os.path.join(extensions_dir, 'cpu', 'image', '*.cpp')) source_cuda = glob.glob(os.path.join(extensions_dir, 'cuda', '*.cu')) - sources = main_file + source_cpu + sources = main_file + source_cpu + source_image_cpu extension = CppExtension compile_cpp_tests = os.getenv('WITH_CPP_MODELS_TEST', '0') == '1' @@ -142,6 +143,7 @@ def get_extensions(): extension( 'torchvision._C', sources, + libraries=['png'], include_dirs=include_dirs, define_macros=define_macros, extra_compile_args=extra_compile_args, diff --git a/test/test_image.py b/test/test_image.py new file mode 100644 index 00000000000..38e74eaec62 --- /dev/null +++ b/test/test_image.py @@ -0,0 +1,37 @@ +import os +import unittest + +import torch +from PIL import Image +from torchvision.io.image import read_png, decode_png +import numpy as np + +IMAGE_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "assets", "fakedata", "imagefolder") + + +def get_png_images(directory): + assert os.path.isdir(directory) + for root, dir, files in os.walk(directory): + for fl in files: + _, ext = os.path.splitext(fl) + if ext == ".png": + yield os.path.join(root, fl) + + +class ImageTester(unittest.TestCase): + def test_read_png(self): + for img_path in get_png_images(IMAGE_DIR): + img_pil = torch.from_numpy(np.array(Image.open(img_path))) + img_lpng = read_png(img_path) + self.assertTrue(torch.all(img_lpng == img_pil)) + + def test_decode_png(self): + for img_path in get_png_images(IMAGE_DIR): + img_pil = torch.from_numpy(np.array(Image.open(img_path))) + size = os.path.getsize(img_path) + img_lpng = decode_png(torch.from_file(img_path, dtype=torch.uint8, size=size)) + self.assertTrue(torch.all(img_lpng == img_pil)) + + +if __name__ == '__main__': + unittest.main() diff --git a/torchvision/csrc/image/readpng.cpp b/torchvision/csrc/cpu/image/readpng_cpu.cpp similarity index 59% rename from torchvision/csrc/image/readpng.cpp rename to torchvision/csrc/cpu/image/readpng_cpu.cpp index d589d60912e..a395e5ed663 100644 --- a/torchvision/csrc/image/readpng.cpp +++ b/torchvision/csrc/cpu/image/readpng_cpu.cpp @@ -1,48 +1,43 @@ -#include "readpng.h" - -#include +#include "readpng_cpu.h" #include +#include +#include -namespace torch { -namespace vision { -namespace image { -namespace impl { -bool is_png(const void* data) { - return png_sig_cmp(png_const_bytep(data), 0, 8) == 0; -} - -torch::Tensor readpng(const void* data) { - struct Reader { - png_const_bytep ptr; - } reader; - - reader.ptr = png_const_bytep(data) + 8; - auto read_callback = - [](png_structp png_ptr, png_bytep output, png_size_t bytes) { - auto reader = static_cast(png_get_io_ptr(png_ptr)); - std::copy(reader->ptr, reader->ptr + bytes, output); - reader->ptr += bytes; - }; - +torch::Tensor decodePNG(const torch::Tensor& data) { auto png_ptr = png_create_read_struct(PNG_LIBPNG_VER_STRING, nullptr, nullptr, nullptr); - if (!png_ptr) - return torch::tensor({0}); - + TORCH_CHECK(png_ptr, "libpng read structure allocation failed!") auto info_ptr = png_create_info_struct(png_ptr); if (!info_ptr) { png_destroy_read_struct(&png_ptr, nullptr, nullptr); - return torch::tensor({0}); + // Seems redundant with the if statement. done here to avoid leaking memory. + TORCH_CHECK(info_ptr, "libpng info structure allocation failed!") } + auto datap = data.accessor().data(); + if (setjmp(png_jmpbuf(png_ptr)) != 0) { png_destroy_read_struct(&png_ptr, &info_ptr, nullptr); - return torch::tensor({0}); + auto errptr = static_cast(png_get_error_ptr(png_ptr)); + TORCH_CHECK(false, "Internal error."); } + auto is_png = !png_sig_cmp(datap, 0, 8); + TORCH_CHECK(is_png, "Content is not png!") - png_set_read_fn(png_ptr, &reader, read_callback); + struct Reader { + png_const_bytep ptr; + } reader; + reader.ptr = png_const_bytep(datap) + 8; + + auto read_callback = + [](png_structp png_ptr, png_bytep output, png_size_t bytes) { + auto reader = static_cast(png_get_io_ptr(png_ptr)); + std::copy(reader->ptr, reader->ptr + bytes, output); + reader->ptr += bytes; + }; png_set_sig_bytes(png_ptr, 8); + png_set_read_fn(png_ptr, &reader, read_callback); png_read_info(png_ptr, info_ptr); png_uint_32 width, height; @@ -58,26 +53,24 @@ torch::Tensor readpng(const void* data) { nullptr, nullptr); - if (retval != 1 || color_type != PNG_COLOR_TYPE_RGB) { + if (retval != 1) { + png_destroy_read_struct(&png_ptr, &info_ptr, nullptr); + TORCH_CHECK(retval == 1, "Could read image metadata from content.") + } + if (color_type != PNG_COLOR_TYPE_RGB) { png_destroy_read_struct(&png_ptr, &info_ptr, nullptr); - return torch::tensor({0}); + TORCH_CHECK( + color_type == PNG_COLOR_TYPE_RGB, "Non RGB images are not supported.") } auto tensor = torch::empty({int64_t(height), int64_t(width), int64_t(3)}, torch::kU8); - auto ptr = tensor.data(); + auto ptr = tensor.accessor().data(); auto bytes = png_get_rowbytes(png_ptr, info_ptr); - for (decltype(height) i = 0; i < height; ++i) { png_read_row(png_ptr, ptr, nullptr); ptr += bytes; } - png_destroy_read_struct(&png_ptr, &info_ptr, nullptr); return tensor; } - -} // namespace impl -} // namespace image -} // namespace vision -} // namespace torch diff --git a/torchvision/csrc/cpu/image/readpng_cpu.h b/torchvision/csrc/cpu/image/readpng_cpu.h new file mode 100644 index 00000000000..d2151a43aa9 --- /dev/null +++ b/torchvision/csrc/cpu/image/readpng_cpu.h @@ -0,0 +1,6 @@ +#pragma once + +#include +#include + +torch::Tensor decodePNG(const torch::Tensor& data); diff --git a/torchvision/csrc/image.h b/torchvision/csrc/image.h new file mode 100644 index 00000000000..01d2063564d --- /dev/null +++ b/torchvision/csrc/image.h @@ -0,0 +1,3 @@ +#pragma once + +#include "cpu/image/readpng_cpu.h" diff --git a/torchvision/csrc/image/image.h b/torchvision/csrc/image/image.h deleted file mode 100644 index f3b6973ecf7..00000000000 --- a/torchvision/csrc/image/image.h +++ /dev/null @@ -1,6 +0,0 @@ -#ifndef IMAGE_H -#define IMAGE_H - -#include "readpng.h" - -#endif // IMAGE_H diff --git a/torchvision/csrc/image/readpng.h b/torchvision/csrc/image/readpng.h deleted file mode 100644 index 135f07e3f47..00000000000 --- a/torchvision/csrc/image/readpng.h +++ /dev/null @@ -1,19 +0,0 @@ -#ifndef READPNG_H -#define READPNG_H - -#include - -namespace torch { -namespace vision { -namespace image { -namespace impl { - -bool is_png(const void* data); -torch::Tensor readpng(const void* data); - -} // namespace impl -} // namespace image -} // namespace vision -} // namespace torch - -#endif // READPNG_H diff --git a/torchvision/csrc/vision.cpp b/torchvision/csrc/vision.cpp index 8d8699ecc26..c01537de7b6 100644 --- a/torchvision/csrc/vision.cpp +++ b/torchvision/csrc/vision.cpp @@ -11,6 +11,7 @@ #include "ROIAlign.h" #include "ROIPool.h" #include "empty_tensor_op.h" +#include "image.h" #include "nms.h" // If we are in a Windows environment, we need to define @@ -49,4 +50,5 @@ static auto registry = .op("torchvision::ps_roi_align", &ps_roi_align) .op("torchvision::ps_roi_pool", &ps_roi_pool) .op("torchvision::deform_conv2d", &deform_conv2d) + .op("torchvision::decode_png", &decodePNG) .op("torchvision::_cuda_version", &_cuda_version); diff --git a/torchvision/io/image.py b/torchvision/io/image.py new file mode 100644 index 00000000000..ded604fdd60 --- /dev/null +++ b/torchvision/io/image.py @@ -0,0 +1,47 @@ +import torch +from torch import nn, Tensor +import os + + +def decode_png(input): + # type: (Tensor) -> Tensor + """ + Decodes a PNG image into a 3 dimensional RGB Tensor. + The values of the output tensor are uint8 between 0 and 255. + + Arguments: + input (Tensor[1]): a one dimensional int8 tensor containing + the raw bytes of the PNG image. + + Returns: + output (Tensor[image_width, image_height, 3]) + """ + if not isinstance(input, torch.Tensor) or len(input) == 0: + raise ValueError("Expected a non empty 1-dimensional tensor.") + + if not input.dtype == torch.uint8: + raise ValueError("Expected a torch.uint8 tensor.") + output = torch.ops.torchvision.decode_png(input) + return output + + +def read_png(path): + # type: (str) -> Tensor + """ + Reads a PNG image into a 3 dimensional RGB Tensor. + The values of the output tensor are uint8 between 0 and 255. + + Arguments: + path (str): path of the PNG image. + + Returns: + output (Tensor[image_width, image_height, 3]) + """ + if not os.path.isfile(path): + raise ValueError("Excepted a valid file path.") + + size = os.path.getsize(path) + if size == 0: + raise ValueError("Excepted a non empty file.") + data = torch.from_file(path, dtype=torch.uint8, size=size) + return decode_png(data) From c4528e8cd5b5783e0170f09225d525b13c8b5469 Mon Sep 17 00:00:00 2001 From: Ryad ZENINE Date: Fri, 27 Dec 2019 23:42:49 +0100 Subject: [PATCH 03/35] Adds Jpeg Reading Capabilities --- .travis.yml | 2 +- CMakeLists.txt | 2 + packaging/build_conda.sh | 1 + packaging/build_wheel.sh | 2 +- setup.py | 2 +- test/test_image.py | 23 ++++++++--- torchvision/csrc/cpu/image/readjpeg_cpu.cpp | 37 ++++++++++++++++++ torchvision/csrc/cpu/image/readjpeg_cpu.h | 6 +++ torchvision/csrc/cpu/image/readpng_cpu.cpp | 2 +- torchvision/csrc/image.h | 2 + torchvision/csrc/vision.cpp | 1 + torchvision/io/image.py | 43 +++++++++++++++++++++ 12 files changed, 114 insertions(+), 9 deletions(-) create mode 100644 torchvision/csrc/cpu/image/readjpeg_cpu.cpp create mode 100644 torchvision/csrc/cpu/image/readjpeg_cpu.h diff --git a/.travis.yml b/.travis.yml index 4960ece27da..50c87822fc6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,7 +26,7 @@ matrix: before_install: - sudo apt-get update - - sudo apt-get install -y libpng16-16 libpng16-dev + - sudo apt-get install -y libpng16-16 libpng16-dev libjpeg-turbo8-dev libjpeg-turbo8 - wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh; - bash miniconda.sh -b -p $HOME/miniconda - export PATH="$HOME/miniconda/bin:$PATH" diff --git a/CMakeLists.txt b/CMakeLists.txt index a3ad8fe6afd..a8c0571161b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -11,6 +11,7 @@ if(WITH_CUDA) endif() find_package(PNG REQUIRED) +add_library(LibJPEGTurbo SHARED IMPORTED) find_package(Torch REQUIRED) find_package(pybind11 REQUIRED) @@ -27,6 +28,7 @@ file(GLOB IMAGE_SOURCES torchvision/csrc/cpu/image/*.h torchvision/csrc/cpu/imag add_library(${PROJECT_NAME} SHARED ${MODELS_SOURCES} ${IMAGE_SOURCES}) target_link_libraries(${PROJECT_NAME} PUBLIC "${PNG_LIBRARY}") +target_link_libraries(${PROJECT_NAME} PUBLIC "${LibJPEGTurbo}") target_link_libraries(${PROJECT_NAME} PUBLIC "${TORCH_LIBRARIES}") add_library(${PROJECT_NAME} SHARED ${MODELS_SOURCES} ${OPERATOR_SOURCES}) diff --git a/packaging/build_conda.sh b/packaging/build_conda.sh index ed7725326c9..8b518c5804d 100755 --- a/packaging/build_conda.sh +++ b/packaging/build_conda.sh @@ -11,4 +11,5 @@ setup_conda_pytorch_constraint setup_conda_cudatoolkit_constraint setup_visual_studio_constraint conda install -yq libpng +conda install -c conda-forge libjpeg-turbo conda build $CONDA_CHANNEL_FLAGS -c defaults -c conda-forge --no-anaconda-upload --python "$PYTHON_VERSION" packaging/torchvision diff --git a/packaging/build_wheel.sh b/packaging/build_wheel.sh index ebcc318c0aa..f7a49fc35a6 100755 --- a/packaging/build_wheel.sh +++ b/packaging/build_wheel.sh @@ -7,7 +7,7 @@ script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" export BUILD_TYPE=wheel setup_env 0.6.0 setup_wheel_python -yum install -y libpng libpng-devel +yum install -y libpng libpng-devel turbojpeg turbojpeg-devel pip_install numpy pyyaml future ninja setup_pip_pytorch_version python setup.py clean diff --git a/setup.py b/setup.py index 4050b098fcb..548ac773082 100644 --- a/setup.py +++ b/setup.py @@ -143,7 +143,7 @@ def get_extensions(): extension( 'torchvision._C', sources, - libraries=['png'], + libraries=['png', 'turbojpeg'], include_dirs=include_dirs, define_macros=define_macros, extra_compile_args=extra_compile_args, diff --git a/test/test_image.py b/test/test_image.py index 38e74eaec62..aadf4bcf775 100644 --- a/test/test_image.py +++ b/test/test_image.py @@ -3,35 +3,48 @@ import torch from PIL import Image -from torchvision.io.image import read_png, decode_png +from torchvision.io.image import read_png, decode_png, decode_jpeg, read_jpeg import numpy as np IMAGE_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "assets", "fakedata", "imagefolder") -def get_png_images(directory): +def get_images(directory, img_ext): assert os.path.isdir(directory) for root, dir, files in os.walk(directory): for fl in files: _, ext = os.path.splitext(fl) - if ext == ".png": + if ext == img_ext: yield os.path.join(root, fl) class ImageTester(unittest.TestCase): def test_read_png(self): - for img_path in get_png_images(IMAGE_DIR): + for img_path in get_images(IMAGE_DIR, "png"): img_pil = torch.from_numpy(np.array(Image.open(img_path))) img_lpng = read_png(img_path) self.assertTrue(torch.all(img_lpng == img_pil)) def test_decode_png(self): - for img_path in get_png_images(IMAGE_DIR): + for img_path in get_images(IMAGE_DIR, "png"): img_pil = torch.from_numpy(np.array(Image.open(img_path))) size = os.path.getsize(img_path) img_lpng = decode_png(torch.from_file(img_path, dtype=torch.uint8, size=size)) self.assertTrue(torch.all(img_lpng == img_pil)) + def test_read_jpeg(self): + for img_path in get_images(IMAGE_DIR, "jpg"): + img_pil = torch.from_numpy(np.array(Image.open(img_path))) + img_ljpeg = read_jpeg(img_path) + self.assertTrue(torch.all(img_ljpeg == img_pil)) + + def test_decode_jpeg(self): + for img_path in get_images(IMAGE_DIR, "jpg"): + img_pil = torch.from_numpy(np.array(Image.open(img_path))) + size = os.path.getsize(img_path) + img_ljpeg = decode_png(torch.from_file(img_path, dtype=torch.uint8, size=size)) + self.assertTrue(torch.all(img_ljpeg == img_pil)) + if __name__ == '__main__': unittest.main() diff --git a/torchvision/csrc/cpu/image/readjpeg_cpu.cpp b/torchvision/csrc/cpu/image/readjpeg_cpu.cpp new file mode 100644 index 00000000000..a1668d16d6e --- /dev/null +++ b/torchvision/csrc/cpu/image/readjpeg_cpu.cpp @@ -0,0 +1,37 @@ +#include "readpng_cpu.h" + +#include +#include +#include +#include + +torch::Tensor decodeJPEG(const torch::Tensor& data) { + + tjhandle tjInstance = tjInitDecompress(); + if (tjInstance == NULL) { + TORCH_CHECK("libjpeg-turbo decompression initialization failed."); + } + + auto datap = data.accessor().data(); + + int width, height; + if (tjDecompressHeader(tjInstance, datap, data.numel(), &width, &height) < 0) { + tjDestroy(tjInstance); + TORCH_CHECK("Error while reading jpeg headers"); + } + + auto tensor = + torch::empty({int64_t(height), int64_t(width), int64_t(3)}, torch::kU8); + auto ptr = tensor.accessor().data(); + + int pixelFormat = TJPF_RGB; + if (tjDecompress(tjInstance, datap, data.numel(), ptr, width, 0, height, + pixelFormat, 0) < 0){ + tjDestroy(tjInstance); + TORCH_CHECK("decompressing JPEG image"); + } + + tjDestroy(tjInstance); + + return tensor; +} diff --git a/torchvision/csrc/cpu/image/readjpeg_cpu.h b/torchvision/csrc/cpu/image/readjpeg_cpu.h new file mode 100644 index 00000000000..ea3e87858dd --- /dev/null +++ b/torchvision/csrc/cpu/image/readjpeg_cpu.h @@ -0,0 +1,6 @@ +#pragma once + +#include +#include + +torch::Tensor decodeJPEG(const torch::Tensor& data); diff --git a/torchvision/csrc/cpu/image/readpng_cpu.cpp b/torchvision/csrc/cpu/image/readpng_cpu.cpp index a395e5ed663..aaa5d060d6f 100644 --- a/torchvision/csrc/cpu/image/readpng_cpu.cpp +++ b/torchvision/csrc/cpu/image/readpng_cpu.cpp @@ -52,7 +52,7 @@ torch::Tensor decodePNG(const torch::Tensor& data) { nullptr, nullptr, nullptr); - + if (retval != 1) { png_destroy_read_struct(&png_ptr, &info_ptr, nullptr); TORCH_CHECK(retval == 1, "Could read image metadata from content.") diff --git a/torchvision/csrc/image.h b/torchvision/csrc/image.h index 01d2063564d..c93673c2cf5 100644 --- a/torchvision/csrc/image.h +++ b/torchvision/csrc/image.h @@ -1,3 +1,5 @@ #pragma once #include "cpu/image/readpng_cpu.h" +#include "cpu/image/readjpeg_cpu.h" + diff --git a/torchvision/csrc/vision.cpp b/torchvision/csrc/vision.cpp index c01537de7b6..eb96180a5f8 100644 --- a/torchvision/csrc/vision.cpp +++ b/torchvision/csrc/vision.cpp @@ -51,4 +51,5 @@ static auto registry = .op("torchvision::ps_roi_pool", &ps_roi_pool) .op("torchvision::deform_conv2d", &deform_conv2d) .op("torchvision::decode_png", &decodePNG) + .op("torchvision::decode_jpeg", &decodeJPEG) .op("torchvision::_cuda_version", &_cuda_version); diff --git a/torchvision/io/image.py b/torchvision/io/image.py index ded604fdd60..fe1330b036a 100644 --- a/torchvision/io/image.py +++ b/torchvision/io/image.py @@ -45,3 +45,46 @@ def read_png(path): raise ValueError("Excepted a non empty file.") data = torch.from_file(path, dtype=torch.uint8, size=size) return decode_png(data) + +def decode_jpeg(input): + # type: (Tensor) -> Tensor + """ + Decodes a JPEG image into a 3 dimensional RGB Tensor. + The values of the output tensor are uint8 between 0 and 255. + + Arguments: + input (Tensor[1]): a one dimensional int8 tensor containing + the raw bytes of the JPEG image. + + Returns: + output (Tensor[image_width, image_height, 3]) + """ + if not isinstance(input, torch.Tensor) or len(input) == 0: + raise ValueError("Expected a non empty 1-dimensional tensor.") + + if not input.dtype == torch.uint8: + raise ValueError("Expected a torch.uint8 tensor.") + output = torch.ops.torchvision.decode_jpeg(input) + return output + + +def read_jpeg(path): + # type: (str) -> Tensor + """ + Reads a JPEG image into a 3 dimensional RGB Tensor. + The values of the output tensor are uint8 between 0 and 255. + + Arguments: + path (str): path of the JPEG image. + + Returns: + output (Tensor[image_width, image_height, 3]) + """ + if not os.path.isfile(path): + raise ValueError("Excepted a valid file path.") + + size = os.path.getsize(path) + if size == 0: + raise ValueError("Excepted a non empty file.") + data = torch.from_file(path, dtype=torch.uint8, size=size) + return decode_jpeg(data) From 5ab970c8a0b15a30386667559cb1b77e8d35b085 Mon Sep 17 00:00:00 2001 From: Ryad ZENINE Date: Wed, 15 Jan 2020 09:18:08 +0100 Subject: [PATCH 04/35] Fix type in error message following code review --- torchvision/io/image.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/torchvision/io/image.py b/torchvision/io/image.py index fe1330b036a..3faf365abcd 100644 --- a/torchvision/io/image.py +++ b/torchvision/io/image.py @@ -38,11 +38,11 @@ def read_png(path): output (Tensor[image_width, image_height, 3]) """ if not os.path.isfile(path): - raise ValueError("Excepted a valid file path.") + raise ValueError("Expected a valid file path.") size = os.path.getsize(path) if size == 0: - raise ValueError("Excepted a non empty file.") + raise ValueError("Expected a non empty file.") data = torch.from_file(path, dtype=torch.uint8, size=size) return decode_png(data) @@ -81,10 +81,10 @@ def read_jpeg(path): output (Tensor[image_width, image_height, 3]) """ if not os.path.isfile(path): - raise ValueError("Excepted a valid file path.") + raise ValueError("Expected a valid file path.") size = os.path.getsize(path) if size == 0: - raise ValueError("Excepted a non empty file.") + raise ValueError("Expected a non empty file.") data = torch.from_file(path, dtype=torch.uint8, size=size) return decode_jpeg(data) From eed4947e539553f9af45a26624d65e7a20bb75e7 Mon Sep 17 00:00:00 2001 From: Ryad ZENINE Date: Sun, 9 Feb 2020 16:54:50 +0100 Subject: [PATCH 05/35] Add's libjpeg-turbo and libpng as third party submodules --- .gitmodules | 6 ++++++ third_party/libjpeg-turbo | 1 + third_party/libpng | 1 + 3 files changed, 8 insertions(+) create mode 100644 .gitmodules create mode 160000 third_party/libjpeg-turbo create mode 160000 third_party/libpng diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 00000000000..62db8eb762b --- /dev/null +++ b/.gitmodules @@ -0,0 +1,6 @@ +[submodule "third_party/libpng"] + path = third_party/libpng + url = https://github.com/glennrp/libpng +[submodule "third_party/libjpeg-turbo"] + path = third_party/libjpeg-turbo + url = https://github.com/libjpeg-turbo/libjpeg-turbo diff --git a/third_party/libjpeg-turbo b/third_party/libjpeg-turbo new file mode 160000 index 00000000000..f81833aed68 --- /dev/null +++ b/third_party/libjpeg-turbo @@ -0,0 +1 @@ +Subproject commit f81833aed68ec7d04ecdeb7f4f7566040f24300e diff --git a/third_party/libpng b/third_party/libpng new file mode 160000 index 00000000000..301f7a14295 --- /dev/null +++ b/third_party/libpng @@ -0,0 +1 @@ +Subproject commit 301f7a14295a3bdfaf406dbb5004d0784dc137ea From ca2df6ec5292134ec042f42bb214fe4556ce7fa0 Mon Sep 17 00:00:00 2001 From: Ryad ZENINE Date: Sun, 9 Feb 2020 16:55:07 +0100 Subject: [PATCH 06/35] Updates setup.py --- setup.py | 61 ++++++++++++++++++++++++++++++-------------------------- 1 file changed, 33 insertions(+), 28 deletions(-) diff --git a/setup.py b/setup.py index 548ac773082..e7c30776e0a 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,6 @@ import torch from torch.utils.cpp_extension import BuildExtension, CppExtension, CUDAExtension, CUDA_HOME - def read(*names, **kwargs): with io.open( os.path.join(os.path.dirname(__file__), *names), @@ -76,6 +75,11 @@ def write_version_file(): pillow_req = 'pillow-simd' if get_dist('pillow-simd') is not None else 'pillow' requirements.append(pillow_req + pillow_ver) +third_party_libraries = ['png', 'turbojpeg'] +third_party_dir = os.path.join(cwd, "third_party") +third_party_lib_directories = ['libpng', 'libjpeg-turbo'] +third_party_search_directories = [os.path.join(third_party_dir, directory) for directory in third_party_lib_directories] + def get_extensions(): this_dir = os.path.dirname(os.path.abspath(__file__)) @@ -143,9 +147,10 @@ def get_extensions(): extension( 'torchvision._C', sources, - libraries=['png', 'turbojpeg'], - include_dirs=include_dirs, - define_macros=define_macros, + #libraries= ['turbojpeg', 'png'], + include_dirs=include_dirs + third_party_search_directories, + #library_dirs=third_party_search_directories, + #define_macros=define_macros, extra_compile_args=extra_compile_args, ) ] @@ -200,27 +205,27 @@ def run(self): setup( - # Metadata - name=package_name, - version=version, - author='PyTorch Core Team', - author_email='soumith@pytorch.org', - url='https://github.com/pytorch/vision', - description='image and video datasets and models for torch deep learning', - long_description=readme, - license='BSD', - - # Package info - packages=find_packages(exclude=('test',)), - - zip_safe=False, - install_requires=requirements, - extras_require={ - "scipy": ["scipy"], - }, - ext_modules=get_extensions(), - cmdclass={ - 'build_ext': BuildExtension.with_options(no_python_abi_suffix=True), - 'clean': clean, - } -) + # Metadata + name=package_name, + version=version, + author='PyTorch Core Team', + author_email='soumith@pytorch.org', + url='https://github.com/pytorch/vision', + description='image and video datasets and models for torch deep learning', + long_description=readme, + license='BSD', + + # Package info + packages=find_packages(exclude=('test',)), + + zip_safe=False, + install_requires=requirements, + extras_require={ + "scipy": ["scipy"], + }, + ext_modules=get_extensions(), + cmdclass={ + 'build_ext': BuildExtension.with_options(no_python_abi_suffix=True), + 'clean': clean, + } + ) From be38d3e0bba9d1a213a86a924a7dcad28338e94a Mon Sep 17 00:00:00 2001 From: Ryad ZENINE Date: Sun, 9 Feb 2020 17:25:20 +0100 Subject: [PATCH 07/35] update CI to initialize submodules --- .circleci/config.yml | 4 ++++ .circleci/config.yml.in | 4 ++++ CMakeLists.txt | 5 +++-- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4986e9a6687..b71add5b02b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -21,6 +21,10 @@ commands: description: "checkout merge branch" steps: - checkout + - run: + name: init submodules + command: | + git submodule update --init --recursive # - run: # name: Checkout merge branch # command: | diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index 7a14720dbff..86cef43ac5f 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -21,6 +21,10 @@ commands: description: "checkout merge branch" steps: - checkout + - run: + name: init submodules + command: | + git submodule update --init --recursive # - run: # name: Checkout merge branch # command: | diff --git a/CMakeLists.txt b/CMakeLists.txt index a8c0571161b..0c4ec30d900 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -10,8 +10,8 @@ if(WITH_CUDA) add_definitions(-D__CUDA_NO_HALF_OPERATORS__) endif() -find_package(PNG REQUIRED) -add_library(LibJPEGTurbo SHARED IMPORTED) +include(${CMAKE_CURRENT_SOURCE_DIR}/third_party/libpng/CMakeLists.txt) +include(${CMAKE_CURRENT_SOURCE_DIR}/third_party/libjpeg-turbo/CMakeLists.txt) find_package(Torch REQUIRED) find_package(pybind11 REQUIRED) @@ -27,6 +27,7 @@ file(GLOB IMAGE_HEADERS torchvision/csrc/image.h) file(GLOB IMAGE_SOURCES torchvision/csrc/cpu/image/*.h torchvision/csrc/cpu/image/*.cpp) add_library(${PROJECT_NAME} SHARED ${MODELS_SOURCES} ${IMAGE_SOURCES}) + target_link_libraries(${PROJECT_NAME} PUBLIC "${PNG_LIBRARY}") target_link_libraries(${PROJECT_NAME} PUBLIC "${LibJPEGTurbo}") target_link_libraries(${PROJECT_NAME} PUBLIC "${TORCH_LIBRARIES}") From f813ba9e2b60d8350b7640c45955d0997be8cb7f Mon Sep 17 00:00:00 2001 From: Ryad ZENINE Date: Sun, 9 Feb 2020 20:55:36 +0100 Subject: [PATCH 08/35] Change Setup.py to compile third_party before_hand --- .circleci/config.yml | 11 +++-------- .circleci/config.yml.in | 11 +++-------- CMakeLists.txt | 4 ++-- packaging/build_conda.sh | 2 -- packaging/build_wheel.sh | 1 - packaging/build_wheel_macos.sh | 15 --------------- packaging/torchvision/meta.yaml | 2 -- packaging/wheel/linux_manywheel.sh | 1 - setup.py | 22 ++++++++++++++++++---- 9 files changed, 26 insertions(+), 43 deletions(-) delete mode 100755 packaging/build_wheel_macos.sh diff --git a/.circleci/config.yml b/.circleci/config.yml index b71add5b02b..bc40d167d5a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -21,10 +21,7 @@ commands: description: "checkout merge branch" steps: - checkout - - run: - name: init submodules - command: | - git submodule update --init --recursive + - run: git submodule update --init --recursive # - run: # name: Checkout merge branch # command: | @@ -223,8 +220,7 @@ jobs: curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh sh conda.sh -b source $HOME/miniconda3/bin/activate - conda install -yq libpng - packaging/build_wheel_macos.sh + packaging/build_wheel.sh - store_artifacts: path: dist - persist_to_workspace: @@ -244,9 +240,8 @@ jobs: sh conda.sh -b source $HOME/miniconda3/bin/activate conda install -yq conda-build - conda install -yq libpng - packaging/build_conda.sh - store_artifacts: + packaging/build_conda.sh path: /Users/distiller/miniconda3/conda-bld/osx-64 - persist_to_workspace: root: /Users/distiller/miniconda3/conda-bld/osx-64 diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index 86cef43ac5f..15190d4b4fa 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -21,10 +21,7 @@ commands: description: "checkout merge branch" steps: - checkout - - run: - name: init submodules - command: | - git submodule update --init --recursive + - run: git submodule update --init --recursive # - run: # name: Checkout merge branch # command: | @@ -223,8 +220,7 @@ jobs: curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh sh conda.sh -b source $HOME/miniconda3/bin/activate - conda install -yq libpng - packaging/build_wheel_macos.sh + packaging/build_wheel.sh - store_artifacts: path: dist - persist_to_workspace: @@ -244,9 +240,8 @@ jobs: sh conda.sh -b source $HOME/miniconda3/bin/activate conda install -yq conda-build - conda install -yq libpng - packaging/build_conda.sh - store_artifacts: + packaging/build_conda.sh path: /Users/distiller/miniconda3/conda-bld/osx-64 - persist_to_workspace: root: /Users/distiller/miniconda3/conda-bld/osx-64 diff --git a/CMakeLists.txt b/CMakeLists.txt index 0c4ec30d900..464f6676751 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -10,8 +10,8 @@ if(WITH_CUDA) add_definitions(-D__CUDA_NO_HALF_OPERATORS__) endif() -include(${CMAKE_CURRENT_SOURCE_DIR}/third_party/libpng/CMakeLists.txt) -include(${CMAKE_CURRENT_SOURCE_DIR}/third_party/libjpeg-turbo/CMakeLists.txt) +add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/third_party/libpng) +add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/third_party/libjpeg-turbo) find_package(Torch REQUIRED) find_package(pybind11 REQUIRED) diff --git a/packaging/build_conda.sh b/packaging/build_conda.sh index 8b518c5804d..61022f3c5fa 100755 --- a/packaging/build_conda.sh +++ b/packaging/build_conda.sh @@ -10,6 +10,4 @@ export SOURCE_ROOT_DIR="$PWD" setup_conda_pytorch_constraint setup_conda_cudatoolkit_constraint setup_visual_studio_constraint -conda install -yq libpng -conda install -c conda-forge libjpeg-turbo conda build $CONDA_CHANNEL_FLAGS -c defaults -c conda-forge --no-anaconda-upload --python "$PYTHON_VERSION" packaging/torchvision diff --git a/packaging/build_wheel.sh b/packaging/build_wheel.sh index f7a49fc35a6..5d073d9c104 100755 --- a/packaging/build_wheel.sh +++ b/packaging/build_wheel.sh @@ -7,7 +7,6 @@ script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" export BUILD_TYPE=wheel setup_env 0.6.0 setup_wheel_python -yum install -y libpng libpng-devel turbojpeg turbojpeg-devel pip_install numpy pyyaml future ninja setup_pip_pytorch_version python setup.py clean diff --git a/packaging/build_wheel_macos.sh b/packaging/build_wheel_macos.sh deleted file mode 100755 index 7d37239563d..00000000000 --- a/packaging/build_wheel_macos.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash -set -ex - -script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -. "$script_dir/pkg_helpers.bash" - -export BUILD_TYPE=wheel -setup_env 0.5.0 -setup_wheel_python -pip_install numpy pyyaml future ninja -# TODO remove after https://github.com/pytorch/pytorch/pull/27282 gets merged -pip_install six -setup_pip_pytorch_version -python setup.py clean -IS_WHEEL=1 python setup.py bdist_wheel diff --git a/packaging/torchvision/meta.yaml b/packaging/torchvision/meta.yaml index c40df3f5b2c..1bc199e437b 100644 --- a/packaging/torchvision/meta.yaml +++ b/packaging/torchvision/meta.yaml @@ -12,7 +12,6 @@ requirements: host: - python - setuptools - - libpng >=1.6.37 {{ environ.get('CONDA_PYTORCH_BUILD_CONSTRAINT') }} {{ environ.get('CONDA_CUDATOOLKIT_CONSTRAINT') }} {{ environ.get('CONDA_CPUONLY_FEATURE') }} @@ -21,7 +20,6 @@ requirements: - python - pillow >=4.1.1 - numpy >=1.11 - - libpng >=1.6.37 - six {{ environ.get('CONDA_PYTORCH_CONSTRAINT') }} {{ environ.get('CONDA_CUDATOOLKIT_CONSTRAINT') }} diff --git a/packaging/wheel/linux_manywheel.sh b/packaging/wheel/linux_manywheel.sh index a3bb22d7f88..435873769e5 100644 --- a/packaging/wheel/linux_manywheel.sh +++ b/packaging/wheel/linux_manywheel.sh @@ -32,7 +32,6 @@ rm -rf vision git clone https://github.com/pytorch/vision cd /tmp/vision -yum install -y libpng libpng-devel for PYDIR in "${python_installations[@]}"; do export PATH=$PYDIR/bin:$OLD_PATH pip install --upgrade pip diff --git a/setup.py b/setup.py index e7c30776e0a..4fefed006b8 100644 --- a/setup.py +++ b/setup.py @@ -80,6 +80,17 @@ def write_version_file(): third_party_lib_directories = ['libpng', 'libjpeg-turbo'] third_party_search_directories = [os.path.join(third_party_dir, directory) for directory in third_party_lib_directories] +def build_dependencies(): + for directory in third_party_search_directories: + os.chdir(directory) + if sys.platform == 'win32': + os.system("cmake3.exe --clean .") + os.system("cmake3.exe --build .") + else: + os.system("cmake --clean .") + os.system("cmake --build .") + os.chdir(cwd) + def get_extensions(): this_dir = os.path.dirname(os.path.abspath(__file__)) @@ -147,10 +158,10 @@ def get_extensions(): extension( 'torchvision._C', sources, - #libraries= ['turbojpeg', 'png'], + libraries= third_party_libraries, include_dirs=include_dirs + third_party_search_directories, - #library_dirs=third_party_search_directories, - #define_macros=define_macros, + library_dirs=third_party_search_directories, + define_macros=define_macros, extra_compile_args=extra_compile_args, ) ] @@ -203,6 +214,9 @@ def run(self): # It's an old-style class in Python 2.7... distutils.command.clean.clean.run(self) +def build_ext_with_dependencies(self): + build_dependencies() + return BuildExtension.with_options(no_python_abi_suffix=True)(self) setup( # Metadata @@ -225,7 +239,7 @@ def run(self): }, ext_modules=get_extensions(), cmdclass={ - 'build_ext': BuildExtension.with_options(no_python_abi_suffix=True), + 'build_ext': build_ext_with_dependencies, 'clean': clean, } ) From 7b2a48e97efbf854cfe782a883755c1835d7bd7d Mon Sep 17 00:00:00 2001 From: Ryad ZENINE Date: Sun, 9 Feb 2020 20:58:53 +0100 Subject: [PATCH 09/35] fix type --- .circleci/config.yml | 2 +- .circleci/config.yml.in | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index bc40d167d5a..2f70a615228 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -240,8 +240,8 @@ jobs: sh conda.sh -b source $HOME/miniconda3/bin/activate conda install -yq conda-build - - store_artifacts: packaging/build_conda.sh + - store_artifacts: path: /Users/distiller/miniconda3/conda-bld/osx-64 - persist_to_workspace: root: /Users/distiller/miniconda3/conda-bld/osx-64 diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index 15190d4b4fa..c51d8981b62 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -240,8 +240,8 @@ jobs: sh conda.sh -b source $HOME/miniconda3/bin/activate conda install -yq conda-build - - store_artifacts: packaging/build_conda.sh + - store_artifacts: path: /Users/distiller/miniconda3/conda-bld/osx-64 - persist_to_workspace: root: /Users/distiller/miniconda3/conda-bld/osx-64 From 399fce3652db092e51b2c68e73fed737e90896b4 Mon Sep 17 00:00:00 2001 From: Ryad ZENINE Date: Sun, 9 Feb 2020 21:22:19 +0100 Subject: [PATCH 10/35] add's zlib as a submodule to build libpng --- .gitmodules | 3 +++ third_party/zlib | 1 + 2 files changed, 4 insertions(+) create mode 160000 third_party/zlib diff --git a/.gitmodules b/.gitmodules index 62db8eb762b..444a210658e 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,3 +4,6 @@ [submodule "third_party/libjpeg-turbo"] path = third_party/libjpeg-turbo url = https://github.com/libjpeg-turbo/libjpeg-turbo +[submodule "third_party/zlib"] + path = third_party/zlib + url = https://github.com/madler/zlib diff --git a/third_party/zlib b/third_party/zlib new file mode 160000 index 00000000000..cacf7f1d4e3 --- /dev/null +++ b/third_party/zlib @@ -0,0 +1 @@ +Subproject commit cacf7f1d4e3d44d871b605da3b647f07d718623f From f47d33efa9c5d65d52b3b40bbd417967b0da6943 Mon Sep 17 00:00:00 2001 From: Ryad ZENINE Date: Sun, 9 Feb 2020 21:22:44 +0100 Subject: [PATCH 11/35] updates setup.py to build zlib with libpn --- setup.py | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/setup.py b/setup.py index 4fefed006b8..0ea0e2cfd74 100644 --- a/setup.py +++ b/setup.py @@ -80,16 +80,21 @@ def write_version_file(): third_party_lib_directories = ['libpng', 'libjpeg-turbo'] third_party_search_directories = [os.path.join(third_party_dir, directory) for directory in third_party_lib_directories] +def _build_cmake_dependency(directory, args=""): + os.chdir(directory) + if sys.platform == 'win32': + os.system("cmake3.exe --clean .") + os.system("cmake3.exe {} --build .".format(args) ) + else: + os.system("cmake --clean .") + os.system("cmake {} --build .".format(args) ) + os.chdir(cwd) + + def build_dependencies(): - for directory in third_party_search_directories: - os.chdir(directory) - if sys.platform == 'win32': - os.system("cmake3.exe --clean .") - os.system("cmake3.exe --build .") - else: - os.system("cmake --clean .") - os.system("cmake --build .") - os.chdir(cwd) + zlib_dep = os.path.join(third_party_dir, "zlib") + _build_cmake_dependency("./third_party/libpng", "-DPNG_BUILD_ZLIB={}".format(zlib_dep)) + _build_cmake_dependency("./third_party/libjpeg-turbo") def get_extensions(): From 566c88c085fa5035d4ff6882ebd5662c6781122a Mon Sep 17 00:00:00 2001 From: Ryad ZENINE Date: Mon, 10 Feb 2020 07:22:12 +0100 Subject: [PATCH 12/35] Fix setup.py to compile zlib and libpng correctly --- setup.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index 0ea0e2cfd74..8a39ef465d1 100644 --- a/setup.py +++ b/setup.py @@ -84,16 +84,21 @@ def _build_cmake_dependency(directory, args=""): os.chdir(directory) if sys.platform == 'win32': os.system("cmake3.exe --clean .") - os.system("cmake3.exe {} --build .".format(args) ) + os.system("cmake3.exe -G='Visual Studio 15 2017' {} .".format(args) ) + os.system("cmake3.exe --build .") else: os.system("cmake --clean .") - os.system("cmake {} --build .".format(args) ) + os.system("cmake {} .".format(args) ) + os.system("cmake --build .") os.chdir(cwd) def build_dependencies(): - zlib_dep = os.path.join(third_party_dir, "zlib") - _build_cmake_dependency("./third_party/libpng", "-DPNG_BUILD_ZLIB={}".format(zlib_dep)) + zlib_path = os.path.join(third_party_dir, "zlib") + libpng_cmake_options="-DPNG_BUILD_ZLIB=ON -DZLIB_INCLUDE_DIR:PATH={zlib_path} -DZLIB_LIBRARY:FILEPATH={zlib_path}/libz.a".format(zlib_path=zlib_path) + + _build_cmake_dependency("./third_party/zlib") + _build_cmake_dependency("./third_party/libpng", libpng_cmake_options) _build_cmake_dependency("./third_party/libjpeg-turbo") From 2daae2c3ced2b6bfb9ca347b89abac5f16bf9801 Mon Sep 17 00:00:00 2001 From: Ryad ZENINE Date: Mon, 10 Feb 2020 07:32:03 +0100 Subject: [PATCH 13/35] update zlib build options --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 8a39ef465d1..0be4afcbe2b 100644 --- a/setup.py +++ b/setup.py @@ -95,8 +95,8 @@ def _build_cmake_dependency(directory, args=""): def build_dependencies(): zlib_path = os.path.join(third_party_dir, "zlib") - libpng_cmake_options="-DPNG_BUILD_ZLIB=ON -DZLIB_INCLUDE_DIR:PATH={zlib_path} -DZLIB_LIBRARY:FILEPATH={zlib_path}/libz.a".format(zlib_path=zlib_path) - + libpng_cmake_options="-DPNG_BUILD_ZLIB=ON -DZLIB_INCLUDE_DIR:PATH={zlib_path} -DZLIB_LIBRARY:FILEPATH={zlib_path}/libz.so".format(zlib_path=zlib_path) + print("Zlib build options:", libpng_cmake_options) _build_cmake_dependency("./third_party/zlib") _build_cmake_dependency("./third_party/libpng", libpng_cmake_options) _build_cmake_dependency("./third_party/libjpeg-turbo") From 36371af97c6ede9ccb5df0712c9709103b04f436 Mon Sep 17 00:00:00 2001 From: Ryad ZENINE Date: Mon, 10 Feb 2020 07:37:01 +0100 Subject: [PATCH 14/35] Add's cmake to makeos and windows builds --- .circleci/config.yml | 4 ++++ .circleci/config.yml.in | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2f70a615228..8f89debba5c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -189,6 +189,7 @@ jobs: (& "C:\tools\miniconda3\Scripts\conda.exe" "shell.powershell" "hook") | Out-String | Invoke-Expression conda activate base conda install -yq conda-build "conda-package-handling!=1.5.0" + conda install -c anaconda cmake bash packaging/build_conda.sh shell: powershell.exe @@ -203,6 +204,7 @@ jobs: (& "C:\tools\miniconda3\Scripts\conda.exe" "shell.powershell" "hook") | Out-String | Invoke-Expression conda activate base conda install -yq conda-build "conda-package-handling!=1.5.0" + conda install -c anaconda cmake bash packaging/build_conda.sh shell: powershell.exe @@ -220,6 +222,7 @@ jobs: curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh sh conda.sh -b source $HOME/miniconda3/bin/activate + conda install -c anaconda cmake packaging/build_wheel.sh - store_artifacts: path: dist @@ -240,6 +243,7 @@ jobs: sh conda.sh -b source $HOME/miniconda3/bin/activate conda install -yq conda-build + conda install -c anaconda cmake packaging/build_conda.sh - store_artifacts: path: /Users/distiller/miniconda3/conda-bld/osx-64 diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index c51d8981b62..998b8efd8d9 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -189,6 +189,7 @@ jobs: (& "C:\tools\miniconda3\Scripts\conda.exe" "shell.powershell" "hook") | Out-String | Invoke-Expression conda activate base conda install -yq conda-build "conda-package-handling!=1.5.0" + conda install -c anaconda cmake bash packaging/build_conda.sh shell: powershell.exe @@ -203,6 +204,7 @@ jobs: (& "C:\tools\miniconda3\Scripts\conda.exe" "shell.powershell" "hook") | Out-String | Invoke-Expression conda activate base conda install -yq conda-build "conda-package-handling!=1.5.0" + conda install -c anaconda cmake bash packaging/build_conda.sh shell: powershell.exe @@ -220,6 +222,7 @@ jobs: curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh sh conda.sh -b source $HOME/miniconda3/bin/activate + conda install -c anaconda cmake packaging/build_wheel.sh - store_artifacts: path: dist @@ -240,6 +243,7 @@ jobs: sh conda.sh -b source $HOME/miniconda3/bin/activate conda install -yq conda-build + conda install -c anaconda cmake packaging/build_conda.sh - store_artifacts: path: /Users/distiller/miniconda3/conda-bld/osx-64 From 84a3b1b4f4d34d1f2fd3a16acd7a8f243683a90d Mon Sep 17 00:00:00 2001 From: Ryad ZENINE Date: Mon, 10 Feb 2020 07:54:16 +0100 Subject: [PATCH 15/35] Fix conda cmake install for macos pipeline --- .circleci/config.yml | 8 ++++---- .circleci/config.yml.in | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8f89debba5c..79f5f08416c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -189,7 +189,7 @@ jobs: (& "C:\tools\miniconda3\Scripts\conda.exe" "shell.powershell" "hook") | Out-String | Invoke-Expression conda activate base conda install -yq conda-build "conda-package-handling!=1.5.0" - conda install -c anaconda cmake + conda install -yq -c anaconda cmake bash packaging/build_conda.sh shell: powershell.exe @@ -204,7 +204,7 @@ jobs: (& "C:\tools\miniconda3\Scripts\conda.exe" "shell.powershell" "hook") | Out-String | Invoke-Expression conda activate base conda install -yq conda-build "conda-package-handling!=1.5.0" - conda install -c anaconda cmake + conda install -yq -c anaconda cmake bash packaging/build_conda.sh shell: powershell.exe @@ -222,7 +222,7 @@ jobs: curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh sh conda.sh -b source $HOME/miniconda3/bin/activate - conda install -c anaconda cmake + conda install -yq -c anaconda cmake packaging/build_wheel.sh - store_artifacts: path: dist @@ -243,7 +243,7 @@ jobs: sh conda.sh -b source $HOME/miniconda3/bin/activate conda install -yq conda-build - conda install -c anaconda cmake + conda install -yq -c anaconda cmake packaging/build_conda.sh - store_artifacts: path: /Users/distiller/miniconda3/conda-bld/osx-64 diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index 998b8efd8d9..2e6c73e4ea5 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -189,7 +189,7 @@ jobs: (& "C:\tools\miniconda3\Scripts\conda.exe" "shell.powershell" "hook") | Out-String | Invoke-Expression conda activate base conda install -yq conda-build "conda-package-handling!=1.5.0" - conda install -c anaconda cmake + conda install -yq -c anaconda cmake bash packaging/build_conda.sh shell: powershell.exe @@ -204,7 +204,7 @@ jobs: (& "C:\tools\miniconda3\Scripts\conda.exe" "shell.powershell" "hook") | Out-String | Invoke-Expression conda activate base conda install -yq conda-build "conda-package-handling!=1.5.0" - conda install -c anaconda cmake + conda install -yq -c anaconda cmake bash packaging/build_conda.sh shell: powershell.exe @@ -222,7 +222,7 @@ jobs: curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh sh conda.sh -b source $HOME/miniconda3/bin/activate - conda install -c anaconda cmake + conda install -yq -c anaconda cmake packaging/build_wheel.sh - store_artifacts: path: dist @@ -243,7 +243,7 @@ jobs: sh conda.sh -b source $HOME/miniconda3/bin/activate conda install -yq conda-build - conda install -c anaconda cmake + conda install -yq -c anaconda cmake packaging/build_conda.sh - store_artifacts: path: /Users/distiller/miniconda3/conda-bld/osx-64 From 9668086ccee27113b5eae7fbfc4377d039ccd075 Mon Sep 17 00:00:00 2001 From: Ryad ZENINE Date: Mon, 10 Feb 2020 08:06:46 +0100 Subject: [PATCH 16/35] Deactivating SIMD support for macOS (nasm failing) --- packaging/pkg_helpers.bash | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packaging/pkg_helpers.bash b/packaging/pkg_helpers.bash index 7fe47970359..07d4c5ea420 100644 --- a/packaging/pkg_helpers.bash +++ b/packaging/pkg_helpers.bash @@ -112,6 +112,8 @@ setup_build_version() { setup_macos() { if [[ "$(uname)" == Darwin ]]; then export MACOSX_DEPLOYMENT_TARGET=10.9 CC=clang CXX=clang++ + # Deactivation SIMD support for MacOS for jpegturbo as the nasm version provided with the CI is failing + export WITH_SIMD=0 fi } From 181353e642249d79102e3ec3794f0fb806e305e5 Mon Sep 17 00:00:00 2001 From: Ryad ZENINE Date: Mon, 10 Feb 2020 08:13:01 +0100 Subject: [PATCH 17/35] fix cmake path for mac_os_wheel pipeline --- packaging/pkg_helpers.bash | 1 + 1 file changed, 1 insertion(+) diff --git a/packaging/pkg_helpers.bash b/packaging/pkg_helpers.bash index 07d4c5ea420..019f87ad242 100644 --- a/packaging/pkg_helpers.bash +++ b/packaging/pkg_helpers.bash @@ -105,6 +105,7 @@ setup_build_version() { export BUILD_VERSION="$1.dev$(date "+%Y%m%d")$VERSION_SUFFIX" else export BUILD_VERSION="$BUILD_VERSION$VERSION_SUFFIX" + export PATH="$PATH:/Users/distiller/miniconda3/bin" fi } From b56fdec46d28ca157dd488044332d7a09fd925cc Mon Sep 17 00:00:00 2001 From: Ryad ZENINE Date: Mon, 10 Feb 2020 08:19:47 +0100 Subject: [PATCH 18/35] Deactivates simd for jpegturbo for macos --- setup.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 0be4afcbe2b..8d81b6d583e 100644 --- a/setup.py +++ b/setup.py @@ -96,10 +96,13 @@ def _build_cmake_dependency(directory, args=""): def build_dependencies(): zlib_path = os.path.join(third_party_dir, "zlib") libpng_cmake_options="-DPNG_BUILD_ZLIB=ON -DZLIB_INCLUDE_DIR:PATH={zlib_path} -DZLIB_LIBRARY:FILEPATH={zlib_path}/libz.so".format(zlib_path=zlib_path) - print("Zlib build options:", libpng_cmake_options) _build_cmake_dependency("./third_party/zlib") + # Deactivate SIMD on macOS + jpeg_turbo_options = "" + if sys.platform == "darwin": + jpeg_turbo_options ="-DWITH_SIMD=0" _build_cmake_dependency("./third_party/libpng", libpng_cmake_options) - _build_cmake_dependency("./third_party/libjpeg-turbo") + _build_cmake_dependency("./third_party/libjpeg-turbo", jpeg_turbo_options) def get_extensions(): From 6f41c5b66171ef53017bc0b9fe87e280706658c2 Mon Sep 17 00:00:00 2001 From: Ryad ZENINE Date: Tue, 11 Feb 2020 06:36:13 +0100 Subject: [PATCH 19/35] Disable ninja for CI --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 8d81b6d583e..a0483b71aba 100644 --- a/setup.py +++ b/setup.py @@ -229,7 +229,7 @@ def run(self): def build_ext_with_dependencies(self): build_dependencies() - return BuildExtension.with_options(no_python_abi_suffix=True)(self) + return BuildExtension.with_options(no_python_abi_suffix=True, use_ninja=False )(self) setup( # Metadata From 27fb720130033f10eb492a2446a81e8ba0b29047 Mon Sep 17 00:00:00 2001 From: Ryad ZENINE Date: Tue, 11 Feb 2020 06:38:19 +0100 Subject: [PATCH 20/35] Installing cmake using choco --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 79f5f08416c..71e0c169af6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -186,10 +186,10 @@ jobs: - run: command: | choco install miniconda3 + choco install cmake (& "C:\tools\miniconda3\Scripts\conda.exe" "shell.powershell" "hook") | Out-String | Invoke-Expression conda activate base conda install -yq conda-build "conda-package-handling!=1.5.0" - conda install -yq -c anaconda cmake bash packaging/build_conda.sh shell: powershell.exe From b5c8274977cd44ab604f3e9adaf2600b292c9e84 Mon Sep 17 00:00:00 2001 From: Ryad ZENINE Date: Tue, 11 Feb 2020 06:54:15 +0100 Subject: [PATCH 21/35] Updates to fix windows CI. --- .circleci/config.yml | 6 +++++- .circleci/config.yml.in | 4 ++++ setup.py | 8 +++----- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 71e0c169af6..e044aa089c3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -185,11 +185,13 @@ jobs: - checkout_merge - run: command: | + choco install cmake --installargs 'ADD_CMAKE_TO_PATH=System' + RefreshEnv.cmd choco install miniconda3 - choco install cmake (& "C:\tools\miniconda3\Scripts\conda.exe" "shell.powershell" "hook") | Out-String | Invoke-Expression conda activate base conda install -yq conda-build "conda-package-handling!=1.5.0" + conda install -yq -c anaconda cmake bash packaging/build_conda.sh shell: powershell.exe @@ -200,6 +202,8 @@ jobs: - checkout_merge - run: command: | + choco install cmake --installargs 'ADD_CMAKE_TO_PATH=System' + RefreshEnv.cmd choco install miniconda3 (& "C:\tools\miniconda3\Scripts\conda.exe" "shell.powershell" "hook") | Out-String | Invoke-Expression conda activate base diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index 2e6c73e4ea5..4fe28fb11e9 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -185,6 +185,8 @@ jobs: - checkout_merge - run: command: | + choco install cmake --installargs 'ADD_CMAKE_TO_PATH=System' + RefreshEnv.cmd choco install miniconda3 (& "C:\tools\miniconda3\Scripts\conda.exe" "shell.powershell" "hook") | Out-String | Invoke-Expression conda activate base @@ -200,6 +202,8 @@ jobs: - checkout_merge - run: command: | + choco install cmake --installargs 'ADD_CMAKE_TO_PATH=System' + RefreshEnv.cmd choco install miniconda3 (& "C:\tools\miniconda3\Scripts\conda.exe" "shell.powershell" "hook") | Out-String | Invoke-Expression conda activate base diff --git a/setup.py b/setup.py index a0483b71aba..2e4049c84f0 100644 --- a/setup.py +++ b/setup.py @@ -83,13 +83,11 @@ def write_version_file(): def _build_cmake_dependency(directory, args=""): os.chdir(directory) if sys.platform == 'win32': - os.system("cmake3.exe --clean .") - os.system("cmake3.exe -G='Visual Studio 15 2017' {} .".format(args) ) - os.system("cmake3.exe --build .") + os.system("cmake.exe --clean .") + os.system("cmake.exe --build --test .") else: os.system("cmake --clean .") - os.system("cmake {} .".format(args) ) - os.system("cmake --build .") + os.system("cmake --build --test .") os.chdir(cwd) From 9bb2d6381381311ece6db0d13b7098f4ad7220f4 Mon Sep 17 00:00:00 2001 From: Ryad ZENINE Date: Tue, 11 Feb 2020 06:58:28 +0100 Subject: [PATCH 22/35] fix setup.py cmake --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 2e4049c84f0..7399344c25b 100644 --- a/setup.py +++ b/setup.py @@ -84,10 +84,10 @@ def _build_cmake_dependency(directory, args=""): os.chdir(directory) if sys.platform == 'win32': os.system("cmake.exe --clean .") - os.system("cmake.exe --build --test .") + os.system("cmake.exe --build .") else: os.system("cmake --clean .") - os.system("cmake --build --test .") + os.system("cmake --build .") os.chdir(cwd) From 2e63e4705402d8f395d20471aa65f6bd168384b7 Mon Sep 17 00:00:00 2001 From: Ryad ZENINE Date: Tue, 11 Feb 2020 07:07:44 +0100 Subject: [PATCH 23/35] fix setup.py --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 7399344c25b..193aa3945a8 100644 --- a/setup.py +++ b/setup.py @@ -84,10 +84,10 @@ def _build_cmake_dependency(directory, args=""): os.chdir(directory) if sys.platform == 'win32': os.system("cmake.exe --clean .") - os.system("cmake.exe --build .") + os.system("cmake.exe {} --build .") else: os.system("cmake --clean .") - os.system("cmake --build .") + os.system("cmake {} --build .") os.chdir(cwd) From 5019f7bf17f6275764d44b5131f4c53b0eeff647 Mon Sep 17 00:00:00 2001 From: Ryad ZENINE Date: Tue, 11 Feb 2020 07:09:50 +0100 Subject: [PATCH 24/35] Use vs2019 as a generator for cmake --- setup.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 193aa3945a8..5c3817f1cbe 100644 --- a/setup.py +++ b/setup.py @@ -84,10 +84,12 @@ def _build_cmake_dependency(directory, args=""): os.chdir(directory) if sys.platform == 'win32': os.system("cmake.exe --clean .") - os.system("cmake.exe {} --build .") + os.system("cmake.exe -G='Visual Studio 16 2019' {} .".format(args) ) + os.system("cmake.exe --build .") else: os.system("cmake --clean .") - os.system("cmake {} --build .") + os.system("cmake {} . ".format(args)) + os.system("cmake --build .") os.chdir(cwd) From 46d25d9087c2d5ae8c0951f1bccce0e6697c7fd1 Mon Sep 17 00:00:00 2001 From: Ryad ZENINE Date: Tue, 11 Feb 2020 07:21:35 +0100 Subject: [PATCH 25/35] Fix quotes for windows --- setup.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 5c3817f1cbe..bff5c889db3 100644 --- a/setup.py +++ b/setup.py @@ -84,7 +84,7 @@ def _build_cmake_dependency(directory, args=""): os.chdir(directory) if sys.platform == 'win32': os.system("cmake.exe --clean .") - os.system("cmake.exe -G='Visual Studio 16 2019' {} .".format(args) ) + os.system('cmake.exe -G="Visual Studio 16 2019" {} .'.format(args) ) os.system("cmake.exe --build .") else: os.system("cmake --clean .") @@ -95,7 +95,12 @@ def _build_cmake_dependency(directory, args=""): def build_dependencies(): zlib_path = os.path.join(third_party_dir, "zlib") - libpng_cmake_options="-DPNG_BUILD_ZLIB=ON -DZLIB_INCLUDE_DIR:PATH={zlib_path} -DZLIB_LIBRARY:FILEPATH={zlib_path}/libz.so".format(zlib_path=zlib_path) + if sys.platform == "win32": + libpng_cmake_options="-DPNG_BUILD_ZLIB=ON -DZLIB_INCLUDE_DIR:PATH={zlib_path} -DZLIB_LIBRARY:FILEPATH={zlib_path}/libz.lib".format(zlib_path=zlib_path) + elif sys.platform == "darwin": + libpng_cmake_options="-DPNG_BUILD_ZLIB=ON -DZLIB_INCLUDE_DIR:PATH={zlib_path} -DZLIB_LIBRARY:FILEPATH={zlib_path}/libz.dylib".format(zlib_path=zlib_path) + else : + libpng_cmake_options="-DPNG_BUILD_ZLIB=ON -DZLIB_INCLUDE_DIR:PATH={zlib_path} -DZLIB_LIBRARY:FILEPATH={zlib_path}/libz.so".format(zlib_path=zlib_path) _build_cmake_dependency("./third_party/zlib") # Deactivate SIMD on macOS jpeg_turbo_options = "" From d4af63a30d8c4a1017ce3719a4f9b9eaa0d7dd1f Mon Sep 17 00:00:00 2001 From: Ryad ZENINE Date: Tue, 11 Feb 2020 08:20:45 +0100 Subject: [PATCH 26/35] Fix type for cmake in windows --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index bff5c889db3..50d2e62809a 100644 --- a/setup.py +++ b/setup.py @@ -84,7 +84,7 @@ def _build_cmake_dependency(directory, args=""): os.chdir(directory) if sys.platform == 'win32': os.system("cmake.exe --clean .") - os.system('cmake.exe -G="Visual Studio 16 2019" {} .'.format(args) ) + os.system('cmake.exe -G"Visual Studio 16 2019" {} .'.format(args) ) os.system("cmake.exe --build .") else: os.system("cmake --clean .") From e5f6a06643771983fc9f41b813cb223696021be4 Mon Sep 17 00:00:00 2001 From: Ryad ZENINE Date: Tue, 11 Feb 2020 08:26:05 +0100 Subject: [PATCH 27/35] Fix environment variable for mac build --- packaging/pkg_helpers.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging/pkg_helpers.bash b/packaging/pkg_helpers.bash index 019f87ad242..25e95cacdc6 100644 --- a/packaging/pkg_helpers.bash +++ b/packaging/pkg_helpers.bash @@ -105,8 +105,8 @@ setup_build_version() { export BUILD_VERSION="$1.dev$(date "+%Y%m%d")$VERSION_SUFFIX" else export BUILD_VERSION="$BUILD_VERSION$VERSION_SUFFIX" - export PATH="$PATH:/Users/distiller/miniconda3/bin" fi + export PATH="$PATH:/Users/distiller/miniconda3/bin" } # Set some useful variables for OS X, if applicable From ee6b1828aa4a54a270b695e8b8274793c679c758 Mon Sep 17 00:00:00 2001 From: Ryad ZENINE Date: Tue, 11 Feb 2020 08:40:04 +0100 Subject: [PATCH 28/35] Fix windown libpng build --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 50d2e62809a..5f2511cddad 100644 --- a/setup.py +++ b/setup.py @@ -96,7 +96,7 @@ def _build_cmake_dependency(directory, args=""): def build_dependencies(): zlib_path = os.path.join(third_party_dir, "zlib") if sys.platform == "win32": - libpng_cmake_options="-DPNG_BUILD_ZLIB=ON -DZLIB_INCLUDE_DIR:PATH={zlib_path} -DZLIB_LIBRARY:FILEPATH={zlib_path}/libz.lib".format(zlib_path=zlib_path) + libpng_cmake_options="-DPNG_BUILD_ZLIB=ON -DZLIB_INCLUDE_DIR:PATH={zlib_path} -DZLIB_LIBRARY:FILEPATH={zlib_path}/libz.dll".format(zlib_path=zlib_path) elif sys.platform == "darwin": libpng_cmake_options="-DPNG_BUILD_ZLIB=ON -DZLIB_INCLUDE_DIR:PATH={zlib_path} -DZLIB_LIBRARY:FILEPATH={zlib_path}/libz.dylib".format(zlib_path=zlib_path) else : From 0993c1d711f6ad90d3cc02e5f1cf5a7a9e795f7b Mon Sep 17 00:00:00 2001 From: Ryad ZENINE Date: Wed, 12 Feb 2020 06:49:17 +0100 Subject: [PATCH 29/35] linking statically against zlib --- setup.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 5f2511cddad..65db309b674 100644 --- a/setup.py +++ b/setup.py @@ -96,11 +96,11 @@ def _build_cmake_dependency(directory, args=""): def build_dependencies(): zlib_path = os.path.join(third_party_dir, "zlib") if sys.platform == "win32": - libpng_cmake_options="-DPNG_BUILD_ZLIB=ON -DZLIB_INCLUDE_DIR:PATH={zlib_path} -DZLIB_LIBRARY:FILEPATH={zlib_path}/libz.dll".format(zlib_path=zlib_path) + libpng_cmake_options="-DPNG_BUILD_ZLIB=ON -DZLIB_INCLUDE_DIR:PATH={zlib_path} -DZLIB_LIBRARY:FILEPATH={zlib_path}/libz.lib".format(zlib_path=zlib_path) elif sys.platform == "darwin": - libpng_cmake_options="-DPNG_BUILD_ZLIB=ON -DZLIB_INCLUDE_DIR:PATH={zlib_path} -DZLIB_LIBRARY:FILEPATH={zlib_path}/libz.dylib".format(zlib_path=zlib_path) + libpng_cmake_options="-DPNG_BUILD_ZLIB=ON -DZLIB_INCLUDE_DIR:PATH={zlib_path} -DZLIB_LIBRARY:FILEPATH={zlib_path}/libz.a".format(zlib_path=zlib_path) else : - libpng_cmake_options="-DPNG_BUILD_ZLIB=ON -DZLIB_INCLUDE_DIR:PATH={zlib_path} -DZLIB_LIBRARY:FILEPATH={zlib_path}/libz.so".format(zlib_path=zlib_path) + libpng_cmake_options="-DPNG_BUILD_ZLIB=ON -DZLIB_INCLUDE_DIR:PATH={zlib_path} -DZLIB_LIBRARY:FILEPATH={zlib_path}/libz.a".format(zlib_path=zlib_path) _build_cmake_dependency("./third_party/zlib") # Deactivate SIMD on macOS jpeg_turbo_options = "" @@ -181,6 +181,7 @@ def get_extensions(): library_dirs=third_party_search_directories, define_macros=define_macros, extra_compile_args=extra_compile_args, + extra_link_args={} ) ] if compile_cpp_tests: From 317ca1485003c96a23dc4c0a5363201282d6b07c Mon Sep 17 00:00:00 2001 From: Ryad ZENINE Date: Wed, 12 Feb 2020 06:58:29 +0100 Subject: [PATCH 30/35] compiling libpng with -fPIC on linux --- setup.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 65db309b674..850788479e7 100644 --- a/setup.py +++ b/setup.py @@ -96,11 +96,11 @@ def _build_cmake_dependency(directory, args=""): def build_dependencies(): zlib_path = os.path.join(third_party_dir, "zlib") if sys.platform == "win32": - libpng_cmake_options="-DPNG_BUILD_ZLIB=ON -DZLIB_INCLUDE_DIR:PATH={zlib_path} -DZLIB_LIBRARY:FILEPATH={zlib_path}/libz.lib".format(zlib_path=zlib_path) + libpng_cmake_options="-DPNG_BUILD_ZLIB=OFF -DZLIB_INCLUDE_DIR:PATH={zlib_path} -DZLIB_LIBRARY:FILEPATH={zlib_path}/libz.lib".format(zlib_path=zlib_path) elif sys.platform == "darwin": - libpng_cmake_options="-DPNG_BUILD_ZLIB=ON -DZLIB_INCLUDE_DIR:PATH={zlib_path} -DZLIB_LIBRARY:FILEPATH={zlib_path}/libz.a".format(zlib_path=zlib_path) + libpng_cmake_options="-DPNG_BUILD_ZLIB=OFF -DZLIB_INCLUDE_DIR:PATH={zlib_path} -DZLIB_LIBRARY:FILEPATH={zlib_path}/libz.a".format(zlib_path=zlib_path) else : - libpng_cmake_options="-DPNG_BUILD_ZLIB=ON -DZLIB_INCLUDE_DIR:PATH={zlib_path} -DZLIB_LIBRARY:FILEPATH={zlib_path}/libz.a".format(zlib_path=zlib_path) + libpng_cmake_options="-DPNG_BUILD_ZLIB=OFF -DCXX_FLAG=-fPIC -DZLIB_INCLUDE_DIR:PATH={zlib_path} -DZLIB_LIBRARY:FILEPATH={zlib_path}/libz.a".format(zlib_path=zlib_path) _build_cmake_dependency("./third_party/zlib") # Deactivate SIMD on macOS jpeg_turbo_options = "" From d6011fe98208987658d683cf1597dc0c4e885151 Mon Sep 17 00:00:00 2001 From: Ryad ZENINE Date: Wed, 12 Feb 2020 07:24:03 +0100 Subject: [PATCH 31/35] Revert "linking statically against zlib" This reverts commit 0993c1d711f6ad90d3cc02e5f1cf5a7a9e795f7b. --- setup.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index 850788479e7..5f2511cddad 100644 --- a/setup.py +++ b/setup.py @@ -96,11 +96,11 @@ def _build_cmake_dependency(directory, args=""): def build_dependencies(): zlib_path = os.path.join(third_party_dir, "zlib") if sys.platform == "win32": - libpng_cmake_options="-DPNG_BUILD_ZLIB=OFF -DZLIB_INCLUDE_DIR:PATH={zlib_path} -DZLIB_LIBRARY:FILEPATH={zlib_path}/libz.lib".format(zlib_path=zlib_path) + libpng_cmake_options="-DPNG_BUILD_ZLIB=ON -DZLIB_INCLUDE_DIR:PATH={zlib_path} -DZLIB_LIBRARY:FILEPATH={zlib_path}/libz.dll".format(zlib_path=zlib_path) elif sys.platform == "darwin": - libpng_cmake_options="-DPNG_BUILD_ZLIB=OFF -DZLIB_INCLUDE_DIR:PATH={zlib_path} -DZLIB_LIBRARY:FILEPATH={zlib_path}/libz.a".format(zlib_path=zlib_path) + libpng_cmake_options="-DPNG_BUILD_ZLIB=ON -DZLIB_INCLUDE_DIR:PATH={zlib_path} -DZLIB_LIBRARY:FILEPATH={zlib_path}/libz.dylib".format(zlib_path=zlib_path) else : - libpng_cmake_options="-DPNG_BUILD_ZLIB=OFF -DCXX_FLAG=-fPIC -DZLIB_INCLUDE_DIR:PATH={zlib_path} -DZLIB_LIBRARY:FILEPATH={zlib_path}/libz.a".format(zlib_path=zlib_path) + libpng_cmake_options="-DPNG_BUILD_ZLIB=ON -DZLIB_INCLUDE_DIR:PATH={zlib_path} -DZLIB_LIBRARY:FILEPATH={zlib_path}/libz.so".format(zlib_path=zlib_path) _build_cmake_dependency("./third_party/zlib") # Deactivate SIMD on macOS jpeg_turbo_options = "" @@ -181,7 +181,6 @@ def get_extensions(): library_dirs=third_party_search_directories, define_macros=define_macros, extra_compile_args=extra_compile_args, - extra_link_args={} ) ] if compile_cpp_tests: From d03ccfc0dcd6a37b1e1fcf054aa2ebb0beb8b3f6 Mon Sep 17 00:00:00 2001 From: Ryad ZENINE Date: Wed, 12 Feb 2020 07:32:17 +0100 Subject: [PATCH 32/35] Update conda dependencies --- packaging/torchvision/meta.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packaging/torchvision/meta.yaml b/packaging/torchvision/meta.yaml index 1bc199e437b..0cafe341616 100644 --- a/packaging/torchvision/meta.yaml +++ b/packaging/torchvision/meta.yaml @@ -8,6 +8,7 @@ source: requirements: build: - {{ compiler('c') }} # [win] + - zlib host: - python @@ -21,6 +22,8 @@ requirements: - pillow >=4.1.1 - numpy >=1.11 - six + - libpng + - libjpeg-turbo {{ environ.get('CONDA_PYTORCH_CONSTRAINT') }} {{ environ.get('CONDA_CUDATOOLKIT_CONSTRAINT') }} From 9e7b04296d5551e0c0f2904cf892466b1c5fe253 Mon Sep 17 00:00:00 2001 From: Ryad ZENINE Date: Wed, 12 Feb 2020 07:56:02 +0100 Subject: [PATCH 33/35] fix windows zlib --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 5f2511cddad..8afb401cdd0 100644 --- a/setup.py +++ b/setup.py @@ -84,7 +84,7 @@ def _build_cmake_dependency(directory, args=""): os.chdir(directory) if sys.platform == 'win32': os.system("cmake.exe --clean .") - os.system('cmake.exe -G"Visual Studio 16 2019" {} .'.format(args) ) + os.system('cmake.exe -G"Visual Studio 16 2019" -DCMAKE_BUILD_TYPE=Release {} .'.format(args) ) os.system("cmake.exe --build .") else: os.system("cmake --clean .") @@ -96,7 +96,7 @@ def _build_cmake_dependency(directory, args=""): def build_dependencies(): zlib_path = os.path.join(third_party_dir, "zlib") if sys.platform == "win32": - libpng_cmake_options="-DPNG_BUILD_ZLIB=ON -DZLIB_INCLUDE_DIR:PATH={zlib_path} -DZLIB_LIBRARY:FILEPATH={zlib_path}/libz.dll".format(zlib_path=zlib_path) + libpng_cmake_options="-DPNG_BUILD_ZLIB=ON -DZLIB_INCLUDE_DIR:PATH={zlib_path} -DZLIB_LIBRARY:FILEPATH={zlib_path}/Release/zlibd.dll".format(zlib_path=zlib_path) elif sys.platform == "darwin": libpng_cmake_options="-DPNG_BUILD_ZLIB=ON -DZLIB_INCLUDE_DIR:PATH={zlib_path} -DZLIB_LIBRARY:FILEPATH={zlib_path}/libz.dylib".format(zlib_path=zlib_path) else : From 86aa9267bb14473391039a719332f6129beb4308 Mon Sep 17 00:00:00 2001 From: Ryad ZENINE Date: Wed, 12 Feb 2020 07:57:11 +0100 Subject: [PATCH 34/35] Increase timeout for conda_macos --- .circleci/config.yml | 1 + .circleci/config.yml.in | 1 + 2 files changed, 2 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index e044aa089c3..7f0e1b37ef6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -249,6 +249,7 @@ jobs: conda install -yq conda-build conda install -yq -c anaconda cmake packaging/build_conda.sh + no_output_timeout: 30m - store_artifacts: path: /Users/distiller/miniconda3/conda-bld/osx-64 - persist_to_workspace: diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index 4fe28fb11e9..bd0d4e83fcb 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -249,6 +249,7 @@ jobs: conda install -yq conda-build conda install -yq -c anaconda cmake packaging/build_conda.sh + no_output_timeout: 30m - store_artifacts: path: /Users/distiller/miniconda3/conda-bld/osx-64 - persist_to_workspace: From 1ed3ed3f5fc65592d497f5ca59675121353796e4 Mon Sep 17 00:00:00 2001 From: Ryad ZENINE Date: Wed, 12 Feb 2020 08:15:06 +0100 Subject: [PATCH 35/35] Fix windows --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 8afb401cdd0..cadbfcd0f81 100644 --- a/setup.py +++ b/setup.py @@ -96,7 +96,7 @@ def _build_cmake_dependency(directory, args=""): def build_dependencies(): zlib_path = os.path.join(third_party_dir, "zlib") if sys.platform == "win32": - libpng_cmake_options="-DPNG_BUILD_ZLIB=ON -DZLIB_INCLUDE_DIR:PATH={zlib_path} -DZLIB_LIBRARY:FILEPATH={zlib_path}/Release/zlibd.dll".format(zlib_path=zlib_path) + libpng_cmake_options="-DPNG_BUILD_ZLIB=ON -DZLIB_INCLUDE_DIR:PATH={zlib_path} -DZLIB_LIBRARY:FILEPATH={zlib_path}/Debug/zlibd.dll".format(zlib_path=zlib_path) elif sys.platform == "darwin": libpng_cmake_options="-DPNG_BUILD_ZLIB=ON -DZLIB_INCLUDE_DIR:PATH={zlib_path} -DZLIB_LIBRARY:FILEPATH={zlib_path}/libz.dylib".format(zlib_path=zlib_path) else :