diff --git a/CMakeLists.txt b/CMakeLists.txt
index 4e222dbfa0e..85b878307cf 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -4,6 +4,8 @@ set(CMAKE_CXX_STANDARD 14)
 file(STRINGS version.txt TORCHVISION_VERSION)
 
 option(WITH_CUDA "Enable CUDA support" OFF)
+option(WITH_PNG "Enable features requiring LibPNG." ON)
+option(WITH_JPEG "Enable features requiring LibJPEG." ON)
 option(USE_PYTHON "Link to Python when building" OFF)
 
 if(WITH_CUDA)
@@ -18,17 +20,23 @@ if(WITH_CUDA)
   endif()
 endif()
 
+find_package(Torch REQUIRED)
+
+if (WITH_PNG)
+    add_definitions(-DPNG_FOUND)
+    find_package(PNG REQUIRED)
+endif()
+
+if (WITH_JPEG)
+    add_definitions(-DJPEG_FOUND)
+    find_package(JPEG REQUIRED)
+endif()
+
 if (USE_PYTHON)
   add_definitions(-DUSE_PYTHON)
   find_package(Python3 REQUIRED COMPONENTS Development)
 endif()
 
-find_package(Torch REQUIRED)
-find_package(PNG REQUIRED)
-find_package(JPEG REQUIRED)
-add_definitions(-DJPEG_FOUND)
-add_definitions(-DPNG_FOUND)
-
 function(CUDA_CONVERT_FLAGS EXISTING_TARGET)
     get_property(old_flags TARGET ${EXISTING_TARGET} PROPERTY INTERFACE_COMPILE_OPTIONS)
     if(NOT "${old_flags}" STREQUAL "")
@@ -82,7 +90,15 @@ FOREACH(DIR ${ALLOW_LISTED})
 ENDFOREACH()
 
 add_library(${PROJECT_NAME} SHARED ${ALL_SOURCES})
-target_link_libraries(${PROJECT_NAME} PRIVATE ${TORCH_LIBRARIES} ${PNG_LIBRARY} ${JPEG_LIBRARIES})
+target_link_libraries(${PROJECT_NAME} PRIVATE ${TORCH_LIBRARIES})
+
+if (WITH_PNG)
+    target_link_libraries(${PROJECT_NAME} PRIVATE ${PNG_LIBRARY})
+endif()
+
+if (WITH_JPEG)
+    target_link_libraries(${PROJECT_NAME} PRIVATE ${JPEG_LIBRARIES})
+endif()
 
 if (USE_PYTHON)
   target_link_libraries(${PROJECT_NAME} PRIVATE Python3::Python)
@@ -92,7 +108,15 @@ set_target_properties(${PROJECT_NAME} PROPERTIES
   EXPORT_NAME TorchVision
   INSTALL_RPATH ${TORCH_INSTALL_PREFIX}/lib)
 
-include_directories(torchvision/csrc ${JPEG_INCLUDE_DIRS} ${PNG_INCLUDE_DIRS})
+include_directories(torchvision/csrc)
+
+if (WITH_PNG)
+    include_directories(${PNG_INCLUDE_DIRS})
+endif()
+
+if (WITH_JPEG)
+    include_directories(${JPEG_INCLUDE_DIRS})
+endif()
 
 set(TORCHVISION_CMAKECONFIG_INSTALL_DIR "share/cmake/TorchVision" CACHE STRING "install path for TorchVisionConfig.cmake")
 
diff --git a/cmake/TorchVisionConfig.cmake.in b/cmake/TorchVisionConfig.cmake.in
index aa042844753..9e92bc3b512 100644
--- a/cmake/TorchVisionConfig.cmake.in
+++ b/cmake/TorchVisionConfig.cmake.in
@@ -22,23 +22,36 @@ if(NOT (CMAKE_VERSION VERSION_LESS 3.0))
 # Don't include targets if this file is being picked up by another
 # project which has already built this as a subproject
 #-----------------------------------------------------------------------------
-if(NOT TARGET ${PN}::TorchVision)
+if(NOT TARGET ${PN}::${PN})
 include("${CMAKE_CURRENT_LIST_DIR}/${PN}Targets.cmake")
 
-if(NOT TARGET torch_library)
+target_include_directories(${PN}::${PN} INTERFACE "${${PN}_INCLUDE_DIR}")
+
+if(@WITH_CUDA@)
+  target_compile_definitions(${PN}::${PN} INTERFACE WITH_CUDA)
+endif()
+
 find_package(Torch REQUIRED)
+target_link_libraries(${PN}::${PN} INTERFACE torch)
+
+if(@WITH_PNG@)
+  find_package(PNG REQUIRED)
+  target_link_libraries(${PN}::${PN} INTERFACE ${PNG_LIBRARY})
+  target_compile_definitions(${PN}::${PN} INTERFACE PNG_FOUND)
+endif()
+
+if(@WITH_JPEG@)
+  find_package(JPEG REQUIRED)
+  target_link_libraries(${PN}::${PN} INTERFACE ${JPEG_LIBRARIES})
+  target_compile_definitions(${PN}::${PN} INTERFACE JPEG_FOUND)
 endif()
+
 if (@USE_PYTHON@)
   if(NOT TARGET Python3::Python)
     find_package(Python3 COMPONENTS Development)
   endif()
-endif()
-
-set_target_properties(TorchVision::TorchVision PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${${PN}_INCLUDE_DIR}" INTERFACE_LINK_LIBRARIES "torch;Python3::Python" )
-
-
-if(@WITH_CUDA@)
-  target_compile_definitions(TorchVision::TorchVision INTERFACE WITH_CUDA)
+  target_link_libraries(torch INTERFACE Python3::Python)
+  target_compile_definitions(${PN}::${PN} INTERFACE USE_PYTHON)
 endif()
 
 endif()