diff --git a/sycl/source/detail/program_manager/program_manager.cpp b/sycl/source/detail/program_manager/program_manager.cpp
index 05dcdb3fc2816..20f9fa5a91004 100644
--- a/sycl/source/detail/program_manager/program_manager.cpp
+++ b/sycl/source/detail/program_manager/program_manager.cpp
@@ -628,9 +628,7 @@ void ProgramManager::populateSpecConstRegistry() {
   }
   for (const auto &Entry : m_DeviceImages) {
     const std::vector<RTDeviceBinaryImageUPtr> &Imgs = *Entry.second.get();
-
-    if (Imgs.size() == 0)
-      continue;
+    assert((Imgs.size() > 0) && "no device binary image for a kernel set");
     OSModuleHandle H = Imgs[0]->getOSModuleHandle();
     SpecConstMapTy &GlobalIDMap = SpecConstRegistry[H];