[SYCL] Fix hang in ProgramManager class (#2274)

This patch fixes sporadic hangs caused by data race in NativePrograms variable of the ProgramManager class. 
The current mutex lock from the program cache is not thread-safe.
When multiple threads use their own queues then they use different contexts and then they use their own program caches.
And since multiple threads can queue own command of kernel execution in parallel, 
then each thread will lock its own program cache mutex but the access into NativePrograms variable remains with a race between the threads.

Signed-off-by: Alexander Flegontov <[email protected]>

Author: alexanderfle

sycl/source/detail/program_manager/program_manager.cpp

@@ -331,7 +331,7 @@ RT::PiProgram ProgramManager::createPIProgram(const RTDeviceBinaryImage &Img,
           : createBinaryProgram(Ctx, RawImg.BinaryStart, ImgSize);
 
   {
-    auto LockGuard = Ctx->getKernelProgramCache().acquireCachedPrograms();
+    std::lock_guard<std::mutex> Lock(MNativeProgramsMutex);
     // associate the PI program with the image it was created for
     NativePrograms[Res] = &Img;
   }
@@ -984,8 +984,7 @@ void ProgramManager::flushSpecConstants(const program_impl &Prg,
   if (!Img) {
     // caller hasn't provided the image object - find it
     { // make sure NativePrograms map access is synchronized
-      ContextImplPtr Ctx = getSyclObjImpl(Prg.get_context());
-      auto LockGuard = Ctx->getKernelProgramCache().acquireCachedPrograms();
+      std::lock_guard<std::mutex> Lock(MNativeProgramsMutex);
       auto It = NativePrograms.find(NativePrg);
       if (It == NativePrograms.end())
         throw sycl::experimental::spec_const_error(

sycl/source/detail/program_manager/program_manager.hpp

@@ -181,9 +181,11 @@ class ProgramManager {
   // NOTE: keys in the map can be invalid (reference count went to zero and
   // the underlying program disposed of), so the map can't be used in any way
   // other than binary image lookup with known live PiProgram as the key.
-  // NOTE: access is synchronized via the same lock as program cache
+  // NOTE: access is synchronized via the MNativeProgramsMutex
   std::unordered_map<pi::PiProgram, const RTDeviceBinaryImage *> NativePrograms;
 
+  /// Protects NativePrograms that can be changed by class' methods.
+  std::mutex MNativeProgramsMutex;
   /// True iff a SPIRV file has been specified with an environment variable
   bool m_UseSpvFile = false;
 };