OpenVINO EP: Put infer_request back into queue upon catching exception (#679)

Author: RyanMetcalfeInt8

onnxruntime/core/providers/openvino/backends/basic_backend.cc

@@ -761,12 +761,14 @@ void BasicBackend::Infer(OrtKernelContext* ctx) {
     try {
       StartAsyncInference(context, infer_request);
     } catch (const std::runtime_error& e) {
+      inferRequestsQueue_->putIdleRequest(std::move(infer_request));
       ORT_THROW(log_tag + " Exception at StartAsyncInference: " + e.what());
     }
 #endif
     try {
       CompleteAsyncInference(context, infer_request);
     } catch (const std::runtime_error& e) {
+      inferRequestsQueue_->putIdleRequest(std::move(infer_request));
       ORT_THROW(log_tag + " Exception at CompleteAsyncInference: " + e.what());
     }
 

onnxruntime/core/providers/openvino/ov_interface.cc

@@ -380,29 +380,29 @@ void OVInferRequest::StartAsync() {
   try {
     ovInfReq.start_async();
   } catch (const Exception& e) {
-    ORT_THROW(log_tag + " Couldn't start Inference: " + e.what());
+    throw std::runtime_error(log_tag + " Couldn't start Inference: " + e.what());
   } catch (...) {
-    ORT_THROW(log_tag + " In Error Couldn't start Inference");
+    throw std::runtime_error(log_tag + " In Error Couldn't start Inference");
   }
 }
 
 void OVInferRequest::Infer() {
   try {
     ovInfReq.infer();
   } catch (const Exception& e) {
-    ORT_THROW(log_tag + " Couldn't start Inference: " + e.what());
+    throw std::runtime_error(log_tag + " Couldn't start Inference: " + e.what());
   } catch (...) {
-    ORT_THROW(log_tag + " In Error Couldn't start Inference");
+    throw std::runtime_error(log_tag + " In Error Couldn't start Inference");
   }
 }
 
 void OVInferRequest::WaitRequest() {
   try {
     ovInfReq.wait();
   } catch (const Exception& e) {
-    ORT_THROW(log_tag + " Wait Model Failed: " + e.what());
+    throw std::runtime_error(log_tag + " Wait Model Failed: " + e.what());
   } catch (...) {
-    ORT_THROW(log_tag + " Wait Mode Failed");
+    throw std::runtime_error(log_tag + " Wait Mode Failed");
   }
 }