handle task artifacts - workflow executer

src/.sonarlint/project-monai_monai-deploy-workflow-managercsharp.ruleset

@@ -1,5 +1,9 @@
-<?xml version="1.0" encoding="utf-8"?>
-<RuleSet xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" Name="SonarQube - monai-deploy-workflow-manager Sonar way" Description="This rule set was automatically generated from SonarQube https://sonarcloud.io/profiles/show?key=AX96ONQSnTk2GEVJ9ILJ" ToolsVersion="14.0">
+<?xml version="1.0" encoding="utf-8"?>
+<RuleSet Name="SonarQube - monai-deploy-workflow-manager Sonar way" Description="This rule set was automatically generated from SonarQube https://sonarcloud.io/profiles/show?key=AX96ONQSnTk2GEVJ9ILJ" ToolsVersion="17.0">
+  <Rules AnalyzerId="Microsoft.CodeAnalysis.CSharp" RuleNamespace="Microsoft.CodeAnalysis.CSharp">
+    <Rule Id="CS8618" Action="None" />
+    <Rule Id="CS8625" Action="None" />
+  </Rules>
   <Rules AnalyzerId="SonarAnalyzer.CSharp" RuleNamespace="SonarAnalyzer.CSharp">
     <Rule Id="S100" Action="None" />
     <Rule Id="S1006" Action="Warning" />
@@ -367,4 +371,7 @@
     <Rule Id="S907" Action="Warning" />
     <Rule Id="S927" Action="Warning" />
   </Rules>
+  <Rules AnalyzerId="StyleCop.Analyzers" RuleNamespace="StyleCop.Analyzers">
+    <Rule Id="SA1101" Action="None" />
+  </Rules>
 </RuleSet>

src/Common/Extensions/StorageListExtensions.cs

@@ -0,0 +1,29 @@
+// SPDX-FileCopyrightText: © 2021-2022 MONAI Consortium
+// SPDX-License-Identifier: Apache License 2.0
+
+using Ardalis.GuardClauses;
+
+namespace Monai.Deploy.WorkflowManager.Common.Extensions
+{
+    public static class StorageListExtensions
+    {
+        public static Dictionary<string, string> ToArtifactDictionary(this List<Messaging.Common.Storage> storageList)
+        {
+            Guard.Against.Null(storageList, nameof(storageList));
+
+            var artifactDict = new Dictionary<string, string>();
+
+            foreach (var storage in storageList)
+            {
+                if (string.IsNullOrWhiteSpace(storage.Name) || string.IsNullOrWhiteSpace(storage.RelativeRootPath))
+                {
+                    continue;
+                }
+
+                artifactDict.Add(storage.Name, storage.RelativeRootPath);
+            }
+
+            return artifactDict;
+        }
+    }
+}

src/Configuration/Monai.Deploy.WorkflowManager.Configuration.csproj

@@ -15,8 +15,8 @@ SPDX-License-Identifier: Apache License 2.0
     <PackageReference Include="Microsoft.Extensions.Configuration" Version="6.0.1" />
     <PackageReference Include="Microsoft.Extensions.Options" Version="6.0.0" />
     <PackageReference Include="Newtonsoft.Json" Version="13.0.1" />
-    <PackageReference Include="Monai.Deploy.Messaging" Version="0.1.0-rc0036" />
-    <PackageReference Include="Monai.Deploy.Storage" Version="0.1.0-rc0019" />
+    <PackageReference Include="Monai.Deploy.Messaging" Version="0.1.0-rc0040" />
+    <PackageReference Include="Monai.Deploy.Storage" Version="0.1.0-rc0021" />
     <PackageReference Include="System.IO.Abstractions" Version="16.1.25" />
   </ItemGroup>
 
@@ -29,10 +29,9 @@ SPDX-License-Identifier: Apache License 2.0
   <ItemGroup>
     <AdditionalFiles Include="..\.sonarlint\project-monai_monai-deploy-workflow-manager\CSharp\SonarLint.xml" Link="SonarLint.xml" />
   </ItemGroup>
-  
+
   <PropertyGroup>
     <RestorePackagesWithLockFile>true</RestorePackagesWithLockFile>
-    <RestoreLockedMode>true</RestoreLockedMode>
     <DisableImplicitNuGetFallbackFolder>true</DisableImplicitNuGetFallbackFolder>
   </PropertyGroup>
 </Project>

src/Configuration/packages.lock.json

@@ -24,9 +24,9 @@
       },
       "Monai.Deploy.Messaging": {
         "type": "Direct",
-        "requested": "[0.1.0-rc0036, )",
-        "resolved": "0.1.0-rc0036",
-        "contentHash": "ONfPbkd0bsJGJHOPVn7U5z0FDNirYuvMswLauN8rxHNSat0GbhFDnObrY8iIuCFY02zfPfpoL9mywkksVwsUAA==",
+        "requested": "[0.1.0-rc0040, )",
+        "resolved": "0.1.0-rc0040",
+        "contentHash": "q5t6h24M+22JQqeBTD9wOOYYAvb1Vk29nkadcOJ05wT569jnF+MGmEherDn1moX45Ehj4TsCvQl3TD4HJlC1mA==",
         "dependencies": {
           "Ardalis.GuardClauses": "4.0.1",
           "Microsoft.Extensions.Configuration": "6.0.1",
@@ -38,9 +38,9 @@
       },
       "Monai.Deploy.Storage": {
         "type": "Direct",
-        "requested": "[0.1.0-rc0019, )",
-        "resolved": "0.1.0-rc0019",
-        "contentHash": "YAXgTE7vTqeMc+ZhprUg/muQ+JDVMaix1Ha1vLJTSSjfZWBOnxyK5SugS50uNlvByJcqHEXh+ygd9219wK1S+Q==",
+        "requested": "[0.1.0-rc0021, )",
+        "resolved": "0.1.0-rc0021",
+        "contentHash": "jealmI38hO1f8iTGt9ACYUx5sIyUnrf5tMVA7DNekzJ9mufhTMpNgukCqmkvYpXeljaPBp6yWn7NzfjiS1fnhw==",
         "dependencies": {
           "AWSSDK.SecurityToken": "3.7.1.141",
           "Ardalis.GuardClauses": "4.0.0",

src/Contracts/Models/Artifact.cs

@@ -12,5 +12,8 @@ public class Artifact
 
         [JsonProperty(PropertyName = "value")]
         public string Value { get; set; }
+
+        [JsonProperty(PropertyName = "mandatory")]
+        public bool Mandatory { get; set; }
     }
 }

src/Contracts/Models/TaskExecution.cs

@@ -27,6 +27,9 @@ public class TaskExecution
         [JsonProperty(PropertyName = "input_artifacts")]
         public Dictionary<string, string> InputArtifacts { get; set; }
 
+        [JsonProperty(PropertyName = "output_artifacts")]
+        public Dictionary<string, string> OutputArtifacts { get; set; }
+
         [JsonProperty(PropertyName = "output_directory")]
         public string OutputDirectory { get; set; }
 

src/Contracts/Monai.Deploy.WorkflowManager.Contracts.csproj

@@ -18,7 +18,7 @@ SPDX-License-Identifier: Apache License 2.0
   <ItemGroup>
     <PackageReference Include="AWSSDK.SecurityToken" Version="3.7.1.141" />
     <PackageReference Include="MongoDB.Bson" Version="2.15.0" />
-    <PackageReference Include="Monai.Deploy.Messaging" Version="0.1.0-rc0036" />
+    <PackageReference Include="Monai.Deploy.Messaging" Version="0.1.0-rc0040" />
     <PackageReference Include="Newtonsoft.Json" Version="13.0.1" />
     <PackageReference Include="Microsoft.Extensions.Configuration" Version="6.0.1" />
   </ItemGroup>

src/Database/Interfaces/IWorkflowInstanceRepository.cs

@@ -37,6 +37,21 @@ public interface IWorkflowInstanceRepository
         /// <param name="status">Status to set.</param>
         Task<bool> UpdateTaskStatusAsync(string workflowInstanceId, string taskId, TaskExecutionStatus status);
 
+        /// <summary>
+        /// Updates the Task output artifacts for a given task within a workflow instance.
+        /// </summary>
+        /// <param name="workflowInstanceId">Workflow Instance to update.</param>
+        /// <param name="taskId">TaskId to update.</param>
+        /// <param name="outputArtifacts">Output artifacts to set.</param>
+        Task<bool> UpdateTaskOutputArtifactsAsync(string workflowInstanceId, string taskId, Dictionary<string, string> outputArtifacts);
+
+        /// <summary>
+        /// Gets a task execution for a given workflow instance id and task id.
+        /// </summary>
+        /// <param name="workflowInstanceId">A Workflow Instance Id to retrieve a task from.</param>
+        /// <param name="taskId">A Task Id to retrieve.</param>
+        Task<TaskExecution> GetTaskByIdAsync(string workflowInstanceId, string taskId);
+
         /// <summary>
         /// Updates the Task list for a given workflow instance.
         /// </summary>

src/Database/Monai.Deploy.WorkflowManager.Database.csproj

@@ -20,7 +20,7 @@
     <PackageReference Include="Ardalis.GuardClauses" Version="4.0.1" />
     <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="6.0.1" />
     <PackageReference Include="Microsoft.Extensions.Options" Version="6.0.0" />
-    <PackageReference Include="Monai.Deploy.Messaging" Version="0.1.0-rc0036" />
+    <PackageReference Include="Monai.Deploy.Messaging" Version="0.1.0-rc0040" />
     <PackageReference Include="MongoDB.Bson" Version="2.15.0" />
     <PackageReference Include="MongoDB.Driver" Version="2.15.0" />
   </ItemGroup>

src/Database/WorkflowInstanceRepository.cs

@@ -55,6 +55,7 @@ public async Task<IList<WorkflowInstance>> GetByWorkflowsIdsAsync(List<string> w
             catch (Exception e)
             {
                 _logger.DbCallFailed(nameof(GetByWorkflowsIdsAsync), e);
+
                 return new List<WorkflowInstance>();
             }
         }
@@ -66,11 +67,13 @@ public async Task<bool> CreateAsync(IList<WorkflowInstance> workflowInstances)
             try
             {
                 await _workflowInstanceCollection.InsertManyAsync(workflowInstances);
+
                 return true;
             }
             catch (Exception e)
             {
                 _logger.DbCallFailed(nameof(CreateAsync), e);
+
                 return false;
             }
         }
@@ -83,14 +86,38 @@ public async Task<bool> UpdateTaskStatusAsync(string workflowInstanceId, string
 
             try
             {
-                var result = await _workflowInstanceCollection.FindOneAndUpdateAsync(
+                await _workflowInstanceCollection.FindOneAndUpdateAsync(
                     i => i.Id == workflowInstanceId && i.Tasks.Any(t => t.TaskId == taskId),
                     Builders<WorkflowInstance>.Update.Set(w => w.Tasks[-1].Status, status));
+
                 return true;
             }
             catch (Exception e)
             {
                 _logger.DbCallFailed(nameof(UpdateTaskStatusAsync), e);
+
+                return false;
+            }
+        }
+
+        public async Task<bool> UpdateTaskOutputArtifactsAsync(string workflowInstanceId, string taskId, Dictionary<string, string> outputArtifacts)
+        {
+            Guard.Against.NullOrWhiteSpace(workflowInstanceId, nameof(workflowInstanceId));
+            Guard.Against.NullOrWhiteSpace(taskId, nameof(taskId));
+            Guard.Against.Null(outputArtifacts, nameof(outputArtifacts));
+
+            try
+            {
+                await _workflowInstanceCollection.FindOneAndUpdateAsync(
+                    i => i.Id == workflowInstanceId && i.Tasks.Any(t => t.TaskId == taskId),
+                    Builders<WorkflowInstance>.Update.Set(w => w.Tasks[-1].OutputArtifacts, outputArtifacts));
+
+                return true;
+            }
+            catch (Exception e)
+            {
+                _logger.DbCallFailed(nameof(UpdateTaskOutputArtifactsAsync), e);
+
                 return false;
             }
         }
@@ -114,6 +141,27 @@ public async Task<bool> UpdateWorkflowInstanceStatusAsync(string workflowInstanc
             }
         }
 
+        public async Task<TaskExecution> GetTaskByIdAsync(string workflowInstanceId, string taskId)
+        {
+            Guard.Against.NullOrWhiteSpace(workflowInstanceId, nameof(workflowInstanceId));
+            Guard.Against.NullOrWhiteSpace(taskId, nameof(taskId));
+
+            try
+            {
+                var workflowInstance = await _workflowInstanceCollection
+                                            .Find(x => x.Id == workflowInstanceId)
+                                            .FirstOrDefaultAsync();
+
+                return workflowInstance?.Tasks?.FirstOrDefault(t => t.TaskId == taskId);
+            }
+            catch (Exception e)
+            {
+                _logger.DbCallFailed(nameof(GetTaskByIdAsync), e);
+
+                return null;
+            }
+        }
+
         public async Task<bool> UpdateTasksAsync(string workflowInstanceId, List<TaskExecution> tasks)
         {
             Guard.Against.NullOrWhiteSpace(workflowInstanceId, nameof(workflowInstanceId));

src/Database/packages.lock.json

@@ -29,9 +29,9 @@
       },
       "Monai.Deploy.Messaging": {
         "type": "Direct",
-        "requested": "[0.1.0-rc0036, )",
-        "resolved": "0.1.0-rc0036",
-        "contentHash": "ONfPbkd0bsJGJHOPVn7U5z0FDNirYuvMswLauN8rxHNSat0GbhFDnObrY8iIuCFY02zfPfpoL9mywkksVwsUAA==",
+        "requested": "[0.1.0-rc0040, )",
+        "resolved": "0.1.0-rc0040",
+        "contentHash": "q5t6h24M+22JQqeBTD9wOOYYAvb1Vk29nkadcOJ05wT569jnF+MGmEherDn1moX45Ehj4TsCvQl3TD4HJlC1mA==",
         "dependencies": {
           "Ardalis.GuardClauses": "4.0.1",
           "Microsoft.Extensions.Configuration": "6.0.1",
@@ -240,7 +240,7 @@
         "dependencies": {
           "AWSSDK.SecurityToken": "3.7.1.141",
           "Microsoft.Extensions.Configuration": "6.0.1",
-          "Monai.Deploy.Messaging": "0.1.0-rc0036",
+          "Monai.Deploy.Messaging": "0.1.0-rc0040",
           "MongoDB.Bson": "2.15.0",
           "Newtonsoft.Json": "13.0.1"
         }

src/Monai.Deploy.WorkflowManager.sln

@@ -55,6 +55,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Monai.Deploy.WorkflowManage
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Monai.Deploy.WorkflowManager.ConditionsResolver.Tests", "..\tests\UnitTests\ConditionsResolver.Tests\Monai.Deploy.WorkflowManager.ConditionsResolver.Tests.csproj", "{918E4DE3-A7BF-4B7F-9B5A-5C36FEFA3C30}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Monai.Deploy.WorkflowManager.Common.Tests", "..\tests\UnitTests\WorkflowManager.Common.Tests\Monai.Deploy.WorkflowManager.Common.Tests.csproj", "{A44F975E-70CA-49D6-8513-78F2D5210EAF}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -157,6 +159,10 @@ Global
 		{918E4DE3-A7BF-4B7F-9B5A-5C36FEFA3C30}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{918E4DE3-A7BF-4B7F-9B5A-5C36FEFA3C30}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{918E4DE3-A7BF-4B7F-9B5A-5C36FEFA3C30}.Release|Any CPU.Build.0 = Release|Any CPU
+		{A44F975E-70CA-49D6-8513-78F2D5210EAF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{A44F975E-70CA-49D6-8513-78F2D5210EAF}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{A44F975E-70CA-49D6-8513-78F2D5210EAF}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{A44F975E-70CA-49D6-8513-78F2D5210EAF}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -176,6 +182,7 @@ Global
 		{2DA40575-4748-4198-BE57-F4AF070DE8E3} = {71DDEE7B-E213-4E39-A7F4-4646783A27F7}
 		{89D3D817-CCFE-4933-9089-D1283F2EA1B5} = {71DDEE7B-E213-4E39-A7F4-4646783A27F7}
 		{918E4DE3-A7BF-4B7F-9B5A-5C36FEFA3C30} = {71DDEE7B-E213-4E39-A7F4-4646783A27F7}
+		{A44F975E-70CA-49D6-8513-78F2D5210EAF} = {71DDEE7B-E213-4E39-A7F4-4646783A27F7}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {DC0D56C8-D8CB-45CE-B528-F3DCF86D63ED}

src/PayloadListener/Monai.Deploy.WorkflowManager.PayloadListener.csproj

@@ -11,11 +11,15 @@ SPDX-License-Identifier: Apache License 2.0
     <Nullable>enable</Nullable>
   </PropertyGroup>
 
+  <ItemGroup>
+    <AdditionalFiles Include="..\.sonarlint\project-monai_monai-deploy-workflow-manager\CSharp\SonarLint.xml" Link="SonarLint.xml" />
+  </ItemGroup>
+
   <ItemGroup>
     <PackageReference Include="Microsoft.Extensions.Hosting.Abstractions" Version="6.0.0" />
     <PackageReference Include="Ardalis.GuardClauses" Version="4.0.1" />
-    <PackageReference Include="Monai.Deploy.Messaging" Version="0.1.0-rc0036" />
-    <PackageReference Include="Monai.Deploy.Storage" Version="0.1.0-rc0019" />
+    <PackageReference Include="Monai.Deploy.Messaging" Version="0.1.0-rc0040" />
+    <PackageReference Include="Monai.Deploy.Storage" Version="0.1.0-rc0021" />
   </ItemGroup>
 
   <ItemGroup>
@@ -26,4 +30,9 @@ SPDX-License-Identifier: Apache License 2.0
     <ProjectReference Include="..\WorkflowExecuter\Monai.Deploy.WorkloadManager.WorkfowExecuter.csproj" />
   </ItemGroup>
 
+  <PropertyGroup>
+    <RestorePackagesWithLockFile>true</RestorePackagesWithLockFile>
+    <DisableImplicitNuGetFallbackFolder>true</DisableImplicitNuGetFallbackFolder>
+  </PropertyGroup>
+
 </Project>