From b1d6215155418eea55333acb24f5e670f0da906e Mon Sep 17 00:00:00 2001 From: Jack Schofield Date: Thu, 19 May 2022 11:40:58 +0100 Subject: [PATCH 1/9] Add artifact mapper Signed-off-by: Jack Schofield --- src/Configuration/packages.lock.json | 12 +-- src/Contracts/Models/TaskExecution.cs | 3 + .../Interfaces/IWorkflowInstanceRepository.cs | 7 ++ src/Database/WorkflowInstanceRepository.cs | 20 ++++ src/TaskManager/API/packages.lock.json | 14 +-- .../Plug-ins/Argo/packages.lock.json | 22 ++--- src/TaskManager/packages.lock.json | 26 ++--- .../Common/ArtifactExtensions.cs | 18 ---- src/WorkflowExecuter/Common/ArtifactMapper.cs | 75 ++++++++++++++ .../Common/IArtifactMapper.cs | 12 +++ .../Services/WorkflowExecuterService.cs | 22 +++-- src/WorkflowManager/Program.cs | 2 + .../Common/ArtifactMapperTests.cs | 99 +++++++++++++++++++ .../Services/WorkflowExecuterServiceTests.cs | 9 +- 14 files changed, 276 insertions(+), 65 deletions(-) delete mode 100644 src/WorkflowExecuter/Common/ArtifactExtensions.cs create mode 100644 src/WorkflowExecuter/Common/ArtifactMapper.cs create mode 100644 src/WorkflowExecuter/Common/IArtifactMapper.cs create mode 100644 tests/UnitTests/WorkflowExecuter.Tests/Common/ArtifactMapperTests.cs diff --git a/src/Configuration/packages.lock.json b/src/Configuration/packages.lock.json index 70ab503a1..b206b78e5 100644 --- a/src/Configuration/packages.lock.json +++ b/src/Configuration/packages.lock.json @@ -24,9 +24,9 @@ }, "Monai.Deploy.Messaging": { "type": "Direct", - "requested": "[0.1.0-rc0036, )", - "resolved": "0.1.0-rc0036", - "contentHash": "ONfPbkd0bsJGJHOPVn7U5z0FDNirYuvMswLauN8rxHNSat0GbhFDnObrY8iIuCFY02zfPfpoL9mywkksVwsUAA==", + "requested": "[0.1.0-rc0038, )", + "resolved": "0.1.0-rc0038", + "contentHash": "j6RswItc5GE7b42T1KMMT9oXsOEizr6HKgso4WAV/evGmcwMnzuBdRJHNIfGWeDTTKO7Ld7+BJrSOEno5+udsg==", "dependencies": { "Ardalis.GuardClauses": "4.0.1", "Microsoft.Extensions.Configuration": "6.0.1", @@ -38,9 +38,9 @@ }, "Monai.Deploy.Storage": { "type": "Direct", - "requested": "[0.1.0-rc0019, )", - "resolved": "0.1.0-rc0019", - "contentHash": "YAXgTE7vTqeMc+ZhprUg/muQ+JDVMaix1Ha1vLJTSSjfZWBOnxyK5SugS50uNlvByJcqHEXh+ygd9219wK1S+Q==", + "requested": "[0.1.0-rc0020, )", + "resolved": "0.1.0-rc0020", + "contentHash": "SNNzbGPte4S2CKzsYI5g723aRMNG8AVFNYR847hHl8JQXCD5f6ugxEJUid4/hpu88HxPPigDm7sMrc5H4EwtZQ==", "dependencies": { "AWSSDK.SecurityToken": "3.7.1.141", "Ardalis.GuardClauses": "4.0.0", diff --git a/src/Contracts/Models/TaskExecution.cs b/src/Contracts/Models/TaskExecution.cs index 6d6085a5f..7c59232ea 100644 --- a/src/Contracts/Models/TaskExecution.cs +++ b/src/Contracts/Models/TaskExecution.cs @@ -27,6 +27,9 @@ public class TaskExecution [JsonProperty(PropertyName = "input_artifacts")] public Dictionary InputArtifacts { get; set; } + [JsonProperty(PropertyName = "output_artifacts")] + public Dictionary OutputArtifacts { get; set; } + [JsonProperty(PropertyName = "output_directory")] public string OutputDirectory { get; set; } diff --git a/src/Database/Interfaces/IWorkflowInstanceRepository.cs b/src/Database/Interfaces/IWorkflowInstanceRepository.cs index 6425a6d13..a5916fd32 100644 --- a/src/Database/Interfaces/IWorkflowInstanceRepository.cs +++ b/src/Database/Interfaces/IWorkflowInstanceRepository.cs @@ -37,6 +37,13 @@ public interface IWorkflowInstanceRepository /// Status to set. Task UpdateTaskStatusAsync(string workflowInstanceId, string taskId, TaskExecutionStatus status); + /// + /// Gets a task execution for a given workflow instance id and task id. + /// + /// A Workflow Instance Id to retrieve a task from. + /// A Task Id to retrieve. + Task GetTaskByIdAsync(string workflowInstanceId, string taskId); + /// /// Updates the Task list for a given workflow instance. /// diff --git a/src/Database/WorkflowInstanceRepository.cs b/src/Database/WorkflowInstanceRepository.cs index b05226973..bb31121ce 100644 --- a/src/Database/WorkflowInstanceRepository.cs +++ b/src/Database/WorkflowInstanceRepository.cs @@ -114,6 +114,26 @@ public async Task UpdateWorkflowInstanceStatusAsync(string workflowInstanc } } + public async Task GetTaskByIdAsync(string workflowInstanceId, string taskId) + { + Guard.Against.NullOrWhiteSpace(workflowInstanceId, nameof(workflowInstanceId)); + Guard.Against.NullOrWhiteSpace(taskId, nameof(taskId)); + + try + { + var workflowInstance = await _workflowInstanceCollection + .Find(x => x.Id == workflowInstanceId) + .FirstOrDefaultAsync(); + + return workflowInstance?.Tasks?.FirstOrDefault(t => t.TaskId == taskId); + } + catch (Exception e) + { + _logger.DbCallFailed(nameof(GetTaskByIdAsync), e); + return null; + } + } + public async Task UpdateTasksAsync(string workflowInstanceId, List tasks) { Guard.Against.NullOrWhiteSpace(workflowInstanceId, nameof(workflowInstanceId)); diff --git a/src/TaskManager/API/packages.lock.json b/src/TaskManager/API/packages.lock.json index c14d3c86d..16ed3b757 100644 --- a/src/TaskManager/API/packages.lock.json +++ b/src/TaskManager/API/packages.lock.json @@ -4,9 +4,9 @@ "net6.0": { "Monai.Deploy.Messaging": { "type": "Direct", - "requested": "[0.1.0-rc0036, )", - "resolved": "0.1.0-rc0036", - "contentHash": "ONfPbkd0bsJGJHOPVn7U5z0FDNirYuvMswLauN8rxHNSat0GbhFDnObrY8iIuCFY02zfPfpoL9mywkksVwsUAA==", + "requested": "[0.1.0-rc0038, )", + "resolved": "0.1.0-rc0038", + "contentHash": "j6RswItc5GE7b42T1KMMT9oXsOEizr6HKgso4WAV/evGmcwMnzuBdRJHNIfGWeDTTKO7Ld7+BJrSOEno5+udsg==", "dependencies": { "Ardalis.GuardClauses": "4.0.1", "Microsoft.Extensions.Configuration": "6.0.1", @@ -18,9 +18,9 @@ }, "Monai.Deploy.Storage": { "type": "Direct", - "requested": "[0.1.0-rc0019, )", - "resolved": "0.1.0-rc0019", - "contentHash": "YAXgTE7vTqeMc+ZhprUg/muQ+JDVMaix1Ha1vLJTSSjfZWBOnxyK5SugS50uNlvByJcqHEXh+ygd9219wK1S+Q==", + "requested": "[0.1.0-rc0020, )", + "resolved": "0.1.0-rc0020", + "contentHash": "SNNzbGPte4S2CKzsYI5g723aRMNG8AVFNYR847hHl8JQXCD5f6ugxEJUid4/hpu88HxPPigDm7sMrc5H4EwtZQ==", "dependencies": { "AWSSDK.SecurityToken": "3.7.1.141", "Ardalis.GuardClauses": "4.0.0", @@ -245,4 +245,4 @@ } } } -} +} \ No newline at end of file diff --git a/src/TaskManager/Plug-ins/Argo/packages.lock.json b/src/TaskManager/Plug-ins/Argo/packages.lock.json index 39fef6fb9..7a9dda53e 100644 --- a/src/TaskManager/Plug-ins/Argo/packages.lock.json +++ b/src/TaskManager/Plug-ins/Argo/packages.lock.json @@ -34,9 +34,9 @@ }, "Monai.Deploy.Messaging": { "type": "Direct", - "requested": "[0.1.0-rc0036, )", - "resolved": "0.1.0-rc0036", - "contentHash": "ONfPbkd0bsJGJHOPVn7U5z0FDNirYuvMswLauN8rxHNSat0GbhFDnObrY8iIuCFY02zfPfpoL9mywkksVwsUAA==", + "requested": "[0.1.0-rc0038, )", + "resolved": "0.1.0-rc0038", + "contentHash": "j6RswItc5GE7b42T1KMMT9oXsOEizr6HKgso4WAV/evGmcwMnzuBdRJHNIfGWeDTTKO7Ld7+BJrSOEno5+udsg==", "dependencies": { "Ardalis.GuardClauses": "4.0.1", "Microsoft.Extensions.Configuration": "6.0.1", @@ -48,9 +48,9 @@ }, "Monai.Deploy.Storage": { "type": "Direct", - "requested": "[0.1.0-rc0019, )", - "resolved": "0.1.0-rc0019", - "contentHash": "YAXgTE7vTqeMc+ZhprUg/muQ+JDVMaix1Ha1vLJTSSjfZWBOnxyK5SugS50uNlvByJcqHEXh+ygd9219wK1S+Q==", + "requested": "[0.1.0-rc0020, )", + "resolved": "0.1.0-rc0020", + "contentHash": "SNNzbGPte4S2CKzsYI5g723aRMNG8AVFNYR847hHl8JQXCD5f6ugxEJUid4/hpu88HxPPigDm7sMrc5H4EwtZQ==", "dependencies": { "AWSSDK.SecurityToken": "3.7.1.141", "Ardalis.GuardClauses": "4.0.0", @@ -601,7 +601,7 @@ "dependencies": { "AWSSDK.SecurityToken": "3.7.1.141", "Microsoft.Extensions.Configuration": "6.0.1", - "Monai.Deploy.Messaging": "0.1.0-rc0036", + "Monai.Deploy.Messaging": "0.1.0-rc0038", "MongoDB.Bson": "2.15.0", "Newtonsoft.Json": "13.0.1" } @@ -612,7 +612,7 @@ "Ardalis.GuardClauses": "4.0.1", "Microsoft.Extensions.Logging.Abstractions": "6.0.1", "Microsoft.Extensions.Options": "6.0.0", - "Monai.Deploy.Messaging": "0.1.0-rc0036", + "Monai.Deploy.Messaging": "0.1.0-rc0038", "Monai.Deploy.WorkflowManager.Contracts": "1.0.0", "Monai.Deploy.WorkflowManager.Logging": "1.0.0", "MongoDB.Bson": "2.15.0", @@ -628,10 +628,10 @@ "monai.deploy.workflowmanager.taskmanager.api": { "type": "Project", "dependencies": { - "Monai.Deploy.Messaging": "0.1.0-rc0036", - "Monai.Deploy.Storage": "0.1.0-rc0019" + "Monai.Deploy.Messaging": "0.1.0-rc0038", + "Monai.Deploy.Storage": "0.1.0-rc0020" } } } } -} +} \ No newline at end of file diff --git a/src/TaskManager/packages.lock.json b/src/TaskManager/packages.lock.json index 17077ed77..8cfac393f 100644 --- a/src/TaskManager/packages.lock.json +++ b/src/TaskManager/packages.lock.json @@ -60,9 +60,9 @@ }, "Monai.Deploy.Messaging": { "type": "Direct", - "requested": "[0.1.0-rc0036, )", - "resolved": "0.1.0-rc0036", - "contentHash": "ONfPbkd0bsJGJHOPVn7U5z0FDNirYuvMswLauN8rxHNSat0GbhFDnObrY8iIuCFY02zfPfpoL9mywkksVwsUAA==", + "requested": "[0.1.0-rc0038, )", + "resolved": "0.1.0-rc0038", + "contentHash": "j6RswItc5GE7b42T1KMMT9oXsOEizr6HKgso4WAV/evGmcwMnzuBdRJHNIfGWeDTTKO7Ld7+BJrSOEno5+udsg==", "dependencies": { "Ardalis.GuardClauses": "4.0.1", "Microsoft.Extensions.Configuration": "6.0.1", @@ -74,9 +74,9 @@ }, "Monai.Deploy.Storage": { "type": "Direct", - "requested": "[0.1.0-rc0019, )", - "resolved": "0.1.0-rc0019", - "contentHash": "YAXgTE7vTqeMc+ZhprUg/muQ+JDVMaix1Ha1vLJTSSjfZWBOnxyK5SugS50uNlvByJcqHEXh+ygd9219wK1S+Q==", + "requested": "[0.1.0-rc0020, )", + "resolved": "0.1.0-rc0020", + "contentHash": "SNNzbGPte4S2CKzsYI5g723aRMNG8AVFNYR847hHl8JQXCD5f6ugxEJUid4/hpu88HxPPigDm7sMrc5H4EwtZQ==", "dependencies": { "AWSSDK.SecurityToken": "3.7.1.141", "Ardalis.GuardClauses": "4.0.0", @@ -562,8 +562,8 @@ "dependencies": { "Microsoft.Extensions.Configuration": "6.0.1", "Microsoft.Extensions.Options": "6.0.0", - "Monai.Deploy.Messaging": "0.1.0-rc0036", - "Monai.Deploy.Storage": "0.1.0-rc0019", + "Monai.Deploy.Messaging": "0.1.0-rc0038", + "Monai.Deploy.Storage": "0.1.0-rc0020", "Newtonsoft.Json": "13.0.1", "System.IO.Abstractions": "16.1.25" } @@ -573,7 +573,7 @@ "dependencies": { "AWSSDK.SecurityToken": "3.7.1.141", "Microsoft.Extensions.Configuration": "6.0.1", - "Monai.Deploy.Messaging": "0.1.0-rc0036", + "Monai.Deploy.Messaging": "0.1.0-rc0038", "MongoDB.Bson": "2.15.0", "Newtonsoft.Json": "13.0.1" } @@ -584,7 +584,7 @@ "Ardalis.GuardClauses": "4.0.1", "Microsoft.Extensions.Logging.Abstractions": "6.0.1", "Microsoft.Extensions.Options": "6.0.0", - "Monai.Deploy.Messaging": "0.1.0-rc0036", + "Monai.Deploy.Messaging": "0.1.0-rc0038", "Monai.Deploy.WorkflowManager.Contracts": "1.0.0", "Monai.Deploy.WorkflowManager.Logging": "1.0.0", "MongoDB.Bson": "2.15.0", @@ -600,10 +600,10 @@ "monai.deploy.workflowmanager.taskmanager.api": { "type": "Project", "dependencies": { - "Monai.Deploy.Messaging": "0.1.0-rc0036", - "Monai.Deploy.Storage": "0.1.0-rc0019" + "Monai.Deploy.Messaging": "0.1.0-rc0038", + "Monai.Deploy.Storage": "0.1.0-rc0020" } } } } -} +} \ No newline at end of file diff --git a/src/WorkflowExecuter/Common/ArtifactExtensions.cs b/src/WorkflowExecuter/Common/ArtifactExtensions.cs deleted file mode 100644 index 174daec2c..000000000 --- a/src/WorkflowExecuter/Common/ArtifactExtensions.cs +++ /dev/null @@ -1,18 +0,0 @@ -// SPDX-FileCopyrightText: © 2021-2022 MONAI Consortium -// SPDX-License-Identifier: Apache License 2.0 - -using Ardalis.GuardClauses; -using Monai.Deploy.WorkflowManager.Contracts.Models; - -namespace Monai.Deploy.WorkloadManager.WorkfowExecuter.Common -{ - public static class ArtifactExtensions - { - public static Dictionary ToDictionary(this Artifact[] artifacts) - { - Guard.Against.NullOrEmpty(artifacts, nameof(artifacts)); - - return artifacts.ToDictionary(a => a.Name, a => a.Value); - } - } -} diff --git a/src/WorkflowExecuter/Common/ArtifactMapper.cs b/src/WorkflowExecuter/Common/ArtifactMapper.cs new file mode 100644 index 000000000..6ed3a0dd3 --- /dev/null +++ b/src/WorkflowExecuter/Common/ArtifactMapper.cs @@ -0,0 +1,75 @@ +// SPDX-FileCopyrightText: © 2021-2022 MONAI Consortium +// SPDX-License-Identifier: Apache License 2.0 + +using Ardalis.GuardClauses; +using Monai.Deploy.WorkflowManager.Contracts.Models; +using Monai.Deploy.WorkflowManager.Database.Interfaces; + +namespace Monai.Deploy.WorkloadManager.WorkfowExecuter.Common +{ + public class ArtifactMapper : IArtifactMapper + { + private readonly IWorkflowInstanceRepository _workflowInstanceRepository; + + public ArtifactMapper(IWorkflowInstanceRepository workflowInstanceRepository) + { + _workflowInstanceRepository = workflowInstanceRepository ?? throw new ArgumentNullException(nameof(workflowInstanceRepository)); + } + + public async Task> ConvertArtifactVariablesToPath(Artifact[] artifacts, string payloadId, string workflowInstanceId) + { + Guard.Against.Null(artifacts); + Guard.Against.NullOrWhiteSpace(payloadId); + Guard.Against.NullOrWhiteSpace(workflowInstanceId); + + var artifactPathDictionary = new Dictionary(); + + foreach (var artifact in artifacts) + { + Guard.Against.NullOrWhiteSpace(artifact.Value); + Guard.Against.NullOrWhiteSpace(artifact.Name); + + var variableString = artifact.Value.Split(" ")?[1]; + + if (variableString.StartsWith("context.input")) + { + artifactPathDictionary.Add(artifact.Name, $"{payloadId}/dcm"); + continue; + } + + if (variableString.StartsWith("context.executions")) + { + var variableWords = variableString.Split("."); + + var variableTaskId = variableWords[2]; + var variableLocation = variableWords[3]; + + var task = await _workflowInstanceRepository.GetTaskByIdAsync(workflowInstanceId, variableTaskId); + + if (variableLocation == "output_dir") + { + artifactPathDictionary.Add(artifact.Name, task.OutputDirectory); + continue; + } + + if (variableLocation == "artifacts") + { + var artifactName = variableWords[4]; + var outputArtifact = task.OutputArtifacts.FirstOrDefault(a => a.Key == artifactName); + + if (!outputArtifact.Equals(default(KeyValuePair))) + { + artifactPathDictionary.Add(outputArtifact.Key, outputArtifact.Value); + } + + continue; + } + + } + + } + + return artifactPathDictionary; + } + } +} diff --git a/src/WorkflowExecuter/Common/IArtifactMapper.cs b/src/WorkflowExecuter/Common/IArtifactMapper.cs new file mode 100644 index 000000000..d6215296e --- /dev/null +++ b/src/WorkflowExecuter/Common/IArtifactMapper.cs @@ -0,0 +1,12 @@ +// SPDX-FileCopyrightText: © 2021-2022 MONAI Consortium +// SPDX-License-Identifier: Apache License 2.0 + +using Monai.Deploy.WorkflowManager.Contracts.Models; + +namespace Monai.Deploy.WorkloadManager.WorkfowExecuter.Common +{ + public interface IArtifactMapper + { + Task> ConvertArtifactVariablesToPath(Artifact[] artifacts, string payloadId, string workflowInstanceId); + } +} diff --git a/src/WorkflowExecuter/Services/WorkflowExecuterService.cs b/src/WorkflowExecuter/Services/WorkflowExecuterService.cs index a2619bd7b..f4720385d 100644 --- a/src/WorkflowExecuter/Services/WorkflowExecuterService.cs +++ b/src/WorkflowExecuter/Services/WorkflowExecuterService.cs @@ -23,6 +23,7 @@ public class WorkflowExecuterService : IWorkflowExecuterService private readonly IWorkflowRepository _workflowRepository; private readonly IWorkflowInstanceRepository _workflowInstanceRepository; private readonly IMessageBrokerPublisherService _messageBrokerPublisherService; + private readonly IArtifactMapper _artifactMapper; private readonly StorageServiceConfiguration _storageConfiguration; private string TaskDispatchRoutingKey { get; } @@ -33,7 +34,8 @@ public WorkflowExecuterService( IOptions storageConfiguration, IWorkflowRepository workflowRepository, IWorkflowInstanceRepository workflowInstanceRepository, - IMessageBrokerPublisherService messageBrokerPublisherService) + IMessageBrokerPublisherService messageBrokerPublisherService, + IArtifactMapper artifactMapper) { if (configuration is null) { @@ -53,6 +55,7 @@ public WorkflowExecuterService( _workflowRepository = workflowRepository ?? throw new ArgumentNullException(nameof(workflowRepository)); _workflowInstanceRepository = workflowInstanceRepository ?? throw new ArgumentNullException(nameof(workflowInstanceRepository)); _messageBrokerPublisherService = messageBrokerPublisherService ?? throw new ArgumentNullException(nameof(messageBrokerPublisherService)); + _artifactMapper = artifactMapper ?? throw new ArgumentNullException(nameof(artifactMapper)); } public async Task ProcessPayload(WorkflowRequestEvent message) @@ -73,7 +76,7 @@ await _workflowRepository.GetWorkflowsByAeTitleAsync(message.CalledAeTitle) as L var workflowInstances = new List(); - workflows.ForEach((workflow) => workflowInstances.Add(CreateWorkFlowIntsance(message, workflow))); + workflows.ForEach(async (workflow) => workflowInstances.Add(await CreateWorkFlowIntsance(message, workflow))); var existingInstances = await _workflowInstanceRepository.GetByWorkflowsIdsAsync(workflowInstances.Select(w => w.WorkflowId).ToList()); workflowInstances.RemoveAll(i => existingInstances.ToList().Exists(e => e.WorkflowId == i.WorkflowId && e.PayloadId == i.PayloadId)); @@ -154,7 +157,7 @@ public async Task ProcessTaskUpdate(TaskUpdateEvent message) } var currentTaskDestinations = workflow.Workflow?.Tasks?.SingleOrDefault(t => t.Id == message.TaskId)?.TaskDestinations; - var newTaskExecutions = HandleTaskDestinations(workflowInstance, workflow, currentTaskDestinations); + var newTaskExecutions = await HandleTaskDestinations(workflowInstance, workflow, currentTaskDestinations); if (!newTaskExecutions.Any()) { @@ -243,7 +246,7 @@ private List HandleTaskDestinations(WorkflowInstance workflowInst continue; } - newTaskExecutions.Add(CreateTaskExecution(newTask, workflowInstance.Id, workflowInstance.BucketId)); + newTaskExecutions.Add(await CreateTaskExecution(newTask, workflowInstance.Id, workflowInstance.BucketId, workflowInstance.PayloadId)); } return newTaskExecutions; @@ -259,7 +262,7 @@ private async Task DispatchTask(WorkflowInstance workflowInstance, TaskExe return await _workflowInstanceRepository.UpdateTaskStatusAsync(workflowInstance.Id, taskExec.TaskId, TaskExecutionStatus.Dispatched); } - private WorkflowInstance CreateWorkFlowIntsance(WorkflowRequestEvent message, WorkflowRevision workflow) + private async Task CreateWorkFlowIntsance(WorkflowRequestEvent message, WorkflowRevision workflow) { Guard.Against.Null(message, nameof(message)); Guard.Against.Null(workflow, nameof(workflow)); @@ -293,7 +296,7 @@ private WorkflowInstance CreateWorkFlowIntsance(WorkflowRequestEvent message, Wo // firstTask = template ?? firstTask; //} - tasks.Add(CreateTaskExecution(firstTask, workflowInstance.Id, message.Bucket)); + tasks.Add(await CreateTaskExecution(firstTask, workflowInstance.Id, message.Bucket, message.PayloadId.ToString())); } workflowInstance.Tasks = tasks; @@ -301,13 +304,14 @@ private WorkflowInstance CreateWorkFlowIntsance(WorkflowRequestEvent message, Wo return workflowInstance; } - private TaskExecution CreateTaskExecution(TaskObject task, string workflowInstanceId, string bucketName) + private async Task CreateTaskExecution(TaskObject task, string workflowInstanceId, string bucketName, string payloadId) { Guard.Against.Null(task, nameof(task)); Guard.Against.NullOrWhiteSpace(task.Type, nameof(task.Type)); Guard.Against.NullOrWhiteSpace(task.Id, nameof(task.Id)); Guard.Against.NullOrWhiteSpace(workflowInstanceId, nameof(workflowInstanceId)); Guard.Against.NullOrWhiteSpace(bucketName, nameof(bucketName)); + Guard.Against.NullOrWhiteSpace(payloadId, nameof(payloadId)); var executionId = Guid.NewGuid().ToString(); @@ -318,8 +322,8 @@ private TaskExecution CreateTaskExecution(TaskObject task, string workflowInstan TaskPluginArguments = task.Args ?? new Dictionary { }, TaskId = task.Id, Status = TaskExecutionStatus.Created, - InputArtifacts = task.Artifacts?.Input?.ToDictionary() ?? new Dictionary { }, - OutputDirectory = $"{bucketName}/{workflowInstanceId}/{executionId}", + InputArtifacts = await _artifactMapper.ConvertArtifactVariablesToPath(task?.Artifacts?.Input ?? new Artifact[] { }, payloadId, workflowInstanceId), + OutputDirectory = $"{payloadId}/workflows/{workflowInstanceId}/{executionId}", Metadata = { } }; } diff --git a/src/WorkflowManager/Program.cs b/src/WorkflowManager/Program.cs index 9ff59e6a5..4988dfee7 100644 --- a/src/WorkflowManager/Program.cs +++ b/src/WorkflowManager/Program.cs @@ -27,6 +27,7 @@ using Monai.Deploy.WorkflowManager.PayloadListener.Validators; using Monai.Deploy.WorkflowManager.Services.DataRetentionService; using Monai.Deploy.WorkflowManager.Services.Http; +using Monai.Deploy.WorkloadManager.WorkfowExecuter.Common; using Monai.Deploy.WorkloadManager.WorkfowExecuter.Services; using MongoDB.Driver; @@ -126,6 +127,7 @@ internal static IHostBuilder CreateHostBuilder(string[] args) => services.AddSingleton(); services.AddTransient(); services.AddSingleton(); + services.AddSingleton(); services.AddSingleton(); diff --git a/tests/UnitTests/WorkflowExecuter.Tests/Common/ArtifactMapperTests.cs b/tests/UnitTests/WorkflowExecuter.Tests/Common/ArtifactMapperTests.cs new file mode 100644 index 000000000..7d02d505a --- /dev/null +++ b/tests/UnitTests/WorkflowExecuter.Tests/Common/ArtifactMapperTests.cs @@ -0,0 +1,99 @@ +// SPDX-FileCopyrightText: © 2021-2022 MONAI Consortium +// SPDX-License-Identifier: Apache License 2.0 + +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using FluentAssertions; +using Monai.Deploy.Messaging.Events; +using Monai.Deploy.WorkflowManager.Contracts.Models; +using Monai.Deploy.WorkflowManager.Database.Interfaces; +using Monai.Deploy.WorkloadManager.Contracts.Models; +using Monai.Deploy.WorkloadManager.WorkfowExecuter.Common; +using Moq; +using Xunit; + +namespace Monai.Deploy.WorkflowManager.WorkflowExecuter.Tests.Common +{ + public class ArtifactMapperTests + { + private IArtifactMapper ArtifactMapper { get; set; } + + private readonly Mock _workflowInstanceRepository; + public ArtifactMapperTests() + { + _workflowInstanceRepository = new Mock(); + + ArtifactMapper = new ArtifactMapper(_workflowInstanceRepository.Object); + } + + [Fact] + public async Task ConvertArtifactVariablesToPath_MultipleArtifacts_ReturnsMappedPaths() + { + var artifacts = new Artifact[] + { + new Artifact + { + Name = "taskartifact", + Value = "{{ context.executions.image_type_detector.artifacts.dicom }}" + }, + new Artifact + { + Name = "dicomimage", + Value = "{{ context.input }}" + }, + new Artifact + { + Name = "outputtaskdir", + Value = "{{ context.executions.coffee.output_dir }}" + } + }; + + var payloadId = Guid.NewGuid().ToString(); + var workflowInstanceId = Guid.NewGuid().ToString(); + var executionId = Guid.NewGuid().ToString(); + + var expected = new Dictionary + { + { "dicom", $"{payloadId}/workflows/{workflowInstanceId}/{executionId}" }, + { "dicomimage", $"{payloadId}/dcm" }, + { "outputtaskdir", $"{payloadId}/workflows/{workflowInstanceId}/{executionId}" } + }; + + var workflowInstance = new WorkflowInstance + { + Id = workflowInstanceId, + WorkflowId = Guid.NewGuid().ToString(), + PayloadId = payloadId, + Status = Status.Created, + BucketId = "bucket", + Tasks = new List + { + new TaskExecution + { + TaskId = "image_type_detector", + ExecutionId = executionId, + Status = TaskExecutionStatus.Dispatched, + OutputArtifacts = new Dictionary + { + { "dicom", $"{payloadId}/workflows/{workflowInstanceId}/{executionId}" } + } + }, + new TaskExecution + { + TaskId = "coffee", + Status = TaskExecutionStatus.Created, + OutputDirectory = $"{payloadId}/workflows/{workflowInstanceId}/{executionId}" + } + } + }; + + _workflowInstanceRepository.Setup(w => w.GetTaskByIdAsync(workflowInstance.Id, "image_type_detector")).ReturnsAsync(workflowInstance.Tasks[0]); + _workflowInstanceRepository.Setup(w => w.GetTaskByIdAsync(workflowInstance.Id, "coffee")).ReturnsAsync(workflowInstance.Tasks[1]); + + var response = await ArtifactMapper.ConvertArtifactVariablesToPath(artifacts, workflowInstance.PayloadId, workflowInstance.Id); + + response.Should().BeEquivalentTo(expected); + } + } +} diff --git a/tests/UnitTests/WorkflowExecuter.Tests/Services/WorkflowExecuterServiceTests.cs b/tests/UnitTests/WorkflowExecuter.Tests/Services/WorkflowExecuterServiceTests.cs index d30cec919..1f12c95dc 100644 --- a/tests/UnitTests/WorkflowExecuter.Tests/Services/WorkflowExecuterServiceTests.cs +++ b/tests/UnitTests/WorkflowExecuter.Tests/Services/WorkflowExecuterServiceTests.cs @@ -15,6 +15,7 @@ using Monai.Deploy.WorkflowManager.Contracts.Models; using Monai.Deploy.WorkflowManager.Database.Interfaces; using Monai.Deploy.WorkloadManager.Contracts.Models; +using Monai.Deploy.WorkloadManager.WorkfowExecuter.Common; using Monai.Deploy.WorkloadManager.WorkfowExecuter.Services; using Moq; using Xunit; @@ -26,6 +27,7 @@ public class WorkflowExecuterServiceTests private IWorkflowExecuterService WorkflowExecuterService { get; set; } private readonly Mock _workflowRepository; + private readonly Mock _artifactMapper; private readonly Mock> _logger; private readonly Mock _workflowInstanceRepository; private readonly Mock _messageBrokerPublisherService; @@ -35,13 +37,14 @@ public class WorkflowExecuterServiceTests public WorkflowExecuterServiceTests() { _workflowRepository = new Mock(); + _artifactMapper = new Mock(); _logger = new Mock>(); _workflowInstanceRepository = new Mock(); _messageBrokerPublisherService = new Mock(); _configuration = Options.Create(new WorkflowManagerOptions() { Messaging = new MessageBrokerConfiguration { Topics = new MessageBrokerConfigurationKeys { TaskDispatchRequest = "md.task.dispatch" } } }); _storageConfiguration = Options.Create(new StorageServiceConfiguration()); - WorkflowExecuterService = new WorkflowExecuterService(_logger.Object, _configuration, _storageConfiguration, _workflowRepository.Object, _workflowInstanceRepository.Object, _messageBrokerPublisherService.Object); + WorkflowExecuterService = new WorkflowExecuterService(_logger.Object, _configuration, _storageConfiguration, _workflowRepository.Object, _workflowInstanceRepository.Object, _messageBrokerPublisherService.Object, _artifactMapper.Object); } [Fact] @@ -173,6 +176,7 @@ public async Task ProcessPayload_ValidWorkflowIdRequest_ReturnesTrue() _workflowInstanceRepository.Setup(w => w.CreateAsync(It.IsAny>())).ReturnsAsync(true); _workflowInstanceRepository.Setup(w => w.GetByWorkflowsIdsAsync(It.IsAny>())).ReturnsAsync(new List()); _workflowInstanceRepository.Setup(w => w.UpdateTaskStatusAsync(It.IsAny(), It.IsAny(), It.IsAny())).ReturnsAsync(true); + _artifactMapper.Setup(a => a.ConvertArtifactVariablesToPath(It.IsAny(), It.IsAny(), It.IsAny())).ReturnsAsync(new Dictionary()); var result = await WorkflowExecuterService.ProcessPayload(workflowRequest); @@ -279,6 +283,7 @@ public async Task ProcessPayload_WorkflowAlreadyStarted_TaskNotDispatched() _workflowInstanceRepository.Setup(w => w.CreateAsync(It.IsAny>())).ReturnsAsync(true); _workflowInstanceRepository.Setup(w => w.GetByWorkflowsIdsAsync(It.IsAny>())).ReturnsAsync(workflowsInstance); _workflowInstanceRepository.Setup(w => w.UpdateTaskStatusAsync(It.IsAny(), It.IsAny(), It.IsAny())).ReturnsAsync(true); + _artifactMapper.Setup(a => a.ConvertArtifactVariablesToPath(It.IsAny(), It.IsAny(), It.IsAny())).ReturnsAsync(new Dictionary()); var result = await WorkflowExecuterService.ProcessPayload(workflowRequest); @@ -351,6 +356,7 @@ public async Task ProcessTaskUpdate_ValidTaskUpdateEvent_ReturnsTrue() _workflowInstanceRepository.Setup(w => w.UpdateTaskStatusAsync(It.IsAny(), It.IsAny(), It.IsAny())).ReturnsAsync(true); _workflowInstanceRepository.Setup(w => w.GetByWorkflowInstanceIdAsync(workflowInstance.Id)).ReturnsAsync(workflowInstance); _workflowRepository.Setup(w => w.GetByWorkflowIdAsync(workflowInstance.WorkflowId)).ReturnsAsync(workflow); + _artifactMapper.Setup(a => a.ConvertArtifactVariablesToPath(It.IsAny(), It.IsAny(), It.IsAny())).ReturnsAsync(new Dictionary()); var response = await WorkflowExecuterService.ProcessTaskUpdate(updateEvent); @@ -446,6 +452,7 @@ public async Task ProcessTaskUpdate_ValidTaskUpdateEventWithTaskDestinations_Ret _workflowInstanceRepository.Setup(w => w.GetByWorkflowInstanceIdAsync(workflowInstance.Id)).ReturnsAsync(workflowInstance); _workflowInstanceRepository.Setup(w => w.UpdateTasksAsync(workflowInstance.Id, It.IsAny>())).ReturnsAsync(true); _workflowRepository.Setup(w => w.GetByWorkflowIdAsync(workflowInstance.WorkflowId)).ReturnsAsync(workflow); + _artifactMapper.Setup(a => a.ConvertArtifactVariablesToPath(It.IsAny(), It.IsAny(), It.IsAny())).ReturnsAsync(new Dictionary()); var response = await WorkflowExecuterService.ProcessTaskUpdate(updateEvent); From 599a45f7d9ee993d1fe94ee9332dc695cb915c09 Mon Sep 17 00:00:00 2001 From: Jack Schofield Date: Thu, 19 May 2022 14:14:51 +0100 Subject: [PATCH 2/9] update messaging package Signed-off-by: Jack Schofield --- ...i.Deploy.WorkflowManager.Configuration.csproj | 5 ++--- src/Configuration/packages.lock.json | 6 +++--- ...Monai.Deploy.WorkflowManager.Contracts.csproj | 2 +- .../Monai.Deploy.WorkflowManager.Database.csproj | 2 +- src/Database/packages.lock.json | 8 ++++---- ...Deploy.WorkflowManager.PayloadListener.csproj | 2 +- ...Deploy.WorkflowManager.TaskManager.API.csproj | 3 +-- src/TaskManager/API/packages.lock.json | 6 +++--- ...nai.Deploy.WorkflowManager.TaskManager.csproj | 3 +-- ...eploy.WorkflowManager.TaskManager.Argo.csproj | 3 +-- src/TaskManager/Plug-ins/Argo/packages.lock.json | 8 ++++---- src/TaskManager/packages.lock.json | 10 +++++----- ...Deploy.WorkloadManager.WorkfowExecuter.csproj | 2 +- .../Monai.Deploy.WorkflowManager.csproj | 2 +- src/WorkflowManager/packages.lock.json | 16 ++++++++-------- ...oy.WorkflowManager.Configuration.Tests.csproj | 2 +- ....WorkflowManager.PayloadListener.Tests.csproj | 2 +- .../WorkflowManager.Tests/packages.lock.json | 16 ++++++++-------- 18 files changed, 47 insertions(+), 51 deletions(-) diff --git a/src/Configuration/Monai.Deploy.WorkflowManager.Configuration.csproj b/src/Configuration/Monai.Deploy.WorkflowManager.Configuration.csproj index 54050a155..9c1498df7 100644 --- a/src/Configuration/Monai.Deploy.WorkflowManager.Configuration.csproj +++ b/src/Configuration/Monai.Deploy.WorkflowManager.Configuration.csproj @@ -15,7 +15,7 @@ SPDX-License-Identifier: Apache License 2.0 - + @@ -29,10 +29,9 @@ SPDX-License-Identifier: Apache License 2.0 - + true - true true diff --git a/src/Configuration/packages.lock.json b/src/Configuration/packages.lock.json index b206b78e5..b35e6dbe1 100644 --- a/src/Configuration/packages.lock.json +++ b/src/Configuration/packages.lock.json @@ -38,9 +38,9 @@ }, "Monai.Deploy.Storage": { "type": "Direct", - "requested": "[0.1.0-rc0020, )", - "resolved": "0.1.0-rc0020", - "contentHash": "SNNzbGPte4S2CKzsYI5g723aRMNG8AVFNYR847hHl8JQXCD5f6ugxEJUid4/hpu88HxPPigDm7sMrc5H4EwtZQ==", + "requested": "[0.1.0-rc0019, )", + "resolved": "0.1.0-rc0019", + "contentHash": "YAXgTE7vTqeMc+ZhprUg/muQ+JDVMaix1Ha1vLJTSSjfZWBOnxyK5SugS50uNlvByJcqHEXh+ygd9219wK1S+Q==", "dependencies": { "AWSSDK.SecurityToken": "3.7.1.141", "Ardalis.GuardClauses": "4.0.0", diff --git a/src/Contracts/Monai.Deploy.WorkflowManager.Contracts.csproj b/src/Contracts/Monai.Deploy.WorkflowManager.Contracts.csproj index 8e6e4f7d8..f88d2a965 100644 --- a/src/Contracts/Monai.Deploy.WorkflowManager.Contracts.csproj +++ b/src/Contracts/Monai.Deploy.WorkflowManager.Contracts.csproj @@ -18,7 +18,7 @@ SPDX-License-Identifier: Apache License 2.0 - + diff --git a/src/Database/Monai.Deploy.WorkflowManager.Database.csproj b/src/Database/Monai.Deploy.WorkflowManager.Database.csproj index c95ddae26..626b475b9 100644 --- a/src/Database/Monai.Deploy.WorkflowManager.Database.csproj +++ b/src/Database/Monai.Deploy.WorkflowManager.Database.csproj @@ -20,7 +20,7 @@ - + diff --git a/src/Database/packages.lock.json b/src/Database/packages.lock.json index e216297bd..5e90c91ea 100644 --- a/src/Database/packages.lock.json +++ b/src/Database/packages.lock.json @@ -29,9 +29,9 @@ }, "Monai.Deploy.Messaging": { "type": "Direct", - "requested": "[0.1.0-rc0036, )", - "resolved": "0.1.0-rc0036", - "contentHash": "ONfPbkd0bsJGJHOPVn7U5z0FDNirYuvMswLauN8rxHNSat0GbhFDnObrY8iIuCFY02zfPfpoL9mywkksVwsUAA==", + "requested": "[0.1.0-rc0038, )", + "resolved": "0.1.0-rc0038", + "contentHash": "j6RswItc5GE7b42T1KMMT9oXsOEizr6HKgso4WAV/evGmcwMnzuBdRJHNIfGWeDTTKO7Ld7+BJrSOEno5+udsg==", "dependencies": { "Ardalis.GuardClauses": "4.0.1", "Microsoft.Extensions.Configuration": "6.0.1", @@ -240,7 +240,7 @@ "dependencies": { "AWSSDK.SecurityToken": "3.7.1.141", "Microsoft.Extensions.Configuration": "6.0.1", - "Monai.Deploy.Messaging": "0.1.0-rc0036", + "Monai.Deploy.Messaging": "0.1.0-rc0038", "MongoDB.Bson": "2.15.0", "Newtonsoft.Json": "13.0.1" } diff --git a/src/PayloadListener/Monai.Deploy.WorkflowManager.PayloadListener.csproj b/src/PayloadListener/Monai.Deploy.WorkflowManager.PayloadListener.csproj index 0d60d833e..055f5f33d 100644 --- a/src/PayloadListener/Monai.Deploy.WorkflowManager.PayloadListener.csproj +++ b/src/PayloadListener/Monai.Deploy.WorkflowManager.PayloadListener.csproj @@ -14,7 +14,7 @@ SPDX-License-Identifier: Apache License 2.0 - + diff --git a/src/TaskManager/API/Monai.Deploy.WorkflowManager.TaskManager.API.csproj b/src/TaskManager/API/Monai.Deploy.WorkflowManager.TaskManager.API.csproj index cd5dce2ab..cf52da543 100644 --- a/src/TaskManager/API/Monai.Deploy.WorkflowManager.TaskManager.API.csproj +++ b/src/TaskManager/API/Monai.Deploy.WorkflowManager.TaskManager.API.csproj @@ -13,7 +13,6 @@ SPDX-License-Identifier: Apache License 2.0 true - true true ..\..\.sonarlint\project-monai_monai-deploy-workflow-managercsharp.ruleset @@ -23,7 +22,7 @@ SPDX-License-Identifier: Apache License 2.0 - + diff --git a/src/TaskManager/API/packages.lock.json b/src/TaskManager/API/packages.lock.json index 16ed3b757..40d680f47 100644 --- a/src/TaskManager/API/packages.lock.json +++ b/src/TaskManager/API/packages.lock.json @@ -18,9 +18,9 @@ }, "Monai.Deploy.Storage": { "type": "Direct", - "requested": "[0.1.0-rc0020, )", - "resolved": "0.1.0-rc0020", - "contentHash": "SNNzbGPte4S2CKzsYI5g723aRMNG8AVFNYR847hHl8JQXCD5f6ugxEJUid4/hpu88HxPPigDm7sMrc5H4EwtZQ==", + "requested": "[0.1.0-rc0019, )", + "resolved": "0.1.0-rc0019", + "contentHash": "YAXgTE7vTqeMc+ZhprUg/muQ+JDVMaix1Ha1vLJTSSjfZWBOnxyK5SugS50uNlvByJcqHEXh+ygd9219wK1S+Q==", "dependencies": { "AWSSDK.SecurityToken": "3.7.1.141", "Ardalis.GuardClauses": "4.0.0", diff --git a/src/TaskManager/Monai.Deploy.WorkflowManager.TaskManager.csproj b/src/TaskManager/Monai.Deploy.WorkflowManager.TaskManager.csproj index 4748f0f08..7c4377a41 100644 --- a/src/TaskManager/Monai.Deploy.WorkflowManager.TaskManager.csproj +++ b/src/TaskManager/Monai.Deploy.WorkflowManager.TaskManager.csproj @@ -38,7 +38,7 @@ SPDX-License-Identifier: Apache License 2.0 - + @@ -50,7 +50,6 @@ SPDX-License-Identifier: Apache License 2.0 true - true true ..\.sonarlint\project-monai_monai-deploy-workflow-managercsharp.ruleset diff --git a/src/TaskManager/Plug-ins/Argo/Monai.Deploy.WorkflowManager.TaskManager.Argo.csproj b/src/TaskManager/Plug-ins/Argo/Monai.Deploy.WorkflowManager.TaskManager.Argo.csproj index 308b47a3e..fb8342811 100644 --- a/src/TaskManager/Plug-ins/Argo/Monai.Deploy.WorkflowManager.TaskManager.Argo.csproj +++ b/src/TaskManager/Plug-ins/Argo/Monai.Deploy.WorkflowManager.TaskManager.Argo.csproj @@ -22,7 +22,7 @@ SPDX-License-Identifier: Apache License 2.0 all runtime; build; native; contentfiles; analyzers; buildtransitive - + @@ -34,7 +34,6 @@ SPDX-License-Identifier: Apache License 2.0 true - true true ..\..\..\.sonarlint\project-monai_monai-deploy-workflow-managercsharp.ruleset diff --git a/src/TaskManager/Plug-ins/Argo/packages.lock.json b/src/TaskManager/Plug-ins/Argo/packages.lock.json index 7a9dda53e..7d38750a9 100644 --- a/src/TaskManager/Plug-ins/Argo/packages.lock.json +++ b/src/TaskManager/Plug-ins/Argo/packages.lock.json @@ -48,9 +48,9 @@ }, "Monai.Deploy.Storage": { "type": "Direct", - "requested": "[0.1.0-rc0020, )", - "resolved": "0.1.0-rc0020", - "contentHash": "SNNzbGPte4S2CKzsYI5g723aRMNG8AVFNYR847hHl8JQXCD5f6ugxEJUid4/hpu88HxPPigDm7sMrc5H4EwtZQ==", + "requested": "[0.1.0-rc0019, )", + "resolved": "0.1.0-rc0019", + "contentHash": "YAXgTE7vTqeMc+ZhprUg/muQ+JDVMaix1Ha1vLJTSSjfZWBOnxyK5SugS50uNlvByJcqHEXh+ygd9219wK1S+Q==", "dependencies": { "AWSSDK.SecurityToken": "3.7.1.141", "Ardalis.GuardClauses": "4.0.0", @@ -629,7 +629,7 @@ "type": "Project", "dependencies": { "Monai.Deploy.Messaging": "0.1.0-rc0038", - "Monai.Deploy.Storage": "0.1.0-rc0020" + "Monai.Deploy.Storage": "0.1.0-rc0019" } } } diff --git a/src/TaskManager/packages.lock.json b/src/TaskManager/packages.lock.json index 8cfac393f..2a3c2c523 100644 --- a/src/TaskManager/packages.lock.json +++ b/src/TaskManager/packages.lock.json @@ -74,9 +74,9 @@ }, "Monai.Deploy.Storage": { "type": "Direct", - "requested": "[0.1.0-rc0020, )", - "resolved": "0.1.0-rc0020", - "contentHash": "SNNzbGPte4S2CKzsYI5g723aRMNG8AVFNYR847hHl8JQXCD5f6ugxEJUid4/hpu88HxPPigDm7sMrc5H4EwtZQ==", + "requested": "[0.1.0-rc0019, )", + "resolved": "0.1.0-rc0019", + "contentHash": "YAXgTE7vTqeMc+ZhprUg/muQ+JDVMaix1Ha1vLJTSSjfZWBOnxyK5SugS50uNlvByJcqHEXh+ygd9219wK1S+Q==", "dependencies": { "AWSSDK.SecurityToken": "3.7.1.141", "Ardalis.GuardClauses": "4.0.0", @@ -563,7 +563,7 @@ "Microsoft.Extensions.Configuration": "6.0.1", "Microsoft.Extensions.Options": "6.0.0", "Monai.Deploy.Messaging": "0.1.0-rc0038", - "Monai.Deploy.Storage": "0.1.0-rc0020", + "Monai.Deploy.Storage": "0.1.0-rc0019", "Newtonsoft.Json": "13.0.1", "System.IO.Abstractions": "16.1.25" } @@ -601,7 +601,7 @@ "type": "Project", "dependencies": { "Monai.Deploy.Messaging": "0.1.0-rc0038", - "Monai.Deploy.Storage": "0.1.0-rc0020" + "Monai.Deploy.Storage": "0.1.0-rc0019" } } } diff --git a/src/WorkflowExecuter/Monai.Deploy.WorkloadManager.WorkfowExecuter.csproj b/src/WorkflowExecuter/Monai.Deploy.WorkloadManager.WorkfowExecuter.csproj index 769d367fa..16d0af93b 100644 --- a/src/WorkflowExecuter/Monai.Deploy.WorkloadManager.WorkfowExecuter.csproj +++ b/src/WorkflowExecuter/Monai.Deploy.WorkloadManager.WorkfowExecuter.csproj @@ -8,7 +8,7 @@ - + diff --git a/src/WorkflowManager/Monai.Deploy.WorkflowManager.csproj b/src/WorkflowManager/Monai.Deploy.WorkflowManager.csproj index 883e9f057..e98bf5f70 100644 --- a/src/WorkflowManager/Monai.Deploy.WorkflowManager.csproj +++ b/src/WorkflowManager/Monai.Deploy.WorkflowManager.csproj @@ -34,7 +34,7 @@ SPDX-License-Identifier: Apache License 2.0 - + all diff --git a/src/WorkflowManager/packages.lock.json b/src/WorkflowManager/packages.lock.json index d9c2716f8..ec6aae743 100644 --- a/src/WorkflowManager/packages.lock.json +++ b/src/WorkflowManager/packages.lock.json @@ -138,9 +138,9 @@ }, "Monai.Deploy.Messaging": { "type": "Direct", - "requested": "[0.1.0-rc0036, )", - "resolved": "0.1.0-rc0036", - "contentHash": "ONfPbkd0bsJGJHOPVn7U5z0FDNirYuvMswLauN8rxHNSat0GbhFDnObrY8iIuCFY02zfPfpoL9mywkksVwsUAA==", + "requested": "[0.1.0-rc0038, )", + "resolved": "0.1.0-rc0038", + "contentHash": "j6RswItc5GE7b42T1KMMT9oXsOEizr6HKgso4WAV/evGmcwMnzuBdRJHNIfGWeDTTKO7Ld7+BJrSOEno5+udsg==", "dependencies": { "Ardalis.GuardClauses": "4.0.1", "Microsoft.Extensions.Configuration": "6.0.1", @@ -721,7 +721,7 @@ "dependencies": { "Microsoft.Extensions.Configuration": "6.0.1", "Microsoft.Extensions.Options": "6.0.0", - "Monai.Deploy.Messaging": "0.1.0-rc0036", + "Monai.Deploy.Messaging": "0.1.0-rc0038", "Monai.Deploy.Storage": "0.1.0-rc0019", "Newtonsoft.Json": "13.0.1", "System.IO.Abstractions": "16.1.25" @@ -732,7 +732,7 @@ "dependencies": { "AWSSDK.SecurityToken": "3.7.1.141", "Microsoft.Extensions.Configuration": "6.0.1", - "Monai.Deploy.Messaging": "0.1.0-rc0036", + "Monai.Deploy.Messaging": "0.1.0-rc0038", "MongoDB.Bson": "2.15.0", "Newtonsoft.Json": "13.0.1" } @@ -743,7 +743,7 @@ "Ardalis.GuardClauses": "4.0.1", "Microsoft.Extensions.Logging.Abstractions": "6.0.1", "Microsoft.Extensions.Options": "6.0.0", - "Monai.Deploy.Messaging": "0.1.0-rc0036", + "Monai.Deploy.Messaging": "0.1.0-rc0038", "Monai.Deploy.WorkflowManager.Contracts": "1.0.0", "Monai.Deploy.WorkflowManager.Logging": "1.0.0", "MongoDB.Bson": "2.15.0", @@ -761,7 +761,7 @@ "dependencies": { "Ardalis.GuardClauses": "4.0.1", "Microsoft.Extensions.Hosting.Abstractions": "6.0.0", - "Monai.Deploy.Messaging": "0.1.0-rc0036", + "Monai.Deploy.Messaging": "0.1.0-rc0038", "Monai.Deploy.Storage": "0.1.0-rc0019", "Monai.Deploy.WorkflowManager.Common": "1.0.0", "Monai.Deploy.WorkflowManager.Configuration": "1.0.0", @@ -775,7 +775,7 @@ "dependencies": { "AWSSDK.SecurityToken": "3.7.1.141", "Ardalis.GuardClauses": "4.0.1", - "Monai.Deploy.Messaging": "0.1.0-rc0036", + "Monai.Deploy.Messaging": "0.1.0-rc0038", "Monai.Deploy.Storage": "0.1.0-rc0019", "Monai.Deploy.WorkflowManager.Configuration": "1.0.0", "Monai.Deploy.WorkflowManager.Contracts": "1.0.0", diff --git a/tests/UnitTests/Configuration.Tests/Monai.Deploy.WorkflowManager.Configuration.Tests.csproj b/tests/UnitTests/Configuration.Tests/Monai.Deploy.WorkflowManager.Configuration.Tests.csproj index 5a4e186d1..576543024 100644 --- a/tests/UnitTests/Configuration.Tests/Monai.Deploy.WorkflowManager.Configuration.Tests.csproj +++ b/tests/UnitTests/Configuration.Tests/Monai.Deploy.WorkflowManager.Configuration.Tests.csproj @@ -18,7 +18,7 @@ SPDX-License-Identifier: Apache License 2.0 runtime; build; native; contentfiles; analyzers; buildtransitive - + diff --git a/tests/UnitTests/PayloadListener.Tests/Monai.Deploy.WorkflowManager.PayloadListener.Tests.csproj b/tests/UnitTests/PayloadListener.Tests/Monai.Deploy.WorkflowManager.PayloadListener.Tests.csproj index fb692bef8..8ef7dc081 100644 --- a/tests/UnitTests/PayloadListener.Tests/Monai.Deploy.WorkflowManager.PayloadListener.Tests.csproj +++ b/tests/UnitTests/PayloadListener.Tests/Monai.Deploy.WorkflowManager.PayloadListener.Tests.csproj @@ -18,7 +18,7 @@ SPDX-License-Identifier: Apache License 2.0 - + diff --git a/tests/UnitTests/WorkflowManager.Tests/packages.lock.json b/tests/UnitTests/WorkflowManager.Tests/packages.lock.json index a52d2781b..87c3db612 100644 --- a/tests/UnitTests/WorkflowManager.Tests/packages.lock.json +++ b/tests/UnitTests/WorkflowManager.Tests/packages.lock.json @@ -592,8 +592,8 @@ }, "Monai.Deploy.Messaging": { "type": "Transitive", - "resolved": "0.1.0-rc0036", - "contentHash": "ONfPbkd0bsJGJHOPVn7U5z0FDNirYuvMswLauN8rxHNSat0GbhFDnObrY8iIuCFY02zfPfpoL9mywkksVwsUAA==", + "resolved": "0.1.0-rc0038", + "contentHash": "j6RswItc5GE7b42T1KMMT9oXsOEizr6HKgso4WAV/evGmcwMnzuBdRJHNIfGWeDTTKO7Ld7+BJrSOEno5+udsg==", "dependencies": { "Ardalis.GuardClauses": "4.0.1", "Microsoft.Extensions.Configuration": "6.0.1", @@ -1376,7 +1376,7 @@ "Microsoft.Extensions.Logging": "6.0.0", "Microsoft.Extensions.Logging.Console": "6.0.0", "Microsoft.Extensions.Options": "6.0.0", - "Monai.Deploy.Messaging": "0.1.0-rc0036", + "Monai.Deploy.Messaging": "0.1.0-rc0038", "Monai.Deploy.WorkflowManager.Common": "1.0.0", "Monai.Deploy.WorkflowManager.Configuration": "1.0.0", "Monai.Deploy.WorkflowManager.Contracts": "1.0.0", @@ -1401,7 +1401,7 @@ "dependencies": { "Microsoft.Extensions.Configuration": "6.0.1", "Microsoft.Extensions.Options": "6.0.0", - "Monai.Deploy.Messaging": "0.1.0-rc0036", + "Monai.Deploy.Messaging": "0.1.0-rc0038", "Monai.Deploy.Storage": "0.1.0-rc0019", "Newtonsoft.Json": "13.0.1", "System.IO.Abstractions": "16.1.25" @@ -1412,7 +1412,7 @@ "dependencies": { "AWSSDK.SecurityToken": "3.7.1.141", "Microsoft.Extensions.Configuration": "6.0.1", - "Monai.Deploy.Messaging": "0.1.0-rc0036", + "Monai.Deploy.Messaging": "0.1.0-rc0038", "MongoDB.Bson": "2.15.0", "Newtonsoft.Json": "13.0.1" } @@ -1423,7 +1423,7 @@ "Ardalis.GuardClauses": "4.0.1", "Microsoft.Extensions.Logging.Abstractions": "6.0.1", "Microsoft.Extensions.Options": "6.0.0", - "Monai.Deploy.Messaging": "0.1.0-rc0036", + "Monai.Deploy.Messaging": "0.1.0-rc0038", "Monai.Deploy.WorkflowManager.Contracts": "1.0.0", "Monai.Deploy.WorkflowManager.Logging": "1.0.0", "MongoDB.Bson": "2.15.0", @@ -1441,7 +1441,7 @@ "dependencies": { "Ardalis.GuardClauses": "4.0.1", "Microsoft.Extensions.Hosting.Abstractions": "6.0.0", - "Monai.Deploy.Messaging": "0.1.0-rc0036", + "Monai.Deploy.Messaging": "0.1.0-rc0038", "Monai.Deploy.Storage": "0.1.0-rc0019", "Monai.Deploy.WorkflowManager.Common": "1.0.0", "Monai.Deploy.WorkflowManager.Configuration": "1.0.0", @@ -1455,7 +1455,7 @@ "dependencies": { "AWSSDK.SecurityToken": "3.7.1.141", "Ardalis.GuardClauses": "4.0.1", - "Monai.Deploy.Messaging": "0.1.0-rc0036", + "Monai.Deploy.Messaging": "0.1.0-rc0038", "Monai.Deploy.Storage": "0.1.0-rc0019", "Monai.Deploy.WorkflowManager.Configuration": "1.0.0", "Monai.Deploy.WorkflowManager.Contracts": "1.0.0", From 7b9bcf3cfabefdd4836f68febb94409d2c939131 Mon Sep 17 00:00:00 2001 From: Jack Schofield Date: Fri, 20 May 2022 13:25:44 +0100 Subject: [PATCH 3/9] add storage check for artifacts Signed-off-by: Jack Schofield --- .../Interfaces/IWorkflowInstanceRepository.cs | 8 ++ src/Database/WorkflowInstanceRepository.cs | 20 ++++ .../Services/WorkflowExecuterService.cs | 18 ++- .../Services/WorkflowExecuterServiceTests.cs | 109 +++++++++++++++++- 4 files changed, 153 insertions(+), 2 deletions(-) diff --git a/src/Database/Interfaces/IWorkflowInstanceRepository.cs b/src/Database/Interfaces/IWorkflowInstanceRepository.cs index a5916fd32..4a4ea4ac8 100644 --- a/src/Database/Interfaces/IWorkflowInstanceRepository.cs +++ b/src/Database/Interfaces/IWorkflowInstanceRepository.cs @@ -37,6 +37,14 @@ public interface IWorkflowInstanceRepository /// Status to set. Task UpdateTaskStatusAsync(string workflowInstanceId, string taskId, TaskExecutionStatus status); + /// + /// Updates the Task output artifacts for a given task within a workflow instance. + /// + /// Workflow Instance to update. + /// TaskId to update. + /// Output artifacts to set. + Task UpdateTaskOutputArtifactsAsync(string workflowInstanceId, string taskId, Dictionary outputArtifacts); + /// /// Gets a task execution for a given workflow instance id and task id. /// diff --git a/src/Database/WorkflowInstanceRepository.cs b/src/Database/WorkflowInstanceRepository.cs index bb31121ce..052989e21 100644 --- a/src/Database/WorkflowInstanceRepository.cs +++ b/src/Database/WorkflowInstanceRepository.cs @@ -95,6 +95,26 @@ public async Task UpdateTaskStatusAsync(string workflowInstanceId, string } } + public async Task UpdateTaskOutputArtifactsAsync(string workflowInstanceId, string taskId, Dictionary outputArtifacts) + { + Guard.Against.NullOrWhiteSpace(workflowInstanceId, nameof(workflowInstanceId)); + Guard.Against.NullOrWhiteSpace(taskId, nameof(taskId)); + Guard.Against.Null(outputArtifacts, nameof(outputArtifacts)); + + try + { + var result = await _workflowInstanceCollection.FindOneAndUpdateAsync( + i => i.Id == workflowInstanceId && i.Tasks.Any(t => t.TaskId == taskId), + Builders.Update.Set(w => w.Tasks[-1].OutputArtifacts, outputArtifacts)); + return true; + } + catch (Exception e) + { + _logger.DbCallFailed(nameof(UpdateTaskOutputArtifactsAsync), e); + return false; + } + } + public async Task UpdateWorkflowInstanceStatusAsync(string workflowInstanceId, Status status) { Guard.Against.NullOrWhiteSpace(workflowInstanceId, nameof(workflowInstanceId)); diff --git a/src/WorkflowExecuter/Services/WorkflowExecuterService.cs b/src/WorkflowExecuter/Services/WorkflowExecuterService.cs index f4720385d..4dad2fac5 100644 --- a/src/WorkflowExecuter/Services/WorkflowExecuterService.cs +++ b/src/WorkflowExecuter/Services/WorkflowExecuterService.cs @@ -7,6 +7,7 @@ using Monai.Deploy.Messaging; using Monai.Deploy.Messaging.Events; using Monai.Deploy.Messaging.Messages; +using Monai.Deploy.Storage; using Monai.Deploy.Storage.Configuration; using Monai.Deploy.WorkflowManager.Configuration; using Monai.Deploy.WorkflowManager.Contracts.Models; @@ -24,6 +25,7 @@ public class WorkflowExecuterService : IWorkflowExecuterService private readonly IWorkflowInstanceRepository _workflowInstanceRepository; private readonly IMessageBrokerPublisherService _messageBrokerPublisherService; private readonly IArtifactMapper _artifactMapper; + private readonly IStorageService _storageService; private readonly StorageServiceConfiguration _storageConfiguration; private string TaskDispatchRoutingKey { get; } @@ -35,7 +37,8 @@ public WorkflowExecuterService( IWorkflowRepository workflowRepository, IWorkflowInstanceRepository workflowInstanceRepository, IMessageBrokerPublisherService messageBrokerPublisherService, - IArtifactMapper artifactMapper) + IArtifactMapper artifactMapper, + IStorageService storageService) { if (configuration is null) { @@ -56,6 +59,7 @@ public WorkflowExecuterService( _workflowInstanceRepository = workflowInstanceRepository ?? throw new ArgumentNullException(nameof(workflowInstanceRepository)); _messageBrokerPublisherService = messageBrokerPublisherService ?? throw new ArgumentNullException(nameof(messageBrokerPublisherService)); _artifactMapper = artifactMapper ?? throw new ArgumentNullException(nameof(artifactMapper)); + _storageService = storageService ?? throw new ArgumentNullException(nameof(storageService)); } public async Task ProcessPayload(WorkflowRequestEvent message) @@ -156,6 +160,16 @@ public async Task ProcessTaskUpdate(TaskUpdateEvent message) return false; } + var validOutputArtifacts = await _storageService.VerifyObjectsExist(workflowInstance.BucketId, message.OutputArtifacts); + + workflowInstance.Tasks?.ForEach(t => + { + if (t.TaskId == message.TaskId) + { + t.OutputArtifacts = validOutputArtifacts; + } + }); + var currentTaskDestinations = workflow.Workflow?.Tasks?.SingleOrDefault(t => t.Id == message.TaskId)?.TaskDestinations; var newTaskExecutions = await HandleTaskDestinations(workflowInstance, workflow, currentTaskDestinations); @@ -163,6 +177,8 @@ public async Task ProcessTaskUpdate(TaskUpdateEvent message) { await UpdateWorkflowInstanceStatus(workflowInstance, message.TaskId, message.Status); + await _workflowInstanceRepository.UpdateTaskOutputArtifactsAsync(workflowInstance.Id, message.TaskId, validOutputArtifacts); + return await _workflowInstanceRepository.UpdateTaskStatusAsync(message.WorkflowInstanceId, message.TaskId, message.Status); } diff --git a/tests/UnitTests/WorkflowExecuter.Tests/Services/WorkflowExecuterServiceTests.cs b/tests/UnitTests/WorkflowExecuter.Tests/Services/WorkflowExecuterServiceTests.cs index 1f12c95dc..14bac99c3 100644 --- a/tests/UnitTests/WorkflowExecuter.Tests/Services/WorkflowExecuterServiceTests.cs +++ b/tests/UnitTests/WorkflowExecuter.Tests/Services/WorkflowExecuterServiceTests.cs @@ -10,6 +10,7 @@ using Monai.Deploy.Messaging; using Monai.Deploy.Messaging.Events; using Monai.Deploy.Messaging.Messages; +using Monai.Deploy.Storage; using Monai.Deploy.Storage.Configuration; using Monai.Deploy.WorkflowManager.Configuration; using Monai.Deploy.WorkflowManager.Contracts.Models; @@ -31,6 +32,7 @@ public class WorkflowExecuterServiceTests private readonly Mock> _logger; private readonly Mock _workflowInstanceRepository; private readonly Mock _messageBrokerPublisherService; + private readonly Mock _storageService; private readonly IOptions _configuration; private readonly IOptions _storageConfiguration; @@ -41,10 +43,11 @@ public WorkflowExecuterServiceTests() _logger = new Mock>(); _workflowInstanceRepository = new Mock(); _messageBrokerPublisherService = new Mock(); + _storageService = new Mock(); _configuration = Options.Create(new WorkflowManagerOptions() { Messaging = new MessageBrokerConfiguration { Topics = new MessageBrokerConfigurationKeys { TaskDispatchRequest = "md.task.dispatch" } } }); _storageConfiguration = Options.Create(new StorageServiceConfiguration()); - WorkflowExecuterService = new WorkflowExecuterService(_logger.Object, _configuration, _storageConfiguration, _workflowRepository.Object, _workflowInstanceRepository.Object, _messageBrokerPublisherService.Object, _artifactMapper.Object); + WorkflowExecuterService = new WorkflowExecuterService(_logger.Object, _configuration, _storageConfiguration, _workflowRepository.Object, _workflowInstanceRepository.Object, _messageBrokerPublisherService.Object, _artifactMapper.Object, _storageService.Object); } [Fact] @@ -652,6 +655,110 @@ public async Task ProcessTaskUpdate_ValidTaskUpdateEventOneTaskDestinationDispat response.Should().BeTrue(); } + [Fact] + public async Task ProcessTaskUpdate_ValidTaskUpdateEventWithOutputArtifacts_ReturnsTrue() + { + var workflowInstanceId = Guid.NewGuid().ToString(); + + var updateEvent = new TaskUpdateEvent + { + WorkflowInstanceId = workflowInstanceId, + TaskId = "pizza", + ExecutionId = Guid.NewGuid().ToString(), + Status = TaskExecutionStatus.Succeeded, + Reason = FailureReason.None, + Message = "This is a message", + Metadata = new Dictionary(), + CorrelationId = Guid.NewGuid().ToString(), + OutputArtifacts = new Dictionary + { + { "artifact.txt", "path/to/artifact" } + } + }; + + var workflowId = Guid.NewGuid().ToString(); + + var workflow = new WorkflowRevision + { + Id = Guid.NewGuid().ToString(), + WorkflowId = workflowId, + Revision = 1, + Workflow = new Workflow + { + Name = "Workflowname2", + Description = "Workflowdesc2", + Version = "1", + InformaticsGateway = new InformaticsGateway + { + AeTitle = "aetitle" + }, + Tasks = new TaskObject[] + { + new TaskObject { + Id = "pizza", + Type = "type", + Description = "taskdesc", + TaskDestinations = new TaskDestination[] + { + new TaskDestination + { + Name = "coffee" + }, + new TaskDestination + { + Name = "doughnuts" + } + } + }, + new TaskObject { + Id = "coffee", + Type = "type", + Description = "taskdesc" + }, + new TaskObject { + Id = "doughnuts", + Type = "type", + Description = "taskdesc" + } + } + } + }; + + var workflowInstance = new WorkflowInstance + { + Id = workflowInstanceId, + WorkflowId = workflowId, + PayloadId = Guid.NewGuid().ToString(), + Status = Status.Created, + BucketId = "bucket", + Tasks = new List + { + new TaskExecution + { + TaskId = "pizza", + Status = TaskExecutionStatus.Dispatched + }, + new TaskExecution + { + TaskId = "coffee", + Status = TaskExecutionStatus.Dispatched + } + } + }; + + _workflowInstanceRepository.Setup(w => w.UpdateTaskStatusAsync(It.IsAny(), It.IsAny(), It.IsAny())).ReturnsAsync(true); + _workflowInstanceRepository.Setup(w => w.GetByWorkflowInstanceIdAsync(workflowInstance.Id)).ReturnsAsync(workflowInstance); + _workflowInstanceRepository.Setup(w => w.UpdateTasksAsync(workflowInstance.Id, It.IsAny>())).ReturnsAsync(true); + _workflowRepository.Setup(w => w.GetByWorkflowIdAsync(workflowInstance.WorkflowId)).ReturnsAsync(workflow); + _storageService.Setup(w => w.VerifyObjectsExist(workflowInstance.BucketId, updateEvent.OutputArtifacts)).ReturnsAsync(updateEvent.OutputArtifacts); + + var response = await WorkflowExecuterService.ProcessTaskUpdate(updateEvent); + + _messageBrokerPublisherService.Verify(w => w.Publish(_configuration.Value.Messaging.Topics.TaskDispatchRequest, It.IsAny()), Times.Exactly(1)); + + response.Should().BeTrue(); + } + [Fact] public async Task ProcessTaskUpdate_ValidTaskUpdateEventWorkflowDoesNotExist_ReturnsTrue() { From 384cfde4fa140d267c9748d3aebb9c6976259ba8 Mon Sep 17 00:00:00 2001 From: Jack Schofield Date: Mon, 23 May 2022 08:52:22 +0100 Subject: [PATCH 4/9] update messaging package Signed-off-by: Jack Schofield --- ...eploy.WorkflowManager.Configuration.csproj | 2 +- src/Configuration/packages.lock.json | 6 +- ...loy.WorkflowManager.PayloadListener.csproj | 2 +- ...loy.WorkflowManager.TaskManager.API.csproj | 2 +- src/TaskManager/API/packages.lock.json | 6 +- ....Deploy.WorkflowManager.TaskManager.csproj | 2 +- ...oy.WorkflowManager.TaskManager.Argo.csproj | 2 +- .../Plug-ins/Argo/packages.lock.json | 8 +-- src/TaskManager/packages.lock.json | 10 +-- src/WorkflowExecuter/Common/ArtifactMapper.cs | 5 ++ ...loy.WorkloadManager.WorkfowExecuter.csproj | 2 +- .../Services/WorkflowExecuterService.cs | 2 +- src/WorkflowManager/packages.lock.json | 10 +-- .../TestData/WorkflowInstanceTestData.cs | 1 + .../Common/ArtifactMapperTests.cs | 64 +++++++++++++++++++ .../WorkflowManager.Tests/packages.lock.json | 10 +-- 16 files changed, 102 insertions(+), 32 deletions(-) diff --git a/src/Configuration/Monai.Deploy.WorkflowManager.Configuration.csproj b/src/Configuration/Monai.Deploy.WorkflowManager.Configuration.csproj index 9c1498df7..ed834c6a3 100644 --- a/src/Configuration/Monai.Deploy.WorkflowManager.Configuration.csproj +++ b/src/Configuration/Monai.Deploy.WorkflowManager.Configuration.csproj @@ -16,7 +16,7 @@ SPDX-License-Identifier: Apache License 2.0 - + diff --git a/src/Configuration/packages.lock.json b/src/Configuration/packages.lock.json index b35e6dbe1..b206b78e5 100644 --- a/src/Configuration/packages.lock.json +++ b/src/Configuration/packages.lock.json @@ -38,9 +38,9 @@ }, "Monai.Deploy.Storage": { "type": "Direct", - "requested": "[0.1.0-rc0019, )", - "resolved": "0.1.0-rc0019", - "contentHash": "YAXgTE7vTqeMc+ZhprUg/muQ+JDVMaix1Ha1vLJTSSjfZWBOnxyK5SugS50uNlvByJcqHEXh+ygd9219wK1S+Q==", + "requested": "[0.1.0-rc0020, )", + "resolved": "0.1.0-rc0020", + "contentHash": "SNNzbGPte4S2CKzsYI5g723aRMNG8AVFNYR847hHl8JQXCD5f6ugxEJUid4/hpu88HxPPigDm7sMrc5H4EwtZQ==", "dependencies": { "AWSSDK.SecurityToken": "3.7.1.141", "Ardalis.GuardClauses": "4.0.0", diff --git a/src/PayloadListener/Monai.Deploy.WorkflowManager.PayloadListener.csproj b/src/PayloadListener/Monai.Deploy.WorkflowManager.PayloadListener.csproj index 055f5f33d..5793587fc 100644 --- a/src/PayloadListener/Monai.Deploy.WorkflowManager.PayloadListener.csproj +++ b/src/PayloadListener/Monai.Deploy.WorkflowManager.PayloadListener.csproj @@ -15,7 +15,7 @@ SPDX-License-Identifier: Apache License 2.0 - + diff --git a/src/TaskManager/API/Monai.Deploy.WorkflowManager.TaskManager.API.csproj b/src/TaskManager/API/Monai.Deploy.WorkflowManager.TaskManager.API.csproj index cf52da543..04232c90a 100644 --- a/src/TaskManager/API/Monai.Deploy.WorkflowManager.TaskManager.API.csproj +++ b/src/TaskManager/API/Monai.Deploy.WorkflowManager.TaskManager.API.csproj @@ -23,7 +23,7 @@ SPDX-License-Identifier: Apache License 2.0 - + diff --git a/src/TaskManager/API/packages.lock.json b/src/TaskManager/API/packages.lock.json index 40d680f47..16ed3b757 100644 --- a/src/TaskManager/API/packages.lock.json +++ b/src/TaskManager/API/packages.lock.json @@ -18,9 +18,9 @@ }, "Monai.Deploy.Storage": { "type": "Direct", - "requested": "[0.1.0-rc0019, )", - "resolved": "0.1.0-rc0019", - "contentHash": "YAXgTE7vTqeMc+ZhprUg/muQ+JDVMaix1Ha1vLJTSSjfZWBOnxyK5SugS50uNlvByJcqHEXh+ygd9219wK1S+Q==", + "requested": "[0.1.0-rc0020, )", + "resolved": "0.1.0-rc0020", + "contentHash": "SNNzbGPte4S2CKzsYI5g723aRMNG8AVFNYR847hHl8JQXCD5f6ugxEJUid4/hpu88HxPPigDm7sMrc5H4EwtZQ==", "dependencies": { "AWSSDK.SecurityToken": "3.7.1.141", "Ardalis.GuardClauses": "4.0.0", diff --git a/src/TaskManager/Monai.Deploy.WorkflowManager.TaskManager.csproj b/src/TaskManager/Monai.Deploy.WorkflowManager.TaskManager.csproj index 7c4377a41..e4df788e6 100644 --- a/src/TaskManager/Monai.Deploy.WorkflowManager.TaskManager.csproj +++ b/src/TaskManager/Monai.Deploy.WorkflowManager.TaskManager.csproj @@ -39,7 +39,7 @@ SPDX-License-Identifier: Apache License 2.0 - + diff --git a/src/TaskManager/Plug-ins/Argo/Monai.Deploy.WorkflowManager.TaskManager.Argo.csproj b/src/TaskManager/Plug-ins/Argo/Monai.Deploy.WorkflowManager.TaskManager.Argo.csproj index fb8342811..8b574d360 100644 --- a/src/TaskManager/Plug-ins/Argo/Monai.Deploy.WorkflowManager.TaskManager.Argo.csproj +++ b/src/TaskManager/Plug-ins/Argo/Monai.Deploy.WorkflowManager.TaskManager.Argo.csproj @@ -23,7 +23,7 @@ SPDX-License-Identifier: Apache License 2.0 runtime; build; native; contentfiles; analyzers; buildtransitive - + diff --git a/src/TaskManager/Plug-ins/Argo/packages.lock.json b/src/TaskManager/Plug-ins/Argo/packages.lock.json index 7d38750a9..7a9dda53e 100644 --- a/src/TaskManager/Plug-ins/Argo/packages.lock.json +++ b/src/TaskManager/Plug-ins/Argo/packages.lock.json @@ -48,9 +48,9 @@ }, "Monai.Deploy.Storage": { "type": "Direct", - "requested": "[0.1.0-rc0019, )", - "resolved": "0.1.0-rc0019", - "contentHash": "YAXgTE7vTqeMc+ZhprUg/muQ+JDVMaix1Ha1vLJTSSjfZWBOnxyK5SugS50uNlvByJcqHEXh+ygd9219wK1S+Q==", + "requested": "[0.1.0-rc0020, )", + "resolved": "0.1.0-rc0020", + "contentHash": "SNNzbGPte4S2CKzsYI5g723aRMNG8AVFNYR847hHl8JQXCD5f6ugxEJUid4/hpu88HxPPigDm7sMrc5H4EwtZQ==", "dependencies": { "AWSSDK.SecurityToken": "3.7.1.141", "Ardalis.GuardClauses": "4.0.0", @@ -629,7 +629,7 @@ "type": "Project", "dependencies": { "Monai.Deploy.Messaging": "0.1.0-rc0038", - "Monai.Deploy.Storage": "0.1.0-rc0019" + "Monai.Deploy.Storage": "0.1.0-rc0020" } } } diff --git a/src/TaskManager/packages.lock.json b/src/TaskManager/packages.lock.json index 2a3c2c523..8cfac393f 100644 --- a/src/TaskManager/packages.lock.json +++ b/src/TaskManager/packages.lock.json @@ -74,9 +74,9 @@ }, "Monai.Deploy.Storage": { "type": "Direct", - "requested": "[0.1.0-rc0019, )", - "resolved": "0.1.0-rc0019", - "contentHash": "YAXgTE7vTqeMc+ZhprUg/muQ+JDVMaix1Ha1vLJTSSjfZWBOnxyK5SugS50uNlvByJcqHEXh+ygd9219wK1S+Q==", + "requested": "[0.1.0-rc0020, )", + "resolved": "0.1.0-rc0020", + "contentHash": "SNNzbGPte4S2CKzsYI5g723aRMNG8AVFNYR847hHl8JQXCD5f6ugxEJUid4/hpu88HxPPigDm7sMrc5H4EwtZQ==", "dependencies": { "AWSSDK.SecurityToken": "3.7.1.141", "Ardalis.GuardClauses": "4.0.0", @@ -563,7 +563,7 @@ "Microsoft.Extensions.Configuration": "6.0.1", "Microsoft.Extensions.Options": "6.0.0", "Monai.Deploy.Messaging": "0.1.0-rc0038", - "Monai.Deploy.Storage": "0.1.0-rc0019", + "Monai.Deploy.Storage": "0.1.0-rc0020", "Newtonsoft.Json": "13.0.1", "System.IO.Abstractions": "16.1.25" } @@ -601,7 +601,7 @@ "type": "Project", "dependencies": { "Monai.Deploy.Messaging": "0.1.0-rc0038", - "Monai.Deploy.Storage": "0.1.0-rc0019" + "Monai.Deploy.Storage": "0.1.0-rc0020" } } } diff --git a/src/WorkflowExecuter/Common/ArtifactMapper.cs b/src/WorkflowExecuter/Common/ArtifactMapper.cs index 6ed3a0dd3..4238a4201 100644 --- a/src/WorkflowExecuter/Common/ArtifactMapper.cs +++ b/src/WorkflowExecuter/Common/ArtifactMapper.cs @@ -31,6 +31,11 @@ public async Task> ConvertArtifactVariablesToPath(Art var variableString = artifact.Value.Split(" ")?[1]; + if (variableString is null) + { + continue; + } + if (variableString.StartsWith("context.input")) { artifactPathDictionary.Add(artifact.Name, $"{payloadId}/dcm"); diff --git a/src/WorkflowExecuter/Monai.Deploy.WorkloadManager.WorkfowExecuter.csproj b/src/WorkflowExecuter/Monai.Deploy.WorkloadManager.WorkfowExecuter.csproj index 16d0af93b..868f57214 100644 --- a/src/WorkflowExecuter/Monai.Deploy.WorkloadManager.WorkfowExecuter.csproj +++ b/src/WorkflowExecuter/Monai.Deploy.WorkloadManager.WorkfowExecuter.csproj @@ -9,7 +9,7 @@ - + diff --git a/src/WorkflowExecuter/Services/WorkflowExecuterService.cs b/src/WorkflowExecuter/Services/WorkflowExecuterService.cs index 4dad2fac5..ccf4e7335 100644 --- a/src/WorkflowExecuter/Services/WorkflowExecuterService.cs +++ b/src/WorkflowExecuter/Services/WorkflowExecuterService.cs @@ -232,7 +232,7 @@ private async Task UpdateWorkflowInstanceStatus(WorkflowInstance workflowI return true; } - private List HandleTaskDestinations(WorkflowInstance workflowInstance, WorkflowRevision workflow, TaskDestination[]? currentTaskDestinations) + private async Task> HandleTaskDestinations(WorkflowInstance workflowInstance, WorkflowRevision workflow, TaskDestination[]? currentTaskDestinations) { var newTaskExecutions = new List(); diff --git a/src/WorkflowManager/packages.lock.json b/src/WorkflowManager/packages.lock.json index ec6aae743..29990fca1 100644 --- a/src/WorkflowManager/packages.lock.json +++ b/src/WorkflowManager/packages.lock.json @@ -490,8 +490,8 @@ }, "Monai.Deploy.Storage": { "type": "Transitive", - "resolved": "0.1.0-rc0019", - "contentHash": "YAXgTE7vTqeMc+ZhprUg/muQ+JDVMaix1Ha1vLJTSSjfZWBOnxyK5SugS50uNlvByJcqHEXh+ygd9219wK1S+Q==", + "resolved": "0.1.0-rc0020", + "contentHash": "SNNzbGPte4S2CKzsYI5g723aRMNG8AVFNYR847hHl8JQXCD5f6ugxEJUid4/hpu88HxPPigDm7sMrc5H4EwtZQ==", "dependencies": { "AWSSDK.SecurityToken": "3.7.1.141", "Ardalis.GuardClauses": "4.0.0", @@ -722,7 +722,7 @@ "Microsoft.Extensions.Configuration": "6.0.1", "Microsoft.Extensions.Options": "6.0.0", "Monai.Deploy.Messaging": "0.1.0-rc0038", - "Monai.Deploy.Storage": "0.1.0-rc0019", + "Monai.Deploy.Storage": "0.1.0-rc0020", "Newtonsoft.Json": "13.0.1", "System.IO.Abstractions": "16.1.25" } @@ -762,7 +762,7 @@ "Ardalis.GuardClauses": "4.0.1", "Microsoft.Extensions.Hosting.Abstractions": "6.0.0", "Monai.Deploy.Messaging": "0.1.0-rc0038", - "Monai.Deploy.Storage": "0.1.0-rc0019", + "Monai.Deploy.Storage": "0.1.0-rc0020", "Monai.Deploy.WorkflowManager.Common": "1.0.0", "Monai.Deploy.WorkflowManager.Configuration": "1.0.0", "Monai.Deploy.WorkflowManager.Contracts": "1.0.0", @@ -776,7 +776,7 @@ "AWSSDK.SecurityToken": "3.7.1.141", "Ardalis.GuardClauses": "4.0.1", "Monai.Deploy.Messaging": "0.1.0-rc0038", - "Monai.Deploy.Storage": "0.1.0-rc0019", + "Monai.Deploy.Storage": "0.1.0-rc0020", "Monai.Deploy.WorkflowManager.Configuration": "1.0.0", "Monai.Deploy.WorkflowManager.Contracts": "1.0.0", "Monai.Deploy.WorkflowManager.Database": "1.0.0", diff --git a/tests/IntegrationTests/WorkflowManager.IntegrationTests/TestData/WorkflowInstanceTestData.cs b/tests/IntegrationTests/WorkflowManager.IntegrationTests/TestData/WorkflowInstanceTestData.cs index bca83ed33..a087cc4ab 100644 --- a/tests/IntegrationTests/WorkflowManager.IntegrationTests/TestData/WorkflowInstanceTestData.cs +++ b/tests/IntegrationTests/WorkflowManager.IntegrationTests/TestData/WorkflowInstanceTestData.cs @@ -82,6 +82,7 @@ public static class WorkflowInstancesTestData WorkflowId = Helper.GetWorkflowByName("Task_Status_Update_Workflow").WorkflowRevision.WorkflowId, PayloadId = Guid.NewGuid().ToString(), StartTime = DateTime.Now, + BucketId = "bucket1", Status = Status.Created, InputMetaData = new Dictionary() { diff --git a/tests/UnitTests/WorkflowExecuter.Tests/Common/ArtifactMapperTests.cs b/tests/UnitTests/WorkflowExecuter.Tests/Common/ArtifactMapperTests.cs index 7d02d505a..8745ae06e 100644 --- a/tests/UnitTests/WorkflowExecuter.Tests/Common/ArtifactMapperTests.cs +++ b/tests/UnitTests/WorkflowExecuter.Tests/Common/ArtifactMapperTests.cs @@ -95,5 +95,69 @@ public async Task ConvertArtifactVariablesToPath_MultipleArtifacts_ReturnsMapped response.Should().BeEquivalentTo(expected); } + + [Fact] + public async Task ConvertArtifactVariablesToPath_MultipleInvalid_ReturnsEmptyDict() + { + var artifacts = new Artifact[] + { + new Artifact + { + Name = "taskartifact", + Value = "{{ test.not.an.artifact }}" + }, + new Artifact + { + Name = "dicomimage", + Value = "{{ context }}" + }, + new Artifact + { + Name = "outputtaskdir", + Value = "{{ sandwich.executions.coffee.output_dir }}" + } + }; + + var payloadId = Guid.NewGuid().ToString(); + var workflowInstanceId = Guid.NewGuid().ToString(); + var executionId = Guid.NewGuid().ToString(); + + var expected = new Dictionary(); + + var workflowInstance = new WorkflowInstance + { + Id = workflowInstanceId, + WorkflowId = Guid.NewGuid().ToString(), + PayloadId = payloadId, + Status = Status.Created, + BucketId = "bucket", + Tasks = new List + { + new TaskExecution + { + TaskId = "image_type_detector", + ExecutionId = executionId, + Status = TaskExecutionStatus.Dispatched, + OutputArtifacts = new Dictionary + { + { "dicom", $"{payloadId}/workflows/{workflowInstanceId}/{executionId}" } + } + }, + new TaskExecution + { + TaskId = "coffee", + Status = TaskExecutionStatus.Created, + OutputDirectory = $"{payloadId}/workflows/{workflowInstanceId}/{executionId}" + } + } + }; + + _workflowInstanceRepository.Setup(w => w.GetTaskByIdAsync(workflowInstance.Id, "image_type_detector")).ReturnsAsync(workflowInstance.Tasks[0]); + _workflowInstanceRepository.Setup(w => w.GetTaskByIdAsync(workflowInstance.Id, "coffee")).ReturnsAsync(workflowInstance.Tasks[1]); + + var response = await ArtifactMapper.ConvertArtifactVariablesToPath(artifacts, workflowInstance.PayloadId, workflowInstance.Id); + + response.Should().BeEquivalentTo(expected); + } } } diff --git a/tests/UnitTests/WorkflowManager.Tests/packages.lock.json b/tests/UnitTests/WorkflowManager.Tests/packages.lock.json index 87c3db612..c4b2e00b6 100644 --- a/tests/UnitTests/WorkflowManager.Tests/packages.lock.json +++ b/tests/UnitTests/WorkflowManager.Tests/packages.lock.json @@ -605,8 +605,8 @@ }, "Monai.Deploy.Storage": { "type": "Transitive", - "resolved": "0.1.0-rc0019", - "contentHash": "YAXgTE7vTqeMc+ZhprUg/muQ+JDVMaix1Ha1vLJTSSjfZWBOnxyK5SugS50uNlvByJcqHEXh+ygd9219wK1S+Q==", + "resolved": "0.1.0-rc0020", + "contentHash": "SNNzbGPte4S2CKzsYI5g723aRMNG8AVFNYR847hHl8JQXCD5f6ugxEJUid4/hpu88HxPPigDm7sMrc5H4EwtZQ==", "dependencies": { "AWSSDK.SecurityToken": "3.7.1.141", "Ardalis.GuardClauses": "4.0.0", @@ -1402,7 +1402,7 @@ "Microsoft.Extensions.Configuration": "6.0.1", "Microsoft.Extensions.Options": "6.0.0", "Monai.Deploy.Messaging": "0.1.0-rc0038", - "Monai.Deploy.Storage": "0.1.0-rc0019", + "Monai.Deploy.Storage": "0.1.0-rc0020", "Newtonsoft.Json": "13.0.1", "System.IO.Abstractions": "16.1.25" } @@ -1442,7 +1442,7 @@ "Ardalis.GuardClauses": "4.0.1", "Microsoft.Extensions.Hosting.Abstractions": "6.0.0", "Monai.Deploy.Messaging": "0.1.0-rc0038", - "Monai.Deploy.Storage": "0.1.0-rc0019", + "Monai.Deploy.Storage": "0.1.0-rc0020", "Monai.Deploy.WorkflowManager.Common": "1.0.0", "Monai.Deploy.WorkflowManager.Configuration": "1.0.0", "Monai.Deploy.WorkflowManager.Contracts": "1.0.0", @@ -1456,7 +1456,7 @@ "AWSSDK.SecurityToken": "3.7.1.141", "Ardalis.GuardClauses": "4.0.1", "Monai.Deploy.Messaging": "0.1.0-rc0038", - "Monai.Deploy.Storage": "0.1.0-rc0019", + "Monai.Deploy.Storage": "0.1.0-rc0020", "Monai.Deploy.WorkflowManager.Configuration": "1.0.0", "Monai.Deploy.WorkflowManager.Contracts": "1.0.0", "Monai.Deploy.WorkflowManager.Database": "1.0.0", From 9a46245b1043af8404578d2b2026a55b7462defb Mon Sep 17 00:00:00 2001 From: Jack Schofield Date: Mon, 23 May 2022 13:07:11 +0100 Subject: [PATCH 5/9] update sonar linting Signed-off-by: Jack Schofield --- ...onai-deploy-workflow-managercsharp.ruleset | 11 +- ...loy.WorkflowManager.PayloadListener.csproj | 9 + src/PayloadListener/packages.lock.json | 412 ++++++++++++++++++ src/WorkflowExecuter/Common/ArtifactMapper.cs | 76 ++-- ...loy.WorkloadManager.WorkfowExecuter.csproj | 10 + .../Services/WorkflowExecuterService.cs | 20 +- src/WorkflowExecuter/packages.lock.json | 372 ++++++++++++++++ .../Monai.Deploy.WorkflowManager.csproj | 2 +- .../docker-compose.yml | 67 +++ .../nginx.conf | 106 +++++ .../Common/ArtifactMapperTests.cs | 10 + 11 files changed, 1058 insertions(+), 37 deletions(-) create mode 100644 src/PayloadListener/packages.lock.json create mode 100644 src/WorkflowExecuter/packages.lock.json create mode 100644 tests/IntegrationTests/WorkflowManager.IntegrationTests/nginx.conf diff --git a/src/.sonarlint/project-monai_monai-deploy-workflow-managercsharp.ruleset b/src/.sonarlint/project-monai_monai-deploy-workflow-managercsharp.ruleset index 8c6d762ed..af7ee0d2e 100644 --- a/src/.sonarlint/project-monai_monai-deploy-workflow-managercsharp.ruleset +++ b/src/.sonarlint/project-monai_monai-deploy-workflow-managercsharp.ruleset @@ -1,5 +1,9 @@ - - + + + + + + @@ -367,4 +371,7 @@ + + + \ No newline at end of file diff --git a/src/PayloadListener/Monai.Deploy.WorkflowManager.PayloadListener.csproj b/src/PayloadListener/Monai.Deploy.WorkflowManager.PayloadListener.csproj index 5793587fc..a9c396033 100644 --- a/src/PayloadListener/Monai.Deploy.WorkflowManager.PayloadListener.csproj +++ b/src/PayloadListener/Monai.Deploy.WorkflowManager.PayloadListener.csproj @@ -11,6 +11,10 @@ SPDX-License-Identifier: Apache License 2.0 enable + + + + @@ -26,4 +30,9 @@ SPDX-License-Identifier: Apache License 2.0 + + true + true + + diff --git a/src/PayloadListener/packages.lock.json b/src/PayloadListener/packages.lock.json new file mode 100644 index 000000000..895e5acea --- /dev/null +++ b/src/PayloadListener/packages.lock.json @@ -0,0 +1,412 @@ +{ + "version": 1, + "dependencies": { + "net6.0": { + "Ardalis.GuardClauses": { + "type": "Direct", + "requested": "[4.0.1, )", + "resolved": "4.0.1", + "contentHash": "RemnImQf/BWR8oYqFpdw+hn+b4Q1w+pGujkRiSfjQhMPuiERwGn4UMmQv+6UDE4qbPlnIN+e3e40JkvBhzgfzg==", + "dependencies": { + "JetBrains.Annotations": "2021.3.0" + } + }, + "Microsoft.Extensions.Hosting.Abstractions": { + "type": "Direct", + "requested": "[6.0.0, )", + "resolved": "6.0.0", + "contentHash": "GcT5l2CYXL6Sa27KCSh0TixsRfADUgth+ojQSD5EkzisZxmGFh7CwzkcYuGwvmXLjr27uWRNrJ2vuuEjMhU05Q==", + "dependencies": { + "Microsoft.Extensions.Configuration.Abstractions": "6.0.0", + "Microsoft.Extensions.DependencyInjection.Abstractions": "6.0.0", + "Microsoft.Extensions.FileProviders.Abstractions": "6.0.0" + } + }, + "Monai.Deploy.Messaging": { + "type": "Direct", + "requested": "[0.1.0-rc0038, )", + "resolved": "0.1.0-rc0038", + "contentHash": "j6RswItc5GE7b42T1KMMT9oXsOEizr6HKgso4WAV/evGmcwMnzuBdRJHNIfGWeDTTKO7Ld7+BJrSOEno5+udsg==", + "dependencies": { + "Ardalis.GuardClauses": "4.0.1", + "Microsoft.Extensions.Configuration": "6.0.1", + "Microsoft.Extensions.Logging": "6.0.0", + "Newtonsoft.Json": "13.0.1", + "RabbitMQ.Client": "6.2.4", + "System.ComponentModel.Annotations": "5.0.0" + } + }, + "Monai.Deploy.Storage": { + "type": "Direct", + "requested": "[0.1.0-rc0020, )", + "resolved": "0.1.0-rc0020", + "contentHash": "SNNzbGPte4S2CKzsYI5g723aRMNG8AVFNYR847hHl8JQXCD5f6ugxEJUid4/hpu88HxPPigDm7sMrc5H4EwtZQ==", + "dependencies": { + "AWSSDK.SecurityToken": "3.7.1.141", + "Ardalis.GuardClauses": "4.0.0", + "Microsoft.Extensions.Configuration": "6.0.1", + "Microsoft.Extensions.Logging": "6.0.0", + "Minio": "3.1.13", + "Newtonsoft.Json": "13.0.1" + } + }, + "AWSSDK.Core": { + "type": "Transitive", + "resolved": "3.7.10.10", + "contentHash": "eieuNNym2E63hAnUwZiEQtNCNT5knh2PTqcFVuFQsZpYCw22cGXLq9miC8TDENLFRHh2I1mkTBRiC9E8UlKQhw==" + }, + "AWSSDK.SecurityToken": { + "type": "Transitive", + "resolved": "3.7.1.141", + "contentHash": "paspGCeiN5Qjl6vJanqP1Ll8l3t9hqUmVWmeHMELXyvUgozvVzaSh+PTQ/gj6gmFQ64Us7aeCFSQlfBPZag+vQ==", + "dependencies": { + "AWSSDK.Core": "[3.7.10.10, 4.0.0)" + } + }, + "Crc32.NET": { + "type": "Transitive", + "resolved": "1.2.0", + "contentHash": "wNW/huzolu8MNKUnwCVKxjfAlCFpeI8AZVfF46iAWJ1+P6bTU1AZct7VAkDDEjgeeTJCVTkGZaD6jSd/fOiUkA==", + "dependencies": { + "NETStandard.Library": "2.0.0" + } + }, + "DnsClient": { + "type": "Transitive", + "resolved": "1.6.0", + "contentHash": "kVd3IxAs5EQZSKTQhMUaJmuOQ1n2VWKExatdqNszsxxKzTB+Toaxx3M1DsN3yKfHloMXPmt6fgfcOqRwKheoUQ==", + "dependencies": { + "Microsoft.Win32.Registry": "5.0.0" + } + }, + "JetBrains.Annotations": { + "type": "Transitive", + "resolved": "2021.3.0", + "contentHash": "Ddxjs5RRjf+c8m9m++WvhW1lz1bqNhsTjWvCLbQN9bvKbkJeR9MhtfNwKgBRRdG2yLHcXFr5Lf7fsvvkiPaDRg==" + }, + "Microsoft.CSharp": { + "type": "Transitive", + "resolved": "4.5.0", + "contentHash": "kaj6Wb4qoMuH3HySFJhxwQfe8R/sJsNJnANrvv8WdFPMoNbKY5htfNscv+LHCu5ipz+49m2e+WQXpLXr9XYemQ==" + }, + "Microsoft.Extensions.Configuration": { + "type": "Transitive", + "resolved": "6.0.1", + "contentHash": "BUyFU9t+HzlSE7ri4B+AQN2BgTgHv/uM82s5ZkgU1BApyzWzIl48nDsG5wR1t0pniNuuyTBzG3qCW8152/NtSw==", + "dependencies": { + "Microsoft.Extensions.Configuration.Abstractions": "6.0.0", + "Microsoft.Extensions.Primitives": "6.0.0" + } + }, + "Microsoft.Extensions.Configuration.Abstractions": { + "type": "Transitive", + "resolved": "6.0.0", + "contentHash": "qWzV9o+ZRWq+pGm+1dF+R7qTgTYoXvbyowRoBxQJGfqTpqDun2eteerjRQhq5PQ/14S+lqto3Ft4gYaRyl4rdQ==", + "dependencies": { + "Microsoft.Extensions.Primitives": "6.0.0" + } + }, + "Microsoft.Extensions.DependencyInjection": { + "type": "Transitive", + "resolved": "6.0.0", + "contentHash": "k6PWQMuoBDGGHOQTtyois2u4AwyVcIwL2LaSLlTZQm2CYcJ1pxbt6jfAnpWmzENA/wfrYRI/X9DTLoUkE4AsLw==", + "dependencies": { + "Microsoft.Extensions.DependencyInjection.Abstractions": "6.0.0", + "System.Runtime.CompilerServices.Unsafe": "6.0.0" + } + }, + "Microsoft.Extensions.DependencyInjection.Abstractions": { + "type": "Transitive", + "resolved": "6.0.0", + "contentHash": "xlzi2IYREJH3/m6+lUrQlujzX8wDitm4QGnUu6kUXTQAWPuZY8i+ticFJbzfqaetLA6KR/rO6Ew/HuYD+bxifg==" + }, + "Microsoft.Extensions.FileProviders.Abstractions": { + "type": "Transitive", + "resolved": "6.0.0", + "contentHash": "0pd4/fho0gC12rQswaGQxbU34jOS1TPS8lZPpkFCH68ppQjHNHYle9iRuHeev1LhrJ94YPvzcRd8UmIuFk23Qw==", + "dependencies": { + "Microsoft.Extensions.Primitives": "6.0.0" + } + }, + "Microsoft.Extensions.Logging": { + "type": "Transitive", + "resolved": "6.0.0", + "contentHash": "eIbyj40QDg1NDz0HBW0S5f3wrLVnKWnDJ/JtZ+yJDFnDj90VoPuoPmFkeaXrtu+0cKm5GRAwoDf+dBWXK0TUdg==", + "dependencies": { + "Microsoft.Extensions.DependencyInjection": "6.0.0", + "Microsoft.Extensions.DependencyInjection.Abstractions": "6.0.0", + "Microsoft.Extensions.Logging.Abstractions": "6.0.0", + "Microsoft.Extensions.Options": "6.0.0", + "System.Diagnostics.DiagnosticSource": "6.0.0" + } + }, + "Microsoft.Extensions.Logging.Abstractions": { + "type": "Transitive", + "resolved": "6.0.1", + "contentHash": "dzB2Cgg+JmrouhjkcQGzSFjjvpwlq353i8oBQO2GWNjCXSzhbtBRUf28HSauWe7eib3wYOdb3tItdjRwAdwCSg==" + }, + "Microsoft.Extensions.Options": { + "type": "Transitive", + "resolved": "6.0.0", + "contentHash": "dzXN0+V1AyjOe2xcJ86Qbo233KHuLEY0njf/P2Kw8SfJU+d45HNS2ctJdnEnrWbM9Ye2eFgaC5Mj9otRMU6IsQ==", + "dependencies": { + "Microsoft.Extensions.DependencyInjection.Abstractions": "6.0.0", + "Microsoft.Extensions.Primitives": "6.0.0" + } + }, + "Microsoft.Extensions.Primitives": { + "type": "Transitive", + "resolved": "6.0.0", + "contentHash": "9+PnzmQFfEFNR9J2aDTfJGGupShHjOuGw4VUv+JB044biSHrnmCIMD+mJHmb2H7YryrfBEXDurxQ47gJZdCKNQ==", + "dependencies": { + "System.Runtime.CompilerServices.Unsafe": "6.0.0" + } + }, + "Microsoft.NETCore.Platforms": { + "type": "Transitive", + "resolved": "5.0.0", + "contentHash": "VyPlqzH2wavqquTcYpkIIAQ6WdenuKoFN0BdYBbCWsclXacSOHNQn66Gt4z5NBqEYW0FAPm5rlvki9ZiCij5xQ==" + }, + "Microsoft.NETCore.Targets": { + "type": "Transitive", + "resolved": "1.1.0", + "contentHash": "aOZA3BWfz9RXjpzt0sRJJMjAscAUm3Hoa4UWAfceV9UTYxgwZ1lZt5nO2myFf+/jetYQo4uTP7zS8sJY67BBxg==" + }, + "Microsoft.Win32.Registry": { + "type": "Transitive", + "resolved": "5.0.0", + "contentHash": "dDoKi0PnDz31yAyETfRntsLArTlVAVzUzCIvvEDsDsucrl33Dl8pIJG06ePTJTI3tGpeyHS9Cq7Foc/s4EeKcg==", + "dependencies": { + "System.Security.AccessControl": "5.0.0", + "System.Security.Principal.Windows": "5.0.0" + } + }, + "Minio": { + "type": "Transitive", + "resolved": "3.1.13", + "contentHash": "i4h+gXs7kFHn1QUP0ZcuZz4Xa/PgNzZo+QRmi0WbFG5TYec7ozhrV+/cq7zfJbLOgcB89XwieASLLiduGNuwdA==", + "dependencies": { + "Crc32.NET": "1.2.0", + "Microsoft.CSharp": "4.5.0", + "RestSharp": "106.10.1", + "System.Reactive.Linq": "4.0.0" + } + }, + "MongoDB.Bson": { + "type": "Transitive", + "resolved": "2.15.0", + "contentHash": "5uPSADVLydGJ4CbmjrsQn7v3NugSdx6htJ4cMNbo8youlfBa5v6YbEqztbipnl+dT1L/YBkJKZ1DDp6LWl0Hqw==", + "dependencies": { + "System.Runtime.CompilerServices.Unsafe": "5.0.0" + } + }, + "MongoDB.Driver": { + "type": "Transitive", + "resolved": "2.15.0", + "contentHash": "MaDExORt2JLBHFlG3dJZIdQlZsfK8YjfEbXJECqAU/Ez8BwbJwjHS0AjOBQZPj3U4UDU5pnGYiQn4P0dM9DnMw==", + "dependencies": { + "MongoDB.Bson": "2.15.0", + "MongoDB.Driver.Core": "2.15.0", + "MongoDB.Libmongocrypt": "1.3.0" + } + }, + "MongoDB.Driver.Core": { + "type": "Transitive", + "resolved": "2.15.0", + "contentHash": "oEfGhyxHDmH3KhRHPZqVEYSfLqFwOLrY5zMszbELKNxrgkJj82MNJ8AzGVOsayaxtrQqqcAGfyQK8Ecc1yoc0Q==", + "dependencies": { + "DnsClient": "1.6.0", + "MongoDB.Bson": "2.15.0", + "MongoDB.Libmongocrypt": "1.3.0", + "SharpCompress": "0.30.1", + "System.Buffers": "4.5.1" + } + }, + "MongoDB.Libmongocrypt": { + "type": "Transitive", + "resolved": "1.3.0", + "contentHash": "BAvqY/otOsmAGTsmzIYXRFysvpu8X7rYkDzEBX4iBlrF57SB+fD5px9sCPR6sETgfZ6k98qUNS6go5wtY9WOBA==" + }, + "NETStandard.Library": { + "type": "Transitive", + "resolved": "2.0.0", + "contentHash": "7jnbRU+L08FXKMxqUflxEXtVymWvNOrS8yHgu9s6EM8Anr6T/wIX4nZ08j/u3Asz+tCufp3YVwFSEvFTPYmBPA==", + "dependencies": { + "Microsoft.NETCore.Platforms": "1.1.0" + } + }, + "Newtonsoft.Json": { + "type": "Transitive", + "resolved": "13.0.1", + "contentHash": "ppPFpBcvxdsfUonNcvITKqLl3bqxWbDCZIzDWHzjpdAHRFfZe0Dw9HmA0+za13IdyrgJwpkDTDA9fHaxOrt20A==" + }, + "RabbitMQ.Client": { + "type": "Transitive", + "resolved": "6.2.4", + "contentHash": "ttM7F+Ymb00EyQ25UCC44djr5GN/+cZNey2B3xD6JdJQQx7UcCtHdKBCE09zcmWuB+Afp07tFzetE14l/U8xQw==", + "dependencies": { + "System.Memory": "4.5.4", + "System.Threading.Channels": "4.7.1" + } + }, + "RestSharp": { + "type": "Transitive", + "resolved": "106.10.1", + "contentHash": "MhR4w8LbArVaRA5T/oAR3ZnF852QXtDXRBO84GsvyYzsm7FsjXHJ+JaM8x/nR/B/OCB1WbolGH7hqMT1GZpNJw==" + }, + "SharpCompress": { + "type": "Transitive", + "resolved": "0.30.1", + "contentHash": "XqD4TpfyYGa7QTPzaGlMVbcecKnXy4YmYLDWrU+JIj7IuRNl7DH2END+Ll7ekWIY8o3dAMWLFDE1xdhfIWD1nw==" + }, + "System.Buffers": { + "type": "Transitive", + "resolved": "4.5.1", + "contentHash": "Rw7ijyl1qqRS0YQD/WycNst8hUUMgrMH4FCn1nNm27M4VxchZ1js3fVjQaANHO5f3sN4isvP4a+Met9Y4YomAg==" + }, + "System.ComponentModel.Annotations": { + "type": "Transitive", + "resolved": "5.0.0", + "contentHash": "dMkqfy2el8A8/I76n2Hi1oBFEbG1SfxD2l5nhwXV3XjlnOmwxJlQbYpJH4W51odnU9sARCSAgv7S3CyAFMkpYg==" + }, + "System.Diagnostics.DiagnosticSource": { + "type": "Transitive", + "resolved": "6.0.0", + "contentHash": "frQDfv0rl209cKm1lnwTgFPzNigy2EKk1BS3uAvHvlBVKe5cymGyHO+Sj+NLv5VF/AhHsqPIUUwya5oV4CHMUw==", + "dependencies": { + "System.Runtime.CompilerServices.Unsafe": "6.0.0" + } + }, + "System.IO.Abstractions": { + "type": "Transitive", + "resolved": "16.1.25", + "contentHash": "PZ3y8SgASqtIlJyhss47FGLwBMXMWvxL9MKk9dEqIZT/vrfQr5yhKwdJ/vNm2H8FqsWJoxu0P12nYGXFj0irkA==" + }, + "System.Memory": { + "type": "Transitive", + "resolved": "4.5.4", + "contentHash": "1MbJTHS1lZ4bS4FmsJjnuGJOu88ZzTT2rLvrhW7Ygic+pC0NWA+3hgAen0HRdsocuQXCkUTdFn9yHJJhsijDXw==" + }, + "System.Reactive": { + "type": "Transitive", + "resolved": "4.0.0", + "contentHash": "yaVAxT2Yl28p9+DupGwj2oPbIOYZUQimOQ+n7N7vML2oofIRj9BmcfD7DHc6FLrNah65EigOgG6U9VnwWEyz5g==", + "dependencies": { + "System.Runtime.InteropServices.WindowsRuntime": "4.3.0" + } + }, + "System.Reactive.Linq": { + "type": "Transitive", + "resolved": "4.0.0", + "contentHash": "LaYTtUoOtQJ7SBb1/tfM/kcIHy0dsA7E+31/739HyTyGg9uSGzP/fV4VFI3/Atg7MQiks+JbEAwcoTDJY0pJoA==", + "dependencies": { + "System.Reactive": "4.0.0" + } + }, + "System.Runtime": { + "type": "Transitive", + "resolved": "4.3.0", + "contentHash": "JufQi0vPQ0xGnAczR13AUFglDyVYt4Kqnz1AZaiKZ5+GICq0/1MH/mO/eAJHt/mHW1zjKBJd7kV26SrxddAhiw==", + "dependencies": { + "Microsoft.NETCore.Platforms": "1.1.0", + "Microsoft.NETCore.Targets": "1.1.0" + } + }, + "System.Runtime.CompilerServices.Unsafe": { + "type": "Transitive", + "resolved": "6.0.0", + "contentHash": "/iUeP3tq1S0XdNNoMz5C9twLSrM/TH+qElHkXWaPvuNOt+99G75NrV0OS2EqHx5wMN7popYjpc8oTjC1y16DLg==" + }, + "System.Runtime.InteropServices.WindowsRuntime": { + "type": "Transitive", + "resolved": "4.3.0", + "contentHash": "J4GUi3xZQLUBasNwZnjrffN8i5wpHrBtZoLG+OhRyGo/+YunMRWWtwoMDlUAIdmX0uRfpHIBDSV6zyr3yf00TA==", + "dependencies": { + "System.Runtime": "4.3.0" + } + }, + "System.Security.AccessControl": { + "type": "Transitive", + "resolved": "5.0.0", + "contentHash": "dagJ1mHZO3Ani8GH0PHpPEe/oYO+rVdbQjvjJkBRNQkX4t0r1iaeGn8+/ybkSLEan3/slM0t59SVdHzuHf2jmw==", + "dependencies": { + "Microsoft.NETCore.Platforms": "5.0.0", + "System.Security.Principal.Windows": "5.0.0" + } + }, + "System.Security.Principal.Windows": { + "type": "Transitive", + "resolved": "5.0.0", + "contentHash": "t0MGLukB5WAVU9bO3MGzvlGnyJPgUlcwerXn1kzBRjwLKixT96XV0Uza41W49gVd8zEMFu9vQEFlv0IOrytICA==" + }, + "System.Threading.Channels": { + "type": "Transitive", + "resolved": "4.7.1", + "contentHash": "6akRtHK/wab3246t4p5v3HQrtQk8LboOt5T4dtpNgsp3zvDeM4/Gx8V12t0h+c/W9/enUrilk8n6EQqdQorZAA==" + }, + "monai.deploy.workflowmanager.common": { + "type": "Project", + "dependencies": { + "Ardalis.GuardClauses": "4.0.1", + "Microsoft.Extensions.DependencyInjection.Abstractions": "6.0.0", + "Monai.Deploy.WorkflowManager.Contracts": "1.0.0", + "Monai.Deploy.WorkflowManager.Database": "1.0.0" + } + }, + "monai.deploy.workflowmanager.configuration": { + "type": "Project", + "dependencies": { + "Microsoft.Extensions.Configuration": "6.0.1", + "Microsoft.Extensions.Options": "6.0.0", + "Monai.Deploy.Messaging": "0.1.0-rc0038", + "Monai.Deploy.Storage": "0.1.0-rc0020", + "Newtonsoft.Json": "13.0.1", + "System.IO.Abstractions": "16.1.25" + } + }, + "monai.deploy.workflowmanager.contracts": { + "type": "Project", + "dependencies": { + "AWSSDK.SecurityToken": "3.7.1.141", + "Microsoft.Extensions.Configuration": "6.0.1", + "Monai.Deploy.Messaging": "0.1.0-rc0038", + "MongoDB.Bson": "2.15.0", + "Newtonsoft.Json": "13.0.1" + } + }, + "monai.deploy.workflowmanager.database": { + "type": "Project", + "dependencies": { + "Ardalis.GuardClauses": "4.0.1", + "Microsoft.Extensions.Logging.Abstractions": "6.0.1", + "Microsoft.Extensions.Options": "6.0.0", + "Monai.Deploy.Messaging": "0.1.0-rc0038", + "Monai.Deploy.WorkflowManager.Contracts": "1.0.0", + "Monai.Deploy.WorkflowManager.Logging": "1.0.0", + "MongoDB.Bson": "2.15.0", + "MongoDB.Driver": "2.15.0" + } + }, + "monai.deploy.workflowmanager.logging": { + "type": "Project", + "dependencies": { + "Microsoft.Extensions.Logging.Abstractions": "6.0.1" + } + }, + "monai.deploy.workloadmanager.workfowexecuter": { + "type": "Project", + "dependencies": { + "AWSSDK.SecurityToken": "3.7.1.141", + "Ardalis.GuardClauses": "4.0.1", + "Monai.Deploy.Messaging": "0.1.0-rc0038", + "Monai.Deploy.Storage": "0.1.0-rc0020", + "Monai.Deploy.WorkflowManager.Configuration": "1.0.0", + "Monai.Deploy.WorkflowManager.Contracts": "1.0.0", + "Monai.Deploy.WorkflowManager.Database": "1.0.0", + "Monai.Deploy.WorkflowManager.Logging": "1.0.0", + "Newtonsoft.Json": "13.0.1" + } + } + } + } +} \ No newline at end of file diff --git a/src/WorkflowExecuter/Common/ArtifactMapper.cs b/src/WorkflowExecuter/Common/ArtifactMapper.cs index 4238a4201..0ecea822d 100644 --- a/src/WorkflowExecuter/Common/ArtifactMapper.cs +++ b/src/WorkflowExecuter/Common/ArtifactMapper.cs @@ -29,52 +29,74 @@ public async Task> ConvertArtifactVariablesToPath(Art Guard.Against.NullOrWhiteSpace(artifact.Value); Guard.Against.NullOrWhiteSpace(artifact.Name); - var variableString = artifact.Value.Split(" ")?[1]; - - if (variableString is null) + if (!TrimArtifactVariable(artifact.Value, out var variableString)) { continue; } - if (variableString.StartsWith("context.input")) + var mappedArtifact = await ConvertVariableStringToPath(artifact, variableString, workflowInstanceId, payloadId); + + if (mappedArtifact.Equals(default(KeyValuePair))) { - artifactPathDictionary.Add(artifact.Name, $"{payloadId}/dcm"); continue; } - if (variableString.StartsWith("context.executions")) - { - var variableWords = variableString.Split("."); + artifactPathDictionary.Add(mappedArtifact.Key, mappedArtifact.Value); + } - var variableTaskId = variableWords[2]; - var variableLocation = variableWords[3]; + return artifactPathDictionary; + } - var task = await _workflowInstanceRepository.GetTaskByIdAsync(workflowInstanceId, variableTaskId); + private static bool TrimArtifactVariable(string valueString, out string variableString) + { + var variableStrings = valueString.Split(" "); - if (variableLocation == "output_dir") - { - artifactPathDictionary.Add(artifact.Name, task.OutputDirectory); - continue; - } + if (variableStrings.Length < 2) + { + variableString = null; - if (variableLocation == "artifacts") - { - var artifactName = variableWords[4]; - var outputArtifact = task.OutputArtifacts.FirstOrDefault(a => a.Key == artifactName); + return false; + } - if (!outputArtifact.Equals(default(KeyValuePair))) - { - artifactPathDictionary.Add(outputArtifact.Key, outputArtifact.Value); - } + variableString = variableStrings[1]; - continue; - } + return true; + } + private async Task> ConvertVariableStringToPath(Artifact artifact, string variableString, string workflowInstanceId, string payloadId) + { + if (variableString.StartsWith("context.input")) + { + return new KeyValuePair(artifact.Name, $"{payloadId}/dcm"); + } + + if (variableString.StartsWith("context.executions")) + { + var variableWords = variableString.Split("."); + + var variableTaskId = variableWords[2]; + var variableLocation = variableWords[3]; + + var task = await _workflowInstanceRepository.GetTaskByIdAsync(workflowInstanceId, variableTaskId); + + if (variableLocation == "output_dir") + { + return new KeyValuePair(artifact.Name, task.OutputDirectory); } + if (variableLocation == "artifacts") + { + var artifactName = variableWords[4]; + var outputArtifact = task.OutputArtifacts.FirstOrDefault(a => a.Key == artifactName); + + if (!outputArtifact.Equals(default(KeyValuePair))) + { + return new KeyValuePair(outputArtifact.Key, outputArtifact.Value); + } + } } - return artifactPathDictionary; + return default; } } } diff --git a/src/WorkflowExecuter/Monai.Deploy.WorkloadManager.WorkfowExecuter.csproj b/src/WorkflowExecuter/Monai.Deploy.WorkloadManager.WorkfowExecuter.csproj index 868f57214..49db2cb73 100644 --- a/src/WorkflowExecuter/Monai.Deploy.WorkloadManager.WorkfowExecuter.csproj +++ b/src/WorkflowExecuter/Monai.Deploy.WorkloadManager.WorkfowExecuter.csproj @@ -2,10 +2,15 @@ net6.0 + ..\.sonarlint\project-monai_monai-deploy-workflow-managercsharp.ruleset enable enable + + + + @@ -21,4 +26,9 @@ + + true + true + + diff --git a/src/WorkflowExecuter/Services/WorkflowExecuterService.cs b/src/WorkflowExecuter/Services/WorkflowExecuterService.cs index ccf4e7335..ab8e3406b 100644 --- a/src/WorkflowExecuter/Services/WorkflowExecuterService.cs +++ b/src/WorkflowExecuter/Services/WorkflowExecuterService.cs @@ -144,7 +144,9 @@ public async Task ProcessTaskUpdate(TaskUpdateEvent message) return false; } - if (message.Status != TaskExecutionStatus.Succeeded) + var previouslyFailed = workflowInstance.Tasks.Any(t => t.Status == TaskExecutionStatus.Failed) || workflowInstance.Status == Status.Failed; + + if (message.Status != TaskExecutionStatus.Succeeded || previouslyFailed) { await UpdateWorkflowInstanceStatus(workflowInstance, message.TaskId, message.Status); @@ -215,18 +217,22 @@ private async Task UpdateWorkflowInstanceStatus(WorkflowInstance workflowI return false; } + if (workflowInstance.Status == Status.Failed) + { + return true; + } + var previousTasks = workflowInstance.Tasks.Where(t => t.TaskId != taskId); - if (!previousTasks.Any(t => t.Status != TaskExecutionStatus.Succeeded && t.Status != TaskExecutionStatus.Canceled) - && (currentTaskStatus == TaskExecutionStatus.Succeeded || currentTaskStatus == TaskExecutionStatus.Canceled)) + if (previousTasks.Any(t => t.Status == TaskExecutionStatus.Failed) || currentTaskStatus == TaskExecutionStatus.Failed) { - return await _workflowInstanceRepository.UpdateWorkflowInstanceStatusAsync(workflowInstance.Id, Status.Succeeded); + return await _workflowInstanceRepository.UpdateWorkflowInstanceStatusAsync(workflowInstance.Id, Status.Failed); } - if (!previousTasks.Any(t => t.Status != TaskExecutionStatus.Succeeded && t.Status != TaskExecutionStatus.Failed && t.Status != TaskExecutionStatus.Canceled) - && (currentTaskStatus == TaskExecutionStatus.Succeeded || currentTaskStatus == TaskExecutionStatus.Canceled || currentTaskStatus == TaskExecutionStatus.Failed)) + if (!previousTasks.Any(t => t.Status != TaskExecutionStatus.Succeeded && t.Status != TaskExecutionStatus.Canceled) + && (currentTaskStatus == TaskExecutionStatus.Succeeded || currentTaskStatus == TaskExecutionStatus.Canceled)) { - return await _workflowInstanceRepository.UpdateWorkflowInstanceStatusAsync(workflowInstance.Id, Status.Failed); + return await _workflowInstanceRepository.UpdateWorkflowInstanceStatusAsync(workflowInstance.Id, Status.Succeeded); } return true; diff --git a/src/WorkflowExecuter/packages.lock.json b/src/WorkflowExecuter/packages.lock.json new file mode 100644 index 000000000..e05d15b4b --- /dev/null +++ b/src/WorkflowExecuter/packages.lock.json @@ -0,0 +1,372 @@ +{ + "version": 1, + "dependencies": { + "net6.0": { + "Ardalis.GuardClauses": { + "type": "Direct", + "requested": "[4.0.1, )", + "resolved": "4.0.1", + "contentHash": "RemnImQf/BWR8oYqFpdw+hn+b4Q1w+pGujkRiSfjQhMPuiERwGn4UMmQv+6UDE4qbPlnIN+e3e40JkvBhzgfzg==", + "dependencies": { + "JetBrains.Annotations": "2021.3.0" + } + }, + "AWSSDK.SecurityToken": { + "type": "Direct", + "requested": "[3.7.1.141, )", + "resolved": "3.7.1.141", + "contentHash": "paspGCeiN5Qjl6vJanqP1Ll8l3t9hqUmVWmeHMELXyvUgozvVzaSh+PTQ/gj6gmFQ64Us7aeCFSQlfBPZag+vQ==", + "dependencies": { + "AWSSDK.Core": "[3.7.10.10, 4.0.0)" + } + }, + "Monai.Deploy.Messaging": { + "type": "Direct", + "requested": "[0.1.0-rc0038, )", + "resolved": "0.1.0-rc0038", + "contentHash": "j6RswItc5GE7b42T1KMMT9oXsOEizr6HKgso4WAV/evGmcwMnzuBdRJHNIfGWeDTTKO7Ld7+BJrSOEno5+udsg==", + "dependencies": { + "Ardalis.GuardClauses": "4.0.1", + "Microsoft.Extensions.Configuration": "6.0.1", + "Microsoft.Extensions.Logging": "6.0.0", + "Newtonsoft.Json": "13.0.1", + "RabbitMQ.Client": "6.2.4", + "System.ComponentModel.Annotations": "5.0.0" + } + }, + "Monai.Deploy.Storage": { + "type": "Direct", + "requested": "[0.1.0-rc0020, )", + "resolved": "0.1.0-rc0020", + "contentHash": "SNNzbGPte4S2CKzsYI5g723aRMNG8AVFNYR847hHl8JQXCD5f6ugxEJUid4/hpu88HxPPigDm7sMrc5H4EwtZQ==", + "dependencies": { + "AWSSDK.SecurityToken": "3.7.1.141", + "Ardalis.GuardClauses": "4.0.0", + "Microsoft.Extensions.Configuration": "6.0.1", + "Microsoft.Extensions.Logging": "6.0.0", + "Minio": "3.1.13", + "Newtonsoft.Json": "13.0.1" + } + }, + "Newtonsoft.Json": { + "type": "Direct", + "requested": "[13.0.1, )", + "resolved": "13.0.1", + "contentHash": "ppPFpBcvxdsfUonNcvITKqLl3bqxWbDCZIzDWHzjpdAHRFfZe0Dw9HmA0+za13IdyrgJwpkDTDA9fHaxOrt20A==" + }, + "AWSSDK.Core": { + "type": "Transitive", + "resolved": "3.7.10.10", + "contentHash": "eieuNNym2E63hAnUwZiEQtNCNT5knh2PTqcFVuFQsZpYCw22cGXLq9miC8TDENLFRHh2I1mkTBRiC9E8UlKQhw==" + }, + "Crc32.NET": { + "type": "Transitive", + "resolved": "1.2.0", + "contentHash": "wNW/huzolu8MNKUnwCVKxjfAlCFpeI8AZVfF46iAWJ1+P6bTU1AZct7VAkDDEjgeeTJCVTkGZaD6jSd/fOiUkA==", + "dependencies": { + "NETStandard.Library": "2.0.0" + } + }, + "DnsClient": { + "type": "Transitive", + "resolved": "1.6.0", + "contentHash": "kVd3IxAs5EQZSKTQhMUaJmuOQ1n2VWKExatdqNszsxxKzTB+Toaxx3M1DsN3yKfHloMXPmt6fgfcOqRwKheoUQ==", + "dependencies": { + "Microsoft.Win32.Registry": "5.0.0" + } + }, + "JetBrains.Annotations": { + "type": "Transitive", + "resolved": "2021.3.0", + "contentHash": "Ddxjs5RRjf+c8m9m++WvhW1lz1bqNhsTjWvCLbQN9bvKbkJeR9MhtfNwKgBRRdG2yLHcXFr5Lf7fsvvkiPaDRg==" + }, + "Microsoft.CSharp": { + "type": "Transitive", + "resolved": "4.5.0", + "contentHash": "kaj6Wb4qoMuH3HySFJhxwQfe8R/sJsNJnANrvv8WdFPMoNbKY5htfNscv+LHCu5ipz+49m2e+WQXpLXr9XYemQ==" + }, + "Microsoft.Extensions.Configuration": { + "type": "Transitive", + "resolved": "6.0.1", + "contentHash": "BUyFU9t+HzlSE7ri4B+AQN2BgTgHv/uM82s5ZkgU1BApyzWzIl48nDsG5wR1t0pniNuuyTBzG3qCW8152/NtSw==", + "dependencies": { + "Microsoft.Extensions.Configuration.Abstractions": "6.0.0", + "Microsoft.Extensions.Primitives": "6.0.0" + } + }, + "Microsoft.Extensions.Configuration.Abstractions": { + "type": "Transitive", + "resolved": "6.0.0", + "contentHash": "qWzV9o+ZRWq+pGm+1dF+R7qTgTYoXvbyowRoBxQJGfqTpqDun2eteerjRQhq5PQ/14S+lqto3Ft4gYaRyl4rdQ==", + "dependencies": { + "Microsoft.Extensions.Primitives": "6.0.0" + } + }, + "Microsoft.Extensions.DependencyInjection": { + "type": "Transitive", + "resolved": "6.0.0", + "contentHash": "k6PWQMuoBDGGHOQTtyois2u4AwyVcIwL2LaSLlTZQm2CYcJ1pxbt6jfAnpWmzENA/wfrYRI/X9DTLoUkE4AsLw==", + "dependencies": { + "Microsoft.Extensions.DependencyInjection.Abstractions": "6.0.0", + "System.Runtime.CompilerServices.Unsafe": "6.0.0" + } + }, + "Microsoft.Extensions.DependencyInjection.Abstractions": { + "type": "Transitive", + "resolved": "6.0.0", + "contentHash": "xlzi2IYREJH3/m6+lUrQlujzX8wDitm4QGnUu6kUXTQAWPuZY8i+ticFJbzfqaetLA6KR/rO6Ew/HuYD+bxifg==" + }, + "Microsoft.Extensions.Logging": { + "type": "Transitive", + "resolved": "6.0.0", + "contentHash": "eIbyj40QDg1NDz0HBW0S5f3wrLVnKWnDJ/JtZ+yJDFnDj90VoPuoPmFkeaXrtu+0cKm5GRAwoDf+dBWXK0TUdg==", + "dependencies": { + "Microsoft.Extensions.DependencyInjection": "6.0.0", + "Microsoft.Extensions.DependencyInjection.Abstractions": "6.0.0", + "Microsoft.Extensions.Logging.Abstractions": "6.0.0", + "Microsoft.Extensions.Options": "6.0.0", + "System.Diagnostics.DiagnosticSource": "6.0.0" + } + }, + "Microsoft.Extensions.Logging.Abstractions": { + "type": "Transitive", + "resolved": "6.0.1", + "contentHash": "dzB2Cgg+JmrouhjkcQGzSFjjvpwlq353i8oBQO2GWNjCXSzhbtBRUf28HSauWe7eib3wYOdb3tItdjRwAdwCSg==" + }, + "Microsoft.Extensions.Options": { + "type": "Transitive", + "resolved": "6.0.0", + "contentHash": "dzXN0+V1AyjOe2xcJ86Qbo233KHuLEY0njf/P2Kw8SfJU+d45HNS2ctJdnEnrWbM9Ye2eFgaC5Mj9otRMU6IsQ==", + "dependencies": { + "Microsoft.Extensions.DependencyInjection.Abstractions": "6.0.0", + "Microsoft.Extensions.Primitives": "6.0.0" + } + }, + "Microsoft.Extensions.Primitives": { + "type": "Transitive", + "resolved": "6.0.0", + "contentHash": "9+PnzmQFfEFNR9J2aDTfJGGupShHjOuGw4VUv+JB044biSHrnmCIMD+mJHmb2H7YryrfBEXDurxQ47gJZdCKNQ==", + "dependencies": { + "System.Runtime.CompilerServices.Unsafe": "6.0.0" + } + }, + "Microsoft.NETCore.Platforms": { + "type": "Transitive", + "resolved": "5.0.0", + "contentHash": "VyPlqzH2wavqquTcYpkIIAQ6WdenuKoFN0BdYBbCWsclXacSOHNQn66Gt4z5NBqEYW0FAPm5rlvki9ZiCij5xQ==" + }, + "Microsoft.NETCore.Targets": { + "type": "Transitive", + "resolved": "1.1.0", + "contentHash": "aOZA3BWfz9RXjpzt0sRJJMjAscAUm3Hoa4UWAfceV9UTYxgwZ1lZt5nO2myFf+/jetYQo4uTP7zS8sJY67BBxg==" + }, + "Microsoft.Win32.Registry": { + "type": "Transitive", + "resolved": "5.0.0", + "contentHash": "dDoKi0PnDz31yAyETfRntsLArTlVAVzUzCIvvEDsDsucrl33Dl8pIJG06ePTJTI3tGpeyHS9Cq7Foc/s4EeKcg==", + "dependencies": { + "System.Security.AccessControl": "5.0.0", + "System.Security.Principal.Windows": "5.0.0" + } + }, + "Minio": { + "type": "Transitive", + "resolved": "3.1.13", + "contentHash": "i4h+gXs7kFHn1QUP0ZcuZz4Xa/PgNzZo+QRmi0WbFG5TYec7ozhrV+/cq7zfJbLOgcB89XwieASLLiduGNuwdA==", + "dependencies": { + "Crc32.NET": "1.2.0", + "Microsoft.CSharp": "4.5.0", + "RestSharp": "106.10.1", + "System.Reactive.Linq": "4.0.0" + } + }, + "MongoDB.Bson": { + "type": "Transitive", + "resolved": "2.15.0", + "contentHash": "5uPSADVLydGJ4CbmjrsQn7v3NugSdx6htJ4cMNbo8youlfBa5v6YbEqztbipnl+dT1L/YBkJKZ1DDp6LWl0Hqw==", + "dependencies": { + "System.Runtime.CompilerServices.Unsafe": "5.0.0" + } + }, + "MongoDB.Driver": { + "type": "Transitive", + "resolved": "2.15.0", + "contentHash": "MaDExORt2JLBHFlG3dJZIdQlZsfK8YjfEbXJECqAU/Ez8BwbJwjHS0AjOBQZPj3U4UDU5pnGYiQn4P0dM9DnMw==", + "dependencies": { + "MongoDB.Bson": "2.15.0", + "MongoDB.Driver.Core": "2.15.0", + "MongoDB.Libmongocrypt": "1.3.0" + } + }, + "MongoDB.Driver.Core": { + "type": "Transitive", + "resolved": "2.15.0", + "contentHash": "oEfGhyxHDmH3KhRHPZqVEYSfLqFwOLrY5zMszbELKNxrgkJj82MNJ8AzGVOsayaxtrQqqcAGfyQK8Ecc1yoc0Q==", + "dependencies": { + "DnsClient": "1.6.0", + "MongoDB.Bson": "2.15.0", + "MongoDB.Libmongocrypt": "1.3.0", + "SharpCompress": "0.30.1", + "System.Buffers": "4.5.1" + } + }, + "MongoDB.Libmongocrypt": { + "type": "Transitive", + "resolved": "1.3.0", + "contentHash": "BAvqY/otOsmAGTsmzIYXRFysvpu8X7rYkDzEBX4iBlrF57SB+fD5px9sCPR6sETgfZ6k98qUNS6go5wtY9WOBA==" + }, + "NETStandard.Library": { + "type": "Transitive", + "resolved": "2.0.0", + "contentHash": "7jnbRU+L08FXKMxqUflxEXtVymWvNOrS8yHgu9s6EM8Anr6T/wIX4nZ08j/u3Asz+tCufp3YVwFSEvFTPYmBPA==", + "dependencies": { + "Microsoft.NETCore.Platforms": "1.1.0" + } + }, + "RabbitMQ.Client": { + "type": "Transitive", + "resolved": "6.2.4", + "contentHash": "ttM7F+Ymb00EyQ25UCC44djr5GN/+cZNey2B3xD6JdJQQx7UcCtHdKBCE09zcmWuB+Afp07tFzetE14l/U8xQw==", + "dependencies": { + "System.Memory": "4.5.4", + "System.Threading.Channels": "4.7.1" + } + }, + "RestSharp": { + "type": "Transitive", + "resolved": "106.10.1", + "contentHash": "MhR4w8LbArVaRA5T/oAR3ZnF852QXtDXRBO84GsvyYzsm7FsjXHJ+JaM8x/nR/B/OCB1WbolGH7hqMT1GZpNJw==" + }, + "SharpCompress": { + "type": "Transitive", + "resolved": "0.30.1", + "contentHash": "XqD4TpfyYGa7QTPzaGlMVbcecKnXy4YmYLDWrU+JIj7IuRNl7DH2END+Ll7ekWIY8o3dAMWLFDE1xdhfIWD1nw==" + }, + "System.Buffers": { + "type": "Transitive", + "resolved": "4.5.1", + "contentHash": "Rw7ijyl1qqRS0YQD/WycNst8hUUMgrMH4FCn1nNm27M4VxchZ1js3fVjQaANHO5f3sN4isvP4a+Met9Y4YomAg==" + }, + "System.ComponentModel.Annotations": { + "type": "Transitive", + "resolved": "5.0.0", + "contentHash": "dMkqfy2el8A8/I76n2Hi1oBFEbG1SfxD2l5nhwXV3XjlnOmwxJlQbYpJH4W51odnU9sARCSAgv7S3CyAFMkpYg==" + }, + "System.Diagnostics.DiagnosticSource": { + "type": "Transitive", + "resolved": "6.0.0", + "contentHash": "frQDfv0rl209cKm1lnwTgFPzNigy2EKk1BS3uAvHvlBVKe5cymGyHO+Sj+NLv5VF/AhHsqPIUUwya5oV4CHMUw==", + "dependencies": { + "System.Runtime.CompilerServices.Unsafe": "6.0.0" + } + }, + "System.IO.Abstractions": { + "type": "Transitive", + "resolved": "16.1.25", + "contentHash": "PZ3y8SgASqtIlJyhss47FGLwBMXMWvxL9MKk9dEqIZT/vrfQr5yhKwdJ/vNm2H8FqsWJoxu0P12nYGXFj0irkA==" + }, + "System.Memory": { + "type": "Transitive", + "resolved": "4.5.4", + "contentHash": "1MbJTHS1lZ4bS4FmsJjnuGJOu88ZzTT2rLvrhW7Ygic+pC0NWA+3hgAen0HRdsocuQXCkUTdFn9yHJJhsijDXw==" + }, + "System.Reactive": { + "type": "Transitive", + "resolved": "4.0.0", + "contentHash": "yaVAxT2Yl28p9+DupGwj2oPbIOYZUQimOQ+n7N7vML2oofIRj9BmcfD7DHc6FLrNah65EigOgG6U9VnwWEyz5g==", + "dependencies": { + "System.Runtime.InteropServices.WindowsRuntime": "4.3.0" + } + }, + "System.Reactive.Linq": { + "type": "Transitive", + "resolved": "4.0.0", + "contentHash": "LaYTtUoOtQJ7SBb1/tfM/kcIHy0dsA7E+31/739HyTyGg9uSGzP/fV4VFI3/Atg7MQiks+JbEAwcoTDJY0pJoA==", + "dependencies": { + "System.Reactive": "4.0.0" + } + }, + "System.Runtime": { + "type": "Transitive", + "resolved": "4.3.0", + "contentHash": "JufQi0vPQ0xGnAczR13AUFglDyVYt4Kqnz1AZaiKZ5+GICq0/1MH/mO/eAJHt/mHW1zjKBJd7kV26SrxddAhiw==", + "dependencies": { + "Microsoft.NETCore.Platforms": "1.1.0", + "Microsoft.NETCore.Targets": "1.1.0" + } + }, + "System.Runtime.CompilerServices.Unsafe": { + "type": "Transitive", + "resolved": "6.0.0", + "contentHash": "/iUeP3tq1S0XdNNoMz5C9twLSrM/TH+qElHkXWaPvuNOt+99G75NrV0OS2EqHx5wMN7popYjpc8oTjC1y16DLg==" + }, + "System.Runtime.InteropServices.WindowsRuntime": { + "type": "Transitive", + "resolved": "4.3.0", + "contentHash": "J4GUi3xZQLUBasNwZnjrffN8i5wpHrBtZoLG+OhRyGo/+YunMRWWtwoMDlUAIdmX0uRfpHIBDSV6zyr3yf00TA==", + "dependencies": { + "System.Runtime": "4.3.0" + } + }, + "System.Security.AccessControl": { + "type": "Transitive", + "resolved": "5.0.0", + "contentHash": "dagJ1mHZO3Ani8GH0PHpPEe/oYO+rVdbQjvjJkBRNQkX4t0r1iaeGn8+/ybkSLEan3/slM0t59SVdHzuHf2jmw==", + "dependencies": { + "Microsoft.NETCore.Platforms": "5.0.0", + "System.Security.Principal.Windows": "5.0.0" + } + }, + "System.Security.Principal.Windows": { + "type": "Transitive", + "resolved": "5.0.0", + "contentHash": "t0MGLukB5WAVU9bO3MGzvlGnyJPgUlcwerXn1kzBRjwLKixT96XV0Uza41W49gVd8zEMFu9vQEFlv0IOrytICA==" + }, + "System.Threading.Channels": { + "type": "Transitive", + "resolved": "4.7.1", + "contentHash": "6akRtHK/wab3246t4p5v3HQrtQk8LboOt5T4dtpNgsp3zvDeM4/Gx8V12t0h+c/W9/enUrilk8n6EQqdQorZAA==" + }, + "monai.deploy.workflowmanager.configuration": { + "type": "Project", + "dependencies": { + "Microsoft.Extensions.Configuration": "6.0.1", + "Microsoft.Extensions.Options": "6.0.0", + "Monai.Deploy.Messaging": "0.1.0-rc0038", + "Monai.Deploy.Storage": "0.1.0-rc0020", + "Newtonsoft.Json": "13.0.1", + "System.IO.Abstractions": "16.1.25" + } + }, + "monai.deploy.workflowmanager.contracts": { + "type": "Project", + "dependencies": { + "AWSSDK.SecurityToken": "3.7.1.141", + "Microsoft.Extensions.Configuration": "6.0.1", + "Monai.Deploy.Messaging": "0.1.0-rc0038", + "MongoDB.Bson": "2.15.0", + "Newtonsoft.Json": "13.0.1" + } + }, + "monai.deploy.workflowmanager.database": { + "type": "Project", + "dependencies": { + "Ardalis.GuardClauses": "4.0.1", + "Microsoft.Extensions.Logging.Abstractions": "6.0.1", + "Microsoft.Extensions.Options": "6.0.0", + "Monai.Deploy.Messaging": "0.1.0-rc0038", + "Monai.Deploy.WorkflowManager.Contracts": "1.0.0", + "Monai.Deploy.WorkflowManager.Logging": "1.0.0", + "MongoDB.Bson": "2.15.0", + "MongoDB.Driver": "2.15.0" + } + }, + "monai.deploy.workflowmanager.logging": { + "type": "Project", + "dependencies": { + "Microsoft.Extensions.Logging.Abstractions": "6.0.1" + } + } + } + } +} \ No newline at end of file diff --git a/src/WorkflowManager/Monai.Deploy.WorkflowManager.csproj b/src/WorkflowManager/Monai.Deploy.WorkflowManager.csproj index e98bf5f70..95d178669 100644 --- a/src/WorkflowManager/Monai.Deploy.WorkflowManager.csproj +++ b/src/WorkflowManager/Monai.Deploy.WorkflowManager.csproj @@ -8,7 +8,7 @@ SPDX-License-Identifier: Apache License 2.0 Exe net6.0 Monai.Deploy.WorkflowManager - ..\..\StyleCop.Analyzers.ruleset + ..\.sonarlint\project-monai_monai-deploy-workflow-managercsharp.ruleset false false diff --git a/tests/IntegrationTests/WorkflowManager.IntegrationTests/docker-compose.yml b/tests/IntegrationTests/WorkflowManager.IntegrationTests/docker-compose.yml index 34cbe3a5d..8e6b4b43b 100644 --- a/tests/IntegrationTests/WorkflowManager.IntegrationTests/docker-compose.yml +++ b/tests/IntegrationTests/WorkflowManager.IntegrationTests/docker-compose.yml @@ -1,4 +1,21 @@ version: '3.9' + +# Settings and configurations that are common for all containers +x-minio-common: &minio-common + image: quay.io/minio/minio:RELEASE.2022-04-12T06-55-35Z + command: server --console-address ":9001" http://minio{1...4}/data{1...2} + expose: + - "9000" + - "9001" + environment: + MINIO_ROOT_USER: minioadmin + MINIO_ROOT_PASSWORD: minioadmin + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] + interval: 30s + timeout: 20s + retries: 3 + services: rabbit1: image: "rabbitmq:3.8.18-management" @@ -23,6 +40,56 @@ services: - 27017:27017 volumes: - mongodb_data_container:/data/db + + minio1: + <<: *minio-common + hostname: minio1 + volumes: + - data1-1:/data1 + - data1-2:/data2 + + minio2: + <<: *minio-common + hostname: minio2 + volumes: + - data2-1:/data1 + - data2-2:/data2 + + minio3: + <<: *minio-common + hostname: minio3 + volumes: + - data3-1:/data1 + - data3-2:/data2 + + minio4: + <<: *minio-common + hostname: minio4 + volumes: + - data4-1:/data1 + - data4-2:/data2 + + nginx: + image: nginx:1.19.2-alpine + hostname: nginx + volumes: + - ./nginx.conf:/etc/nginx/nginx.conf:ro + ports: + - "9000:9000" + - "9001:9001" + depends_on: + - minio1 + - minio2 + - minio3 + - minio4 volumes: mongodb_data_container: + data1-1: + data1-2: + data2-1: + data2-2: + data3-1: + data3-2: + data4-1: + data4-2: diff --git a/tests/IntegrationTests/WorkflowManager.IntegrationTests/nginx.conf b/tests/IntegrationTests/WorkflowManager.IntegrationTests/nginx.conf new file mode 100644 index 000000000..cca82f6fe --- /dev/null +++ b/tests/IntegrationTests/WorkflowManager.IntegrationTests/nginx.conf @@ -0,0 +1,106 @@ +user nginx; +worker_processes auto; + +error_log /var/log/nginx/error.log warn; +pid /var/run/nginx.pid; + +events { + worker_connections 4096; +} + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + access_log /var/log/nginx/access.log main; + sendfile on; + keepalive_timeout 65; + + # include /etc/nginx/conf.d/*.conf; + + upstream minio { + server minio1:9000; + server minio2:9000; + server minio3:9000; + server minio4:9000; + } + + upstream console { + ip_hash; + server minio1:9001; + server minio2:9001; + server minio3:9001; + server minio4:9001; + } + + server { + listen 9000; + listen [::]:9000; + server_name localhost; + + # To allow special characters in headers + ignore_invalid_headers off; + # Allow any size file to be uploaded. + # Set to a value such as 1000m; to restrict file size to a specific value + client_max_body_size 0; + # To disable buffering + proxy_buffering off; + proxy_request_buffering off; + + location / { + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + proxy_connect_timeout 300; + # Default is HTTP/1, keepalive is only enabled in HTTP/1.1 + proxy_http_version 1.1; + proxy_set_header Connection ""; + chunked_transfer_encoding off; + + proxy_pass http://minio; + } + } + + server { + listen 9001; + listen [::]:9001; + server_name localhost; + + # To allow special characters in headers + ignore_invalid_headers off; + # Allow any size file to be uploaded. + # Set to a value such as 1000m; to restrict file size to a specific value + client_max_body_size 0; + # To disable buffering + proxy_buffering off; + proxy_request_buffering off; + + location / { + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-NginX-Proxy true; + + # This is necessary to pass the correct IP to be hashed + real_ip_header X-Real-IP; + + proxy_connect_timeout 300; + + # To support websocket + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + + chunked_transfer_encoding off; + + proxy_pass http://console; + } + } +} diff --git a/tests/UnitTests/WorkflowExecuter.Tests/Common/ArtifactMapperTests.cs b/tests/UnitTests/WorkflowExecuter.Tests/Common/ArtifactMapperTests.cs index 8745ae06e..0ba61cfda 100644 --- a/tests/UnitTests/WorkflowExecuter.Tests/Common/ArtifactMapperTests.cs +++ b/tests/UnitTests/WorkflowExecuter.Tests/Common/ArtifactMapperTests.cs @@ -115,6 +115,16 @@ public async Task ConvertArtifactVariablesToPath_MultipleInvalid_ReturnsEmptyDic { Name = "outputtaskdir", Value = "{{ sandwich.executions.coffee.output_dir }}" + }, + new Artifact + { + Name = "outputtaskdir4", + Value = "{{ sandwich.executions.coffee.artifacts.test }}" + }, + new Artifact + { + Name = "outputtaskdir2", + Value = "{{sandwich.executions.coffee.output_dir" } }; From 302c26c7c3b392ab427e78dcf4c41b3de4ac9657 Mon Sep 17 00:00:00 2001 From: Jack Schofield Date: Wed, 25 May 2022 09:18:08 +0100 Subject: [PATCH 6/9] workflow executer changes Signed-off-by: Jack Schofield --- ...eploy.WorkflowManager.Configuration.csproj | 2 +- src/Configuration/packages.lock.json | 6 +- src/Contracts/Models/Artifact.cs | 3 + ...loy.WorkflowManager.PayloadListener.csproj | 2 +- src/PayloadListener/packages.lock.json | 10 +- ...loy.WorkflowManager.TaskManager.API.csproj | 2 +- src/TaskManager/API/packages.lock.json | 6 +- ....Deploy.WorkflowManager.TaskManager.csproj | 2 +- ...oy.WorkflowManager.TaskManager.Argo.csproj | 2 +- .../Plug-ins/Argo/packages.lock.json | 8 +- src/TaskManager/packages.lock.json | 10 +- src/WorkflowExecuter/Common/ArtifactMapper.cs | 29 +++-- .../Common/IArtifactMapper.cs | 9 +- ...loy.WorkloadManager.WorkfowExecuter.csproj | 2 +- .../Services/WorkflowExecuterService.cs | 24 +++- src/WorkflowExecuter/packages.lock.json | 8 +- src/WorkflowManager/packages.lock.json | 10 +- .../Common/ArtifactMapperTests.cs | 109 ++++++++++++++++-- .../Services/WorkflowExecuterServiceTests.cs | 96 ++++++++++++++- .../WorkflowManager.Tests/packages.lock.json | 10 +- 20 files changed, 280 insertions(+), 70 deletions(-) diff --git a/src/Configuration/Monai.Deploy.WorkflowManager.Configuration.csproj b/src/Configuration/Monai.Deploy.WorkflowManager.Configuration.csproj index ed834c6a3..383cf2f13 100644 --- a/src/Configuration/Monai.Deploy.WorkflowManager.Configuration.csproj +++ b/src/Configuration/Monai.Deploy.WorkflowManager.Configuration.csproj @@ -16,7 +16,7 @@ SPDX-License-Identifier: Apache License 2.0 - + diff --git a/src/Configuration/packages.lock.json b/src/Configuration/packages.lock.json index b206b78e5..3907c8ee2 100644 --- a/src/Configuration/packages.lock.json +++ b/src/Configuration/packages.lock.json @@ -38,9 +38,9 @@ }, "Monai.Deploy.Storage": { "type": "Direct", - "requested": "[0.1.0-rc0020, )", - "resolved": "0.1.0-rc0020", - "contentHash": "SNNzbGPte4S2CKzsYI5g723aRMNG8AVFNYR847hHl8JQXCD5f6ugxEJUid4/hpu88HxPPigDm7sMrc5H4EwtZQ==", + "requested": "[0.1.0-rc0021, )", + "resolved": "0.1.0-rc0021", + "contentHash": "jealmI38hO1f8iTGt9ACYUx5sIyUnrf5tMVA7DNekzJ9mufhTMpNgukCqmkvYpXeljaPBp6yWn7NzfjiS1fnhw==", "dependencies": { "AWSSDK.SecurityToken": "3.7.1.141", "Ardalis.GuardClauses": "4.0.0", diff --git a/src/Contracts/Models/Artifact.cs b/src/Contracts/Models/Artifact.cs index 309038262..d3e6067dc 100644 --- a/src/Contracts/Models/Artifact.cs +++ b/src/Contracts/Models/Artifact.cs @@ -12,5 +12,8 @@ public class Artifact [JsonProperty(PropertyName = "value")] public string Value { get; set; } + + [JsonProperty(PropertyName = "mandatory")] + public bool Mandatory { get; set; } } } diff --git a/src/PayloadListener/Monai.Deploy.WorkflowManager.PayloadListener.csproj b/src/PayloadListener/Monai.Deploy.WorkflowManager.PayloadListener.csproj index a9c396033..deac063b6 100644 --- a/src/PayloadListener/Monai.Deploy.WorkflowManager.PayloadListener.csproj +++ b/src/PayloadListener/Monai.Deploy.WorkflowManager.PayloadListener.csproj @@ -19,7 +19,7 @@ SPDX-License-Identifier: Apache License 2.0 - + diff --git a/src/PayloadListener/packages.lock.json b/src/PayloadListener/packages.lock.json index 895e5acea..8e2215281 100644 --- a/src/PayloadListener/packages.lock.json +++ b/src/PayloadListener/packages.lock.json @@ -38,9 +38,9 @@ }, "Monai.Deploy.Storage": { "type": "Direct", - "requested": "[0.1.0-rc0020, )", - "resolved": "0.1.0-rc0020", - "contentHash": "SNNzbGPte4S2CKzsYI5g723aRMNG8AVFNYR847hHl8JQXCD5f6ugxEJUid4/hpu88HxPPigDm7sMrc5H4EwtZQ==", + "requested": "[0.1.0-rc0021, )", + "resolved": "0.1.0-rc0021", + "contentHash": "jealmI38hO1f8iTGt9ACYUx5sIyUnrf5tMVA7DNekzJ9mufhTMpNgukCqmkvYpXeljaPBp6yWn7NzfjiS1fnhw==", "dependencies": { "AWSSDK.SecurityToken": "3.7.1.141", "Ardalis.GuardClauses": "4.0.0", @@ -359,7 +359,7 @@ "Microsoft.Extensions.Configuration": "6.0.1", "Microsoft.Extensions.Options": "6.0.0", "Monai.Deploy.Messaging": "0.1.0-rc0038", - "Monai.Deploy.Storage": "0.1.0-rc0020", + "Monai.Deploy.Storage": "0.1.0-rc0021", "Newtonsoft.Json": "13.0.1", "System.IO.Abstractions": "16.1.25" } @@ -399,7 +399,7 @@ "AWSSDK.SecurityToken": "3.7.1.141", "Ardalis.GuardClauses": "4.0.1", "Monai.Deploy.Messaging": "0.1.0-rc0038", - "Monai.Deploy.Storage": "0.1.0-rc0020", + "Monai.Deploy.Storage": "0.1.0-rc0021", "Monai.Deploy.WorkflowManager.Configuration": "1.0.0", "Monai.Deploy.WorkflowManager.Contracts": "1.0.0", "Monai.Deploy.WorkflowManager.Database": "1.0.0", diff --git a/src/TaskManager/API/Monai.Deploy.WorkflowManager.TaskManager.API.csproj b/src/TaskManager/API/Monai.Deploy.WorkflowManager.TaskManager.API.csproj index 04232c90a..8643c3199 100644 --- a/src/TaskManager/API/Monai.Deploy.WorkflowManager.TaskManager.API.csproj +++ b/src/TaskManager/API/Monai.Deploy.WorkflowManager.TaskManager.API.csproj @@ -23,7 +23,7 @@ SPDX-License-Identifier: Apache License 2.0 - + diff --git a/src/TaskManager/API/packages.lock.json b/src/TaskManager/API/packages.lock.json index 16ed3b757..ff765a110 100644 --- a/src/TaskManager/API/packages.lock.json +++ b/src/TaskManager/API/packages.lock.json @@ -18,9 +18,9 @@ }, "Monai.Deploy.Storage": { "type": "Direct", - "requested": "[0.1.0-rc0020, )", - "resolved": "0.1.0-rc0020", - "contentHash": "SNNzbGPte4S2CKzsYI5g723aRMNG8AVFNYR847hHl8JQXCD5f6ugxEJUid4/hpu88HxPPigDm7sMrc5H4EwtZQ==", + "requested": "[0.1.0-rc0021, )", + "resolved": "0.1.0-rc0021", + "contentHash": "jealmI38hO1f8iTGt9ACYUx5sIyUnrf5tMVA7DNekzJ9mufhTMpNgukCqmkvYpXeljaPBp6yWn7NzfjiS1fnhw==", "dependencies": { "AWSSDK.SecurityToken": "3.7.1.141", "Ardalis.GuardClauses": "4.0.0", diff --git a/src/TaskManager/Monai.Deploy.WorkflowManager.TaskManager.csproj b/src/TaskManager/Monai.Deploy.WorkflowManager.TaskManager.csproj index e4df788e6..845d6f017 100644 --- a/src/TaskManager/Monai.Deploy.WorkflowManager.TaskManager.csproj +++ b/src/TaskManager/Monai.Deploy.WorkflowManager.TaskManager.csproj @@ -39,7 +39,7 @@ SPDX-License-Identifier: Apache License 2.0 - + diff --git a/src/TaskManager/Plug-ins/Argo/Monai.Deploy.WorkflowManager.TaskManager.Argo.csproj b/src/TaskManager/Plug-ins/Argo/Monai.Deploy.WorkflowManager.TaskManager.Argo.csproj index 8b574d360..33921a4d4 100644 --- a/src/TaskManager/Plug-ins/Argo/Monai.Deploy.WorkflowManager.TaskManager.Argo.csproj +++ b/src/TaskManager/Plug-ins/Argo/Monai.Deploy.WorkflowManager.TaskManager.Argo.csproj @@ -23,7 +23,7 @@ SPDX-License-Identifier: Apache License 2.0 runtime; build; native; contentfiles; analyzers; buildtransitive - + diff --git a/src/TaskManager/Plug-ins/Argo/packages.lock.json b/src/TaskManager/Plug-ins/Argo/packages.lock.json index 7a9dda53e..ea6b7e999 100644 --- a/src/TaskManager/Plug-ins/Argo/packages.lock.json +++ b/src/TaskManager/Plug-ins/Argo/packages.lock.json @@ -48,9 +48,9 @@ }, "Monai.Deploy.Storage": { "type": "Direct", - "requested": "[0.1.0-rc0020, )", - "resolved": "0.1.0-rc0020", - "contentHash": "SNNzbGPte4S2CKzsYI5g723aRMNG8AVFNYR847hHl8JQXCD5f6ugxEJUid4/hpu88HxPPigDm7sMrc5H4EwtZQ==", + "requested": "[0.1.0-rc0021, )", + "resolved": "0.1.0-rc0021", + "contentHash": "jealmI38hO1f8iTGt9ACYUx5sIyUnrf5tMVA7DNekzJ9mufhTMpNgukCqmkvYpXeljaPBp6yWn7NzfjiS1fnhw==", "dependencies": { "AWSSDK.SecurityToken": "3.7.1.141", "Ardalis.GuardClauses": "4.0.0", @@ -629,7 +629,7 @@ "type": "Project", "dependencies": { "Monai.Deploy.Messaging": "0.1.0-rc0038", - "Monai.Deploy.Storage": "0.1.0-rc0020" + "Monai.Deploy.Storage": "0.1.0-rc0021" } } } diff --git a/src/TaskManager/packages.lock.json b/src/TaskManager/packages.lock.json index 8cfac393f..9f1d21fc8 100644 --- a/src/TaskManager/packages.lock.json +++ b/src/TaskManager/packages.lock.json @@ -74,9 +74,9 @@ }, "Monai.Deploy.Storage": { "type": "Direct", - "requested": "[0.1.0-rc0020, )", - "resolved": "0.1.0-rc0020", - "contentHash": "SNNzbGPte4S2CKzsYI5g723aRMNG8AVFNYR847hHl8JQXCD5f6ugxEJUid4/hpu88HxPPigDm7sMrc5H4EwtZQ==", + "requested": "[0.1.0-rc0021, )", + "resolved": "0.1.0-rc0021", + "contentHash": "jealmI38hO1f8iTGt9ACYUx5sIyUnrf5tMVA7DNekzJ9mufhTMpNgukCqmkvYpXeljaPBp6yWn7NzfjiS1fnhw==", "dependencies": { "AWSSDK.SecurityToken": "3.7.1.141", "Ardalis.GuardClauses": "4.0.0", @@ -563,7 +563,7 @@ "Microsoft.Extensions.Configuration": "6.0.1", "Microsoft.Extensions.Options": "6.0.0", "Monai.Deploy.Messaging": "0.1.0-rc0038", - "Monai.Deploy.Storage": "0.1.0-rc0020", + "Monai.Deploy.Storage": "0.1.0-rc0021", "Newtonsoft.Json": "13.0.1", "System.IO.Abstractions": "16.1.25" } @@ -601,7 +601,7 @@ "type": "Project", "dependencies": { "Monai.Deploy.Messaging": "0.1.0-rc0038", - "Monai.Deploy.Storage": "0.1.0-rc0020" + "Monai.Deploy.Storage": "0.1.0-rc0021" } } } diff --git a/src/WorkflowExecuter/Common/ArtifactMapper.cs b/src/WorkflowExecuter/Common/ArtifactMapper.cs index 0ecea822d..eb3db4db2 100644 --- a/src/WorkflowExecuter/Common/ArtifactMapper.cs +++ b/src/WorkflowExecuter/Common/ArtifactMapper.cs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache License 2.0 using Ardalis.GuardClauses; +using Monai.Deploy.Storage; using Monai.Deploy.WorkflowManager.Contracts.Models; using Monai.Deploy.WorkflowManager.Database.Interfaces; @@ -10,13 +11,18 @@ namespace Monai.Deploy.WorkloadManager.WorkfowExecuter.Common public class ArtifactMapper : IArtifactMapper { private readonly IWorkflowInstanceRepository _workflowInstanceRepository; + private readonly IStorageService _storageService; - public ArtifactMapper(IWorkflowInstanceRepository workflowInstanceRepository) + public ArtifactMapper( + IWorkflowInstanceRepository workflowInstanceRepository, + IStorageService storageService + ) { _workflowInstanceRepository = workflowInstanceRepository ?? throw new ArgumentNullException(nameof(workflowInstanceRepository)); + _storageService = storageService ?? throw new ArgumentNullException(nameof(storageService)); } - public async Task> ConvertArtifactVariablesToPath(Artifact[] artifacts, string payloadId, string workflowInstanceId) + public async Task> ConvertArtifactVariablesToPath(Artifact[] artifacts, string payloadId, string workflowInstanceId, string bucketId) { Guard.Against.Null(artifacts); Guard.Against.NullOrWhiteSpace(payloadId); @@ -34,14 +40,17 @@ public async Task> ConvertArtifactVariablesToPath(Art continue; } - var mappedArtifact = await ConvertVariableStringToPath(artifact, variableString, workflowInstanceId, payloadId); + var mappedArtifact = await ConvertVariableStringToPath(artifact, variableString, workflowInstanceId, payloadId, bucketId); - if (mappedArtifact.Equals(default(KeyValuePair))) + if (!mappedArtifact.Equals(default(KeyValuePair))) { - continue; + artifactPathDictionary.Add(mappedArtifact.Key, mappedArtifact.Value); } - artifactPathDictionary.Add(mappedArtifact.Key, mappedArtifact.Value); + if (artifact.Mandatory) + { + throw new FileNotFoundException($"Mandatory artifact was not found: {artifact.Name}, {artifact.Value}"); + } } return artifactPathDictionary; @@ -63,11 +72,11 @@ private static bool TrimArtifactVariable(string valueString, out string variable return true; } - private async Task> ConvertVariableStringToPath(Artifact artifact, string variableString, string workflowInstanceId, string payloadId) + private async Task> ConvertVariableStringToPath(Artifact artifact, string variableString, string workflowInstanceId, string payloadId, string bucketId) { if (variableString.StartsWith("context.input")) { - return new KeyValuePair(artifact.Name, $"{payloadId}/dcm"); + return _storageService.VerifyObjectExists(bucketId, new KeyValuePair(artifact.Name, $"{payloadId}/dcm/")); } if (variableString.StartsWith("context.executions")) @@ -81,7 +90,7 @@ private async Task> ConvertVariableStringToPath(Art if (variableLocation == "output_dir") { - return new KeyValuePair(artifact.Name, task.OutputDirectory); + return _storageService.VerifyObjectExists(bucketId, new KeyValuePair(artifact.Name, task.OutputDirectory)); } if (variableLocation == "artifacts") @@ -91,7 +100,7 @@ private async Task> ConvertVariableStringToPath(Art if (!outputArtifact.Equals(default(KeyValuePair))) { - return new KeyValuePair(outputArtifact.Key, outputArtifact.Value); + return _storageService.VerifyObjectExists(bucketId, new KeyValuePair(outputArtifact.Key, outputArtifact.Value)); } } } diff --git a/src/WorkflowExecuter/Common/IArtifactMapper.cs b/src/WorkflowExecuter/Common/IArtifactMapper.cs index d6215296e..a957f36c3 100644 --- a/src/WorkflowExecuter/Common/IArtifactMapper.cs +++ b/src/WorkflowExecuter/Common/IArtifactMapper.cs @@ -7,6 +7,13 @@ namespace Monai.Deploy.WorkloadManager.WorkfowExecuter.Common { public interface IArtifactMapper { - Task> ConvertArtifactVariablesToPath(Artifact[] artifacts, string payloadId, string workflowInstanceId); + /// + /// Converts an array of artifacts to a dictionary of artifact path variables. + /// + /// Array of artifacts to convert. + /// Payload id to check against. + /// Workflow instance id to check against. + /// Bucket id used to verify. + Task> ConvertArtifactVariablesToPath(Artifact[] artifacts, string payloadId, string workflowInstanceId, string bucketId); } } diff --git a/src/WorkflowExecuter/Monai.Deploy.WorkloadManager.WorkfowExecuter.csproj b/src/WorkflowExecuter/Monai.Deploy.WorkloadManager.WorkfowExecuter.csproj index 49db2cb73..e4220bce1 100644 --- a/src/WorkflowExecuter/Monai.Deploy.WorkloadManager.WorkfowExecuter.csproj +++ b/src/WorkflowExecuter/Monai.Deploy.WorkloadManager.WorkfowExecuter.csproj @@ -14,7 +14,7 @@ - + diff --git a/src/WorkflowExecuter/Services/WorkflowExecuterService.cs b/src/WorkflowExecuter/Services/WorkflowExecuterService.cs index ab8e3406b..4159513bc 100644 --- a/src/WorkflowExecuter/Services/WorkflowExecuterService.cs +++ b/src/WorkflowExecuter/Services/WorkflowExecuterService.cs @@ -80,7 +80,7 @@ await _workflowRepository.GetWorkflowsByAeTitleAsync(message.CalledAeTitle) as L var workflowInstances = new List(); - workflows.ForEach(async (workflow) => workflowInstances.Add(await CreateWorkFlowIntsance(message, workflow))); + workflows.ForEach(async (workflow) => workflowInstances.Add(await CreateWorkflowInstance(message, workflow))); var existingInstances = await _workflowInstanceRepository.GetByWorkflowsIdsAsync(workflowInstances.Select(w => w.WorkflowId).ToList()); workflowInstances.RemoveAll(i => existingInstances.ToList().Exists(e => e.WorkflowId == i.WorkflowId && e.PayloadId == i.PayloadId)); @@ -99,6 +99,11 @@ await _workflowRepository.GetWorkflowsByAeTitleAsync(message.CalledAeTitle) as L foreach (var workflowInstance in workflowInstances) { + if (workflowInstance.Status == Status.Failed) + { + continue; + } + var task = workflowInstance.Tasks.FirstOrDefault(); if (task.Status != TaskExecutionStatus.Created) @@ -162,7 +167,7 @@ public async Task ProcessTaskUpdate(TaskUpdateEvent message) return false; } - var validOutputArtifacts = await _storageService.VerifyObjectsExist(workflowInstance.BucketId, message.OutputArtifacts); + var validOutputArtifacts = _storageService.VerifyObjectsExist(workflowInstance.BucketId, message.OutputArtifacts); workflowInstance.Tasks?.ForEach(t => { @@ -284,7 +289,7 @@ private async Task DispatchTask(WorkflowInstance workflowInstance, TaskExe return await _workflowInstanceRepository.UpdateTaskStatusAsync(workflowInstance.Id, taskExec.TaskId, TaskExecutionStatus.Dispatched); } - private async Task CreateWorkFlowIntsance(WorkflowRequestEvent message, WorkflowRevision workflow) + private async Task CreateWorkflowInstance(WorkflowRequestEvent message, WorkflowRevision workflow) { Guard.Against.Null(message, nameof(message)); Guard.Against.Null(workflow, nameof(workflow)); @@ -318,7 +323,14 @@ private async Task CreateWorkFlowIntsance(WorkflowRequestEvent // firstTask = template ?? firstTask; //} - tasks.Add(await CreateTaskExecution(firstTask, workflowInstance.Id, message.Bucket, message.PayloadId.ToString())); + try + { + tasks.Add(await CreateTaskExecution(firstTask, workflowInstance.Id, message.Bucket, message.PayloadId.ToString())); + } + catch (FileNotFoundException e) + { + workflowInstance.Status = Status.Failed; + } } workflowInstance.Tasks = tasks; @@ -344,8 +356,8 @@ private async Task CreateTaskExecution(TaskObject task, string wo TaskPluginArguments = task.Args ?? new Dictionary { }, TaskId = task.Id, Status = TaskExecutionStatus.Created, - InputArtifacts = await _artifactMapper.ConvertArtifactVariablesToPath(task?.Artifacts?.Input ?? new Artifact[] { }, payloadId, workflowInstanceId), - OutputDirectory = $"{payloadId}/workflows/{workflowInstanceId}/{executionId}", + InputArtifacts = await _artifactMapper.ConvertArtifactVariablesToPath(task?.Artifacts?.Input ?? new Artifact[] { }, payloadId, workflowInstanceId, bucketName), + OutputDirectory = $"{payloadId}/workflows/{workflowInstanceId}/{executionId}/", Metadata = { } }; } diff --git a/src/WorkflowExecuter/packages.lock.json b/src/WorkflowExecuter/packages.lock.json index e05d15b4b..48ddd163a 100644 --- a/src/WorkflowExecuter/packages.lock.json +++ b/src/WorkflowExecuter/packages.lock.json @@ -36,9 +36,9 @@ }, "Monai.Deploy.Storage": { "type": "Direct", - "requested": "[0.1.0-rc0020, )", - "resolved": "0.1.0-rc0020", - "contentHash": "SNNzbGPte4S2CKzsYI5g723aRMNG8AVFNYR847hHl8JQXCD5f6ugxEJUid4/hpu88HxPPigDm7sMrc5H4EwtZQ==", + "requested": "[0.1.0-rc0021, )", + "resolved": "0.1.0-rc0021", + "contentHash": "jealmI38hO1f8iTGt9ACYUx5sIyUnrf5tMVA7DNekzJ9mufhTMpNgukCqmkvYpXeljaPBp6yWn7NzfjiS1fnhw==", "dependencies": { "AWSSDK.SecurityToken": "3.7.1.141", "Ardalis.GuardClauses": "4.0.0", @@ -333,7 +333,7 @@ "Microsoft.Extensions.Configuration": "6.0.1", "Microsoft.Extensions.Options": "6.0.0", "Monai.Deploy.Messaging": "0.1.0-rc0038", - "Monai.Deploy.Storage": "0.1.0-rc0020", + "Monai.Deploy.Storage": "0.1.0-rc0021", "Newtonsoft.Json": "13.0.1", "System.IO.Abstractions": "16.1.25" } diff --git a/src/WorkflowManager/packages.lock.json b/src/WorkflowManager/packages.lock.json index 29990fca1..586d6cd0d 100644 --- a/src/WorkflowManager/packages.lock.json +++ b/src/WorkflowManager/packages.lock.json @@ -490,8 +490,8 @@ }, "Monai.Deploy.Storage": { "type": "Transitive", - "resolved": "0.1.0-rc0020", - "contentHash": "SNNzbGPte4S2CKzsYI5g723aRMNG8AVFNYR847hHl8JQXCD5f6ugxEJUid4/hpu88HxPPigDm7sMrc5H4EwtZQ==", + "resolved": "0.1.0-rc0021", + "contentHash": "jealmI38hO1f8iTGt9ACYUx5sIyUnrf5tMVA7DNekzJ9mufhTMpNgukCqmkvYpXeljaPBp6yWn7NzfjiS1fnhw==", "dependencies": { "AWSSDK.SecurityToken": "3.7.1.141", "Ardalis.GuardClauses": "4.0.0", @@ -722,7 +722,7 @@ "Microsoft.Extensions.Configuration": "6.0.1", "Microsoft.Extensions.Options": "6.0.0", "Monai.Deploy.Messaging": "0.1.0-rc0038", - "Monai.Deploy.Storage": "0.1.0-rc0020", + "Monai.Deploy.Storage": "0.1.0-rc0021", "Newtonsoft.Json": "13.0.1", "System.IO.Abstractions": "16.1.25" } @@ -762,7 +762,7 @@ "Ardalis.GuardClauses": "4.0.1", "Microsoft.Extensions.Hosting.Abstractions": "6.0.0", "Monai.Deploy.Messaging": "0.1.0-rc0038", - "Monai.Deploy.Storage": "0.1.0-rc0020", + "Monai.Deploy.Storage": "0.1.0-rc0021", "Monai.Deploy.WorkflowManager.Common": "1.0.0", "Monai.Deploy.WorkflowManager.Configuration": "1.0.0", "Monai.Deploy.WorkflowManager.Contracts": "1.0.0", @@ -776,7 +776,7 @@ "AWSSDK.SecurityToken": "3.7.1.141", "Ardalis.GuardClauses": "4.0.1", "Monai.Deploy.Messaging": "0.1.0-rc0038", - "Monai.Deploy.Storage": "0.1.0-rc0020", + "Monai.Deploy.Storage": "0.1.0-rc0021", "Monai.Deploy.WorkflowManager.Configuration": "1.0.0", "Monai.Deploy.WorkflowManager.Contracts": "1.0.0", "Monai.Deploy.WorkflowManager.Database": "1.0.0", diff --git a/tests/UnitTests/WorkflowExecuter.Tests/Common/ArtifactMapperTests.cs b/tests/UnitTests/WorkflowExecuter.Tests/Common/ArtifactMapperTests.cs index 0ba61cfda..daac02dee 100644 --- a/tests/UnitTests/WorkflowExecuter.Tests/Common/ArtifactMapperTests.cs +++ b/tests/UnitTests/WorkflowExecuter.Tests/Common/ArtifactMapperTests.cs @@ -3,9 +3,11 @@ using System; using System.Collections.Generic; +using System.IO; using System.Threading.Tasks; using FluentAssertions; using Monai.Deploy.Messaging.Events; +using Monai.Deploy.Storage; using Monai.Deploy.WorkflowManager.Contracts.Models; using Monai.Deploy.WorkflowManager.Database.Interfaces; using Monai.Deploy.WorkloadManager.Contracts.Models; @@ -19,12 +21,17 @@ public class ArtifactMapperTests { private IArtifactMapper ArtifactMapper { get; set; } + private IStorageService StorageService { get; set; } + private readonly Mock _workflowInstanceRepository; + private readonly Mock _storageService; + public ArtifactMapperTests() { _workflowInstanceRepository = new Mock(); + _storageService = new Mock(); - ArtifactMapper = new ArtifactMapper(_workflowInstanceRepository.Object); + ArtifactMapper = new ArtifactMapper(_workflowInstanceRepository.Object, _storageService.Object); } [Fact] @@ -55,9 +62,9 @@ public async Task ConvertArtifactVariablesToPath_MultipleArtifacts_ReturnsMapped var expected = new Dictionary { - { "dicom", $"{payloadId}/workflows/{workflowInstanceId}/{executionId}" }, - { "dicomimage", $"{payloadId}/dcm" }, - { "outputtaskdir", $"{payloadId}/workflows/{workflowInstanceId}/{executionId}" } + { "dicom", $"{payloadId}/workflows/{workflowInstanceId}/{executionId}/" }, + { "dicomimage", $"{payloadId}/dcm/" }, + { "outputtaskdir", $"{payloadId}/workflows/{workflowInstanceId}/{executionId}/" } }; var workflowInstance = new WorkflowInstance @@ -76,14 +83,14 @@ public async Task ConvertArtifactVariablesToPath_MultipleArtifacts_ReturnsMapped Status = TaskExecutionStatus.Dispatched, OutputArtifacts = new Dictionary { - { "dicom", $"{payloadId}/workflows/{workflowInstanceId}/{executionId}" } + { "dicom", $"{payloadId}/workflows/{workflowInstanceId}/{executionId}/" } } }, new TaskExecution { TaskId = "coffee", Status = TaskExecutionStatus.Created, - OutputDirectory = $"{payloadId}/workflows/{workflowInstanceId}/{executionId}" + OutputDirectory = $"{payloadId}/workflows/{workflowInstanceId}/{executionId}/" } } }; @@ -91,11 +98,97 @@ public async Task ConvertArtifactVariablesToPath_MultipleArtifacts_ReturnsMapped _workflowInstanceRepository.Setup(w => w.GetTaskByIdAsync(workflowInstance.Id, "image_type_detector")).ReturnsAsync(workflowInstance.Tasks[0]); _workflowInstanceRepository.Setup(w => w.GetTaskByIdAsync(workflowInstance.Id, "coffee")).ReturnsAsync(workflowInstance.Tasks[1]); - var response = await ArtifactMapper.ConvertArtifactVariablesToPath(artifacts, workflowInstance.PayloadId, workflowInstance.Id); + var value1 = new KeyValuePair("dicom", $"{payloadId}/workflows/{workflowInstanceId}/{executionId}/"); + var value2 = new KeyValuePair("dicomimage", $"{payloadId}/dcm/"); + var value3 = new KeyValuePair("outputtaskdir", $"{payloadId}/workflows/{workflowInstanceId}/{executionId}/"); + + _storageService.Setup(w => w.VerifyObjectExists(workflowInstance.BucketId, value1)).Returns(value1); + _storageService.Setup(w => w.VerifyObjectExists(workflowInstance.BucketId, value2)).Returns(value2); + _storageService.Setup(w => w.VerifyObjectExists(workflowInstance.BucketId, value3)).Returns(value3); + + var response = await ArtifactMapper.ConvertArtifactVariablesToPath(artifacts, workflowInstance.PayloadId, workflowInstance.Id, workflowInstance.BucketId); response.Should().BeEquivalentTo(expected); } + [Fact] + public async Task ConvertArtifactVariablesToPath_RequiredArtifact_ThrowsException() + { + var artifacts = new Artifact[] + { + new Artifact + { + Name = "taskartifact", + Value = "{{ context.executions.image_type_detector.artifacts.dicom }}", + Mandatory = true + }, + new Artifact + { + Name = "dicomimage", + Value = "{{ context.input }}", + Mandatory = true + }, + new Artifact + { + Name = "outputtaskdir", + Value = "{{ context.executions.coffee.output_dir }}", + Mandatory = true + } + }; + + var payloadId = Guid.NewGuid().ToString(); + var workflowInstanceId = Guid.NewGuid().ToString(); + var executionId = Guid.NewGuid().ToString(); + + var expected = new Dictionary + { + { "dicom", $"{payloadId}/workflows/{workflowInstanceId}/{executionId}/" }, + { "dicomimage", $"{payloadId}/dcm/" }, + { "outputtaskdir", $"{payloadId}/workflows/{workflowInstanceId}/{executionId}/" } + }; + + var workflowInstance = new WorkflowInstance + { + Id = workflowInstanceId, + WorkflowId = Guid.NewGuid().ToString(), + PayloadId = payloadId, + Status = Status.Created, + BucketId = "bucket", + Tasks = new List + { + new TaskExecution + { + TaskId = "image_type_detector", + ExecutionId = executionId, + Status = TaskExecutionStatus.Dispatched, + OutputArtifacts = new Dictionary + { + { "dicom", $"{payloadId}/workflows/{workflowInstanceId}/{executionId}/" } + } + }, + new TaskExecution + { + TaskId = "coffee", + Status = TaskExecutionStatus.Created, + OutputDirectory = $"{payloadId}/workflows/{workflowInstanceId}/{executionId}/" + } + } + }; + + _workflowInstanceRepository.Setup(w => w.GetTaskByIdAsync(workflowInstance.Id, "image_type_detector")).ReturnsAsync(workflowInstance.Tasks[0]); + _workflowInstanceRepository.Setup(w => w.GetTaskByIdAsync(workflowInstance.Id, "coffee")).ReturnsAsync(workflowInstance.Tasks[1]); + + var value1 = new KeyValuePair("dicom", $"{payloadId}/workflows/{workflowInstanceId}/{executionId}/"); + var value2 = new KeyValuePair("dicomimage", $"{payloadId}/dcm/"); + var value3 = new KeyValuePair("outputtaskdir", $"{payloadId}/workflows/{workflowInstanceId}/{executionId}/"); + + _storageService.Setup(w => w.VerifyObjectExists(workflowInstance.BucketId, value1)).Returns(new KeyValuePair()); + _storageService.Setup(w => w.VerifyObjectExists(workflowInstance.BucketId, value2)).Returns(value2); + _storageService.Setup(w => w.VerifyObjectExists(workflowInstance.BucketId, value3)).Returns(value3); + + await Assert.ThrowsAsync(() => ArtifactMapper.ConvertArtifactVariablesToPath(artifacts, workflowInstance.PayloadId, workflowInstance.Id, workflowInstance.BucketId)); + } + [Fact] public async Task ConvertArtifactVariablesToPath_MultipleInvalid_ReturnsEmptyDict() { @@ -165,7 +258,7 @@ public async Task ConvertArtifactVariablesToPath_MultipleInvalid_ReturnsEmptyDic _workflowInstanceRepository.Setup(w => w.GetTaskByIdAsync(workflowInstance.Id, "image_type_detector")).ReturnsAsync(workflowInstance.Tasks[0]); _workflowInstanceRepository.Setup(w => w.GetTaskByIdAsync(workflowInstance.Id, "coffee")).ReturnsAsync(workflowInstance.Tasks[1]); - var response = await ArtifactMapper.ConvertArtifactVariablesToPath(artifacts, workflowInstance.PayloadId, workflowInstance.Id); + var response = await ArtifactMapper.ConvertArtifactVariablesToPath(artifacts, workflowInstance.PayloadId, workflowInstance.Id, workflowInstance.BucketId); response.Should().BeEquivalentTo(expected); } diff --git a/tests/UnitTests/WorkflowExecuter.Tests/Services/WorkflowExecuterServiceTests.cs b/tests/UnitTests/WorkflowExecuter.Tests/Services/WorkflowExecuterServiceTests.cs index 14bac99c3..77afd3f98 100644 --- a/tests/UnitTests/WorkflowExecuter.Tests/Services/WorkflowExecuterServiceTests.cs +++ b/tests/UnitTests/WorkflowExecuter.Tests/Services/WorkflowExecuterServiceTests.cs @@ -3,6 +3,7 @@ using System; using System.Collections.Generic; +using System.IO; using System.Threading.Tasks; using FluentAssertions; using Microsoft.Extensions.Logging; @@ -179,7 +180,7 @@ public async Task ProcessPayload_ValidWorkflowIdRequest_ReturnesTrue() _workflowInstanceRepository.Setup(w => w.CreateAsync(It.IsAny>())).ReturnsAsync(true); _workflowInstanceRepository.Setup(w => w.GetByWorkflowsIdsAsync(It.IsAny>())).ReturnsAsync(new List()); _workflowInstanceRepository.Setup(w => w.UpdateTaskStatusAsync(It.IsAny(), It.IsAny(), It.IsAny())).ReturnsAsync(true); - _artifactMapper.Setup(a => a.ConvertArtifactVariablesToPath(It.IsAny(), It.IsAny(), It.IsAny())).ReturnsAsync(new Dictionary()); + _artifactMapper.Setup(a => a.ConvertArtifactVariablesToPath(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())).ReturnsAsync(new Dictionary()); var result = await WorkflowExecuterService.ProcessPayload(workflowRequest); @@ -188,6 +189,91 @@ public async Task ProcessPayload_ValidWorkflowIdRequest_ReturnesTrue() Assert.True(result); } + [Fact] + public async Task ProcessPayload_FileNotFound_FailsWorkflow() + { + var workflowId1 = Guid.NewGuid().ToString(); + var workflowId2 = Guid.NewGuid().ToString(); + var workflowRequest = new WorkflowRequestEvent + { + Bucket = "testbucket", + CalledAeTitle = "aetitle", + CallingAeTitle = "aetitle", + CorrelationId = Guid.NewGuid().ToString(), + Timestamp = DateTime.UtcNow, + Workflows = new List + { + workflowId1.ToString(), + workflowId2.ToString() + } + }; + + var workflows = new List + { + new WorkflowRevision + { + Id = Guid.NewGuid().ToString(), + WorkflowId = workflowId1, + Revision = 1, + Workflow = new Workflow + { + Name = "Workflowname1", + Description = "Workflowdesc1", + Version = "1", + InformaticsGateway = new InformaticsGateway + { + AeTitle = "aetitle" + }, + Tasks = new TaskObject[] + { + new TaskObject { + Id = Guid.NewGuid().ToString(), + Type = "type", + Description = "taskdesc" + } + } + } + }, + new WorkflowRevision + { + Id = Guid.NewGuid().ToString(), + WorkflowId = workflowId2, + Revision = 1, + Workflow = new Workflow + { + Name = "Workflowname2", + Description = "Workflowdesc2", + Version = "1", + InformaticsGateway = new InformaticsGateway + { + AeTitle = "aetitle" + }, + Tasks = new TaskObject[] + { + new TaskObject { + Id = Guid.NewGuid().ToString(), + Type = "type", + Description = "taskdesc" + } + } + } + } + }; + + _workflowRepository.Setup(w => w.GetByWorkflowsIdsAsync(new List { workflowId1.ToString(), workflowId2.ToString() })).ReturnsAsync(workflows); + + _workflowInstanceRepository.Setup(w => w.CreateAsync(It.IsAny>())).ReturnsAsync(true); + _workflowInstanceRepository.Setup(w => w.GetByWorkflowsIdsAsync(It.IsAny>())).ReturnsAsync(new List()); + _workflowInstanceRepository.Setup(w => w.UpdateTaskStatusAsync(It.IsAny(), It.IsAny(), It.IsAny())).ReturnsAsync(true); + _artifactMapper.Setup(a => a.ConvertArtifactVariablesToPath(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())).ThrowsAsync(new FileNotFoundException()); + + var result = await WorkflowExecuterService.ProcessPayload(workflowRequest); + + _messageBrokerPublisherService.Verify(w => w.Publish(_configuration.Value.Messaging.Topics.TaskDispatchRequest, It.IsAny()), Times.Exactly(0)); + + Assert.True(result); + } + [Fact] public async Task ProcessPayload_WorkflowAlreadyStarted_TaskNotDispatched() { @@ -286,7 +372,7 @@ public async Task ProcessPayload_WorkflowAlreadyStarted_TaskNotDispatched() _workflowInstanceRepository.Setup(w => w.CreateAsync(It.IsAny>())).ReturnsAsync(true); _workflowInstanceRepository.Setup(w => w.GetByWorkflowsIdsAsync(It.IsAny>())).ReturnsAsync(workflowsInstance); _workflowInstanceRepository.Setup(w => w.UpdateTaskStatusAsync(It.IsAny(), It.IsAny(), It.IsAny())).ReturnsAsync(true); - _artifactMapper.Setup(a => a.ConvertArtifactVariablesToPath(It.IsAny(), It.IsAny(), It.IsAny())).ReturnsAsync(new Dictionary()); + _artifactMapper.Setup(a => a.ConvertArtifactVariablesToPath(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())).ReturnsAsync(new Dictionary()); var result = await WorkflowExecuterService.ProcessPayload(workflowRequest); @@ -359,7 +445,7 @@ public async Task ProcessTaskUpdate_ValidTaskUpdateEvent_ReturnsTrue() _workflowInstanceRepository.Setup(w => w.UpdateTaskStatusAsync(It.IsAny(), It.IsAny(), It.IsAny())).ReturnsAsync(true); _workflowInstanceRepository.Setup(w => w.GetByWorkflowInstanceIdAsync(workflowInstance.Id)).ReturnsAsync(workflowInstance); _workflowRepository.Setup(w => w.GetByWorkflowIdAsync(workflowInstance.WorkflowId)).ReturnsAsync(workflow); - _artifactMapper.Setup(a => a.ConvertArtifactVariablesToPath(It.IsAny(), It.IsAny(), It.IsAny())).ReturnsAsync(new Dictionary()); + _artifactMapper.Setup(a => a.ConvertArtifactVariablesToPath(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())).ReturnsAsync(new Dictionary()); var response = await WorkflowExecuterService.ProcessTaskUpdate(updateEvent); @@ -455,7 +541,7 @@ public async Task ProcessTaskUpdate_ValidTaskUpdateEventWithTaskDestinations_Ret _workflowInstanceRepository.Setup(w => w.GetByWorkflowInstanceIdAsync(workflowInstance.Id)).ReturnsAsync(workflowInstance); _workflowInstanceRepository.Setup(w => w.UpdateTasksAsync(workflowInstance.Id, It.IsAny>())).ReturnsAsync(true); _workflowRepository.Setup(w => w.GetByWorkflowIdAsync(workflowInstance.WorkflowId)).ReturnsAsync(workflow); - _artifactMapper.Setup(a => a.ConvertArtifactVariablesToPath(It.IsAny(), It.IsAny(), It.IsAny())).ReturnsAsync(new Dictionary()); + _artifactMapper.Setup(a => a.ConvertArtifactVariablesToPath(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())).ReturnsAsync(new Dictionary()); var response = await WorkflowExecuterService.ProcessTaskUpdate(updateEvent); @@ -750,7 +836,7 @@ public async Task ProcessTaskUpdate_ValidTaskUpdateEventWithOutputArtifacts_Retu _workflowInstanceRepository.Setup(w => w.GetByWorkflowInstanceIdAsync(workflowInstance.Id)).ReturnsAsync(workflowInstance); _workflowInstanceRepository.Setup(w => w.UpdateTasksAsync(workflowInstance.Id, It.IsAny>())).ReturnsAsync(true); _workflowRepository.Setup(w => w.GetByWorkflowIdAsync(workflowInstance.WorkflowId)).ReturnsAsync(workflow); - _storageService.Setup(w => w.VerifyObjectsExist(workflowInstance.BucketId, updateEvent.OutputArtifacts)).ReturnsAsync(updateEvent.OutputArtifacts); + _storageService.Setup(w => w.VerifyObjectsExist(workflowInstance.BucketId, updateEvent.OutputArtifacts)).Returns(updateEvent.OutputArtifacts); var response = await WorkflowExecuterService.ProcessTaskUpdate(updateEvent); diff --git a/tests/UnitTests/WorkflowManager.Tests/packages.lock.json b/tests/UnitTests/WorkflowManager.Tests/packages.lock.json index c4b2e00b6..391842334 100644 --- a/tests/UnitTests/WorkflowManager.Tests/packages.lock.json +++ b/tests/UnitTests/WorkflowManager.Tests/packages.lock.json @@ -605,8 +605,8 @@ }, "Monai.Deploy.Storage": { "type": "Transitive", - "resolved": "0.1.0-rc0020", - "contentHash": "SNNzbGPte4S2CKzsYI5g723aRMNG8AVFNYR847hHl8JQXCD5f6ugxEJUid4/hpu88HxPPigDm7sMrc5H4EwtZQ==", + "resolved": "0.1.0-rc0021", + "contentHash": "jealmI38hO1f8iTGt9ACYUx5sIyUnrf5tMVA7DNekzJ9mufhTMpNgukCqmkvYpXeljaPBp6yWn7NzfjiS1fnhw==", "dependencies": { "AWSSDK.SecurityToken": "3.7.1.141", "Ardalis.GuardClauses": "4.0.0", @@ -1402,7 +1402,7 @@ "Microsoft.Extensions.Configuration": "6.0.1", "Microsoft.Extensions.Options": "6.0.0", "Monai.Deploy.Messaging": "0.1.0-rc0038", - "Monai.Deploy.Storage": "0.1.0-rc0020", + "Monai.Deploy.Storage": "0.1.0-rc0021", "Newtonsoft.Json": "13.0.1", "System.IO.Abstractions": "16.1.25" } @@ -1442,7 +1442,7 @@ "Ardalis.GuardClauses": "4.0.1", "Microsoft.Extensions.Hosting.Abstractions": "6.0.0", "Monai.Deploy.Messaging": "0.1.0-rc0038", - "Monai.Deploy.Storage": "0.1.0-rc0020", + "Monai.Deploy.Storage": "0.1.0-rc0021", "Monai.Deploy.WorkflowManager.Common": "1.0.0", "Monai.Deploy.WorkflowManager.Configuration": "1.0.0", "Monai.Deploy.WorkflowManager.Contracts": "1.0.0", @@ -1456,7 +1456,7 @@ "AWSSDK.SecurityToken": "3.7.1.141", "Ardalis.GuardClauses": "4.0.1", "Monai.Deploy.Messaging": "0.1.0-rc0038", - "Monai.Deploy.Storage": "0.1.0-rc0020", + "Monai.Deploy.Storage": "0.1.0-rc0021", "Monai.Deploy.WorkflowManager.Configuration": "1.0.0", "Monai.Deploy.WorkflowManager.Contracts": "1.0.0", "Monai.Deploy.WorkflowManager.Database": "1.0.0", From 20353ca7cfd1a4530aa889a866bf2fb57876df7c Mon Sep 17 00:00:00 2001 From: Jack Schofield Date: Mon, 30 May 2022 15:08:19 +0100 Subject: [PATCH 7/9] use updated taskUpdate event Signed-off-by: Jack Schofield --- .../Extensions/StorageListExtensions.cs | 26 ++++++++ ...eploy.WorkflowManager.Configuration.csproj | 2 +- src/Configuration/packages.lock.json | 6 +- ...ai.Deploy.WorkflowManager.Contracts.csproj | 2 +- ...nai.Deploy.WorkflowManager.Database.csproj | 2 +- src/Database/packages.lock.json | 8 +-- src/Monai.Deploy.WorkflowManager.sln | 7 ++ ...loy.WorkflowManager.PayloadListener.csproj | 2 +- src/PayloadListener/packages.lock.json | 15 +++-- ...loy.WorkflowManager.TaskManager.API.csproj | 2 +- src/TaskManager/API/packages.lock.json | 6 +- ....Deploy.WorkflowManager.TaskManager.csproj | 2 +- ...oy.WorkflowManager.TaskManager.Argo.csproj | 2 +- .../Plug-ins/Argo/packages.lock.json | 12 ++-- src/TaskManager/packages.lock.json | 14 ++-- ...loy.WorkloadManager.WorkfowExecuter.csproj | 3 +- .../Services/WorkflowExecuterService.cs | 4 +- src/WorkflowExecuter/packages.lock.json | 21 ++++-- .../Monai.Deploy.WorkflowManager.csproj | 2 +- src/WorkflowManager/packages.lock.json | 17 ++--- ...WorkflowManager.Configuration.Tests.csproj | 2 +- ...rkflowManager.PayloadListener.Tests.csproj | 2 +- .../Services/WorkflowExecuterServiceTests.cs | 13 +++- ...Deploy.WorkflowManager.Common.Tests.csproj | 28 ++++++++ .../StorageListExtensionsTests.cs | 66 +++++++++++++++++++ .../WorkflowManager.Tests/packages.lock.json | 17 ++--- 26 files changed, 216 insertions(+), 67 deletions(-) create mode 100644 src/Common/Extensions/StorageListExtensions.cs create mode 100644 tests/UnitTests/WorkflowManager.Common.Tests/Monai.Deploy.WorkflowManager.Common.Tests.csproj create mode 100644 tests/UnitTests/WorkflowManager.Common.Tests/StorageListExtensionsTests.cs diff --git a/src/Common/Extensions/StorageListExtensions.cs b/src/Common/Extensions/StorageListExtensions.cs new file mode 100644 index 000000000..6ebf3c333 --- /dev/null +++ b/src/Common/Extensions/StorageListExtensions.cs @@ -0,0 +1,26 @@ +using Ardalis.GuardClauses; + +namespace Monai.Deploy.WorkflowManager.Common.Extensions +{ + public static class StorageListExtensions + { + public static Dictionary ToArtifactDictionary(this List storageList) + { + Guard.Against.Null(storageList, nameof(storageList)); + + var artifactDict = new Dictionary(); + + foreach (var storage in storageList) + { + if (string.IsNullOrWhiteSpace(storage.Name) || string.IsNullOrWhiteSpace(storage.RelativeRootPath)) + { + continue; + } + + artifactDict.Add(storage.Name, storage.RelativeRootPath); + } + + return artifactDict; + } + } +} diff --git a/src/Configuration/Monai.Deploy.WorkflowManager.Configuration.csproj b/src/Configuration/Monai.Deploy.WorkflowManager.Configuration.csproj index 383cf2f13..4eee66ab0 100644 --- a/src/Configuration/Monai.Deploy.WorkflowManager.Configuration.csproj +++ b/src/Configuration/Monai.Deploy.WorkflowManager.Configuration.csproj @@ -15,7 +15,7 @@ SPDX-License-Identifier: Apache License 2.0 - + diff --git a/src/Configuration/packages.lock.json b/src/Configuration/packages.lock.json index 3907c8ee2..0c8199f4c 100644 --- a/src/Configuration/packages.lock.json +++ b/src/Configuration/packages.lock.json @@ -24,9 +24,9 @@ }, "Monai.Deploy.Messaging": { "type": "Direct", - "requested": "[0.1.0-rc0038, )", - "resolved": "0.1.0-rc0038", - "contentHash": "j6RswItc5GE7b42T1KMMT9oXsOEizr6HKgso4WAV/evGmcwMnzuBdRJHNIfGWeDTTKO7Ld7+BJrSOEno5+udsg==", + "requested": "[0.1.0-rc0040, )", + "resolved": "0.1.0-rc0040", + "contentHash": "q5t6h24M+22JQqeBTD9wOOYYAvb1Vk29nkadcOJ05wT569jnF+MGmEherDn1moX45Ehj4TsCvQl3TD4HJlC1mA==", "dependencies": { "Ardalis.GuardClauses": "4.0.1", "Microsoft.Extensions.Configuration": "6.0.1", diff --git a/src/Contracts/Monai.Deploy.WorkflowManager.Contracts.csproj b/src/Contracts/Monai.Deploy.WorkflowManager.Contracts.csproj index f88d2a965..61fb91d44 100644 --- a/src/Contracts/Monai.Deploy.WorkflowManager.Contracts.csproj +++ b/src/Contracts/Monai.Deploy.WorkflowManager.Contracts.csproj @@ -18,7 +18,7 @@ SPDX-License-Identifier: Apache License 2.0 - + diff --git a/src/Database/Monai.Deploy.WorkflowManager.Database.csproj b/src/Database/Monai.Deploy.WorkflowManager.Database.csproj index 626b475b9..2ff560d94 100644 --- a/src/Database/Monai.Deploy.WorkflowManager.Database.csproj +++ b/src/Database/Monai.Deploy.WorkflowManager.Database.csproj @@ -20,7 +20,7 @@ - + diff --git a/src/Database/packages.lock.json b/src/Database/packages.lock.json index 5e90c91ea..c05ec08f2 100644 --- a/src/Database/packages.lock.json +++ b/src/Database/packages.lock.json @@ -29,9 +29,9 @@ }, "Monai.Deploy.Messaging": { "type": "Direct", - "requested": "[0.1.0-rc0038, )", - "resolved": "0.1.0-rc0038", - "contentHash": "j6RswItc5GE7b42T1KMMT9oXsOEizr6HKgso4WAV/evGmcwMnzuBdRJHNIfGWeDTTKO7Ld7+BJrSOEno5+udsg==", + "requested": "[0.1.0-rc0040, )", + "resolved": "0.1.0-rc0040", + "contentHash": "q5t6h24M+22JQqeBTD9wOOYYAvb1Vk29nkadcOJ05wT569jnF+MGmEherDn1moX45Ehj4TsCvQl3TD4HJlC1mA==", "dependencies": { "Ardalis.GuardClauses": "4.0.1", "Microsoft.Extensions.Configuration": "6.0.1", @@ -240,7 +240,7 @@ "dependencies": { "AWSSDK.SecurityToken": "3.7.1.141", "Microsoft.Extensions.Configuration": "6.0.1", - "Monai.Deploy.Messaging": "0.1.0-rc0038", + "Monai.Deploy.Messaging": "0.1.0-rc0040", "MongoDB.Bson": "2.15.0", "Newtonsoft.Json": "13.0.1" } diff --git a/src/Monai.Deploy.WorkflowManager.sln b/src/Monai.Deploy.WorkflowManager.sln index 765bc1cd5..fecf4a8f8 100644 --- a/src/Monai.Deploy.WorkflowManager.sln +++ b/src/Monai.Deploy.WorkflowManager.sln @@ -55,6 +55,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Monai.Deploy.WorkflowManage EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Monai.Deploy.WorkflowManager.ConditionsResolver.Tests", "..\tests\UnitTests\ConditionsResolver.Tests\Monai.Deploy.WorkflowManager.ConditionsResolver.Tests.csproj", "{918E4DE3-A7BF-4B7F-9B5A-5C36FEFA3C30}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Monai.Deploy.WorkflowManager.Common.Tests", "..\tests\UnitTests\WorkflowManager.Common.Tests\Monai.Deploy.WorkflowManager.Common.Tests.csproj", "{A44F975E-70CA-49D6-8513-78F2D5210EAF}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -157,6 +159,10 @@ Global {918E4DE3-A7BF-4B7F-9B5A-5C36FEFA3C30}.Debug|Any CPU.Build.0 = Debug|Any CPU {918E4DE3-A7BF-4B7F-9B5A-5C36FEFA3C30}.Release|Any CPU.ActiveCfg = Release|Any CPU {918E4DE3-A7BF-4B7F-9B5A-5C36FEFA3C30}.Release|Any CPU.Build.0 = Release|Any CPU + {A44F975E-70CA-49D6-8513-78F2D5210EAF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {A44F975E-70CA-49D6-8513-78F2D5210EAF}.Debug|Any CPU.Build.0 = Debug|Any CPU + {A44F975E-70CA-49D6-8513-78F2D5210EAF}.Release|Any CPU.ActiveCfg = Release|Any CPU + {A44F975E-70CA-49D6-8513-78F2D5210EAF}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -176,6 +182,7 @@ Global {2DA40575-4748-4198-BE57-F4AF070DE8E3} = {71DDEE7B-E213-4E39-A7F4-4646783A27F7} {89D3D817-CCFE-4933-9089-D1283F2EA1B5} = {71DDEE7B-E213-4E39-A7F4-4646783A27F7} {918E4DE3-A7BF-4B7F-9B5A-5C36FEFA3C30} = {71DDEE7B-E213-4E39-A7F4-4646783A27F7} + {A44F975E-70CA-49D6-8513-78F2D5210EAF} = {71DDEE7B-E213-4E39-A7F4-4646783A27F7} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {DC0D56C8-D8CB-45CE-B528-F3DCF86D63ED} diff --git a/src/PayloadListener/Monai.Deploy.WorkflowManager.PayloadListener.csproj b/src/PayloadListener/Monai.Deploy.WorkflowManager.PayloadListener.csproj index deac063b6..c4b810337 100644 --- a/src/PayloadListener/Monai.Deploy.WorkflowManager.PayloadListener.csproj +++ b/src/PayloadListener/Monai.Deploy.WorkflowManager.PayloadListener.csproj @@ -18,7 +18,7 @@ SPDX-License-Identifier: Apache License 2.0 - + diff --git a/src/PayloadListener/packages.lock.json b/src/PayloadListener/packages.lock.json index 8e2215281..d3221817e 100644 --- a/src/PayloadListener/packages.lock.json +++ b/src/PayloadListener/packages.lock.json @@ -24,9 +24,9 @@ }, "Monai.Deploy.Messaging": { "type": "Direct", - "requested": "[0.1.0-rc0038, )", - "resolved": "0.1.0-rc0038", - "contentHash": "j6RswItc5GE7b42T1KMMT9oXsOEizr6HKgso4WAV/evGmcwMnzuBdRJHNIfGWeDTTKO7Ld7+BJrSOEno5+udsg==", + "requested": "[0.1.0-rc0040, )", + "resolved": "0.1.0-rc0040", + "contentHash": "q5t6h24M+22JQqeBTD9wOOYYAvb1Vk29nkadcOJ05wT569jnF+MGmEherDn1moX45Ehj4TsCvQl3TD4HJlC1mA==", "dependencies": { "Ardalis.GuardClauses": "4.0.1", "Microsoft.Extensions.Configuration": "6.0.1", @@ -358,7 +358,7 @@ "dependencies": { "Microsoft.Extensions.Configuration": "6.0.1", "Microsoft.Extensions.Options": "6.0.0", - "Monai.Deploy.Messaging": "0.1.0-rc0038", + "Monai.Deploy.Messaging": "0.1.0-rc0040", "Monai.Deploy.Storage": "0.1.0-rc0021", "Newtonsoft.Json": "13.0.1", "System.IO.Abstractions": "16.1.25" @@ -369,7 +369,7 @@ "dependencies": { "AWSSDK.SecurityToken": "3.7.1.141", "Microsoft.Extensions.Configuration": "6.0.1", - "Monai.Deploy.Messaging": "0.1.0-rc0038", + "Monai.Deploy.Messaging": "0.1.0-rc0040", "MongoDB.Bson": "2.15.0", "Newtonsoft.Json": "13.0.1" } @@ -380,7 +380,7 @@ "Ardalis.GuardClauses": "4.0.1", "Microsoft.Extensions.Logging.Abstractions": "6.0.1", "Microsoft.Extensions.Options": "6.0.0", - "Monai.Deploy.Messaging": "0.1.0-rc0038", + "Monai.Deploy.Messaging": "0.1.0-rc0040", "Monai.Deploy.WorkflowManager.Contracts": "1.0.0", "Monai.Deploy.WorkflowManager.Logging": "1.0.0", "MongoDB.Bson": "2.15.0", @@ -398,8 +398,9 @@ "dependencies": { "AWSSDK.SecurityToken": "3.7.1.141", "Ardalis.GuardClauses": "4.0.1", - "Monai.Deploy.Messaging": "0.1.0-rc0038", + "Monai.Deploy.Messaging": "0.1.0-rc0040", "Monai.Deploy.Storage": "0.1.0-rc0021", + "Monai.Deploy.WorkflowManager.Common": "1.0.0", "Monai.Deploy.WorkflowManager.Configuration": "1.0.0", "Monai.Deploy.WorkflowManager.Contracts": "1.0.0", "Monai.Deploy.WorkflowManager.Database": "1.0.0", diff --git a/src/TaskManager/API/Monai.Deploy.WorkflowManager.TaskManager.API.csproj b/src/TaskManager/API/Monai.Deploy.WorkflowManager.TaskManager.API.csproj index 8643c3199..04e907ed7 100644 --- a/src/TaskManager/API/Monai.Deploy.WorkflowManager.TaskManager.API.csproj +++ b/src/TaskManager/API/Monai.Deploy.WorkflowManager.TaskManager.API.csproj @@ -22,7 +22,7 @@ SPDX-License-Identifier: Apache License 2.0 - + diff --git a/src/TaskManager/API/packages.lock.json b/src/TaskManager/API/packages.lock.json index ff765a110..3286f66c9 100644 --- a/src/TaskManager/API/packages.lock.json +++ b/src/TaskManager/API/packages.lock.json @@ -4,9 +4,9 @@ "net6.0": { "Monai.Deploy.Messaging": { "type": "Direct", - "requested": "[0.1.0-rc0038, )", - "resolved": "0.1.0-rc0038", - "contentHash": "j6RswItc5GE7b42T1KMMT9oXsOEizr6HKgso4WAV/evGmcwMnzuBdRJHNIfGWeDTTKO7Ld7+BJrSOEno5+udsg==", + "requested": "[0.1.0-rc0040, )", + "resolved": "0.1.0-rc0040", + "contentHash": "q5t6h24M+22JQqeBTD9wOOYYAvb1Vk29nkadcOJ05wT569jnF+MGmEherDn1moX45Ehj4TsCvQl3TD4HJlC1mA==", "dependencies": { "Ardalis.GuardClauses": "4.0.1", "Microsoft.Extensions.Configuration": "6.0.1", diff --git a/src/TaskManager/Monai.Deploy.WorkflowManager.TaskManager.csproj b/src/TaskManager/Monai.Deploy.WorkflowManager.TaskManager.csproj index 845d6f017..368a7daaf 100644 --- a/src/TaskManager/Monai.Deploy.WorkflowManager.TaskManager.csproj +++ b/src/TaskManager/Monai.Deploy.WorkflowManager.TaskManager.csproj @@ -38,7 +38,7 @@ SPDX-License-Identifier: Apache License 2.0 - + diff --git a/src/TaskManager/Plug-ins/Argo/Monai.Deploy.WorkflowManager.TaskManager.Argo.csproj b/src/TaskManager/Plug-ins/Argo/Monai.Deploy.WorkflowManager.TaskManager.Argo.csproj index 33921a4d4..5f8c60fd0 100644 --- a/src/TaskManager/Plug-ins/Argo/Monai.Deploy.WorkflowManager.TaskManager.Argo.csproj +++ b/src/TaskManager/Plug-ins/Argo/Monai.Deploy.WorkflowManager.TaskManager.Argo.csproj @@ -22,7 +22,7 @@ SPDX-License-Identifier: Apache License 2.0 all runtime; build; native; contentfiles; analyzers; buildtransitive - + diff --git a/src/TaskManager/Plug-ins/Argo/packages.lock.json b/src/TaskManager/Plug-ins/Argo/packages.lock.json index ea6b7e999..348b62522 100644 --- a/src/TaskManager/Plug-ins/Argo/packages.lock.json +++ b/src/TaskManager/Plug-ins/Argo/packages.lock.json @@ -34,9 +34,9 @@ }, "Monai.Deploy.Messaging": { "type": "Direct", - "requested": "[0.1.0-rc0038, )", - "resolved": "0.1.0-rc0038", - "contentHash": "j6RswItc5GE7b42T1KMMT9oXsOEizr6HKgso4WAV/evGmcwMnzuBdRJHNIfGWeDTTKO7Ld7+BJrSOEno5+udsg==", + "requested": "[0.1.0-rc0040, )", + "resolved": "0.1.0-rc0040", + "contentHash": "q5t6h24M+22JQqeBTD9wOOYYAvb1Vk29nkadcOJ05wT569jnF+MGmEherDn1moX45Ehj4TsCvQl3TD4HJlC1mA==", "dependencies": { "Ardalis.GuardClauses": "4.0.1", "Microsoft.Extensions.Configuration": "6.0.1", @@ -601,7 +601,7 @@ "dependencies": { "AWSSDK.SecurityToken": "3.7.1.141", "Microsoft.Extensions.Configuration": "6.0.1", - "Monai.Deploy.Messaging": "0.1.0-rc0038", + "Monai.Deploy.Messaging": "0.1.0-rc0040", "MongoDB.Bson": "2.15.0", "Newtonsoft.Json": "13.0.1" } @@ -612,7 +612,7 @@ "Ardalis.GuardClauses": "4.0.1", "Microsoft.Extensions.Logging.Abstractions": "6.0.1", "Microsoft.Extensions.Options": "6.0.0", - "Monai.Deploy.Messaging": "0.1.0-rc0038", + "Monai.Deploy.Messaging": "0.1.0-rc0040", "Monai.Deploy.WorkflowManager.Contracts": "1.0.0", "Monai.Deploy.WorkflowManager.Logging": "1.0.0", "MongoDB.Bson": "2.15.0", @@ -628,7 +628,7 @@ "monai.deploy.workflowmanager.taskmanager.api": { "type": "Project", "dependencies": { - "Monai.Deploy.Messaging": "0.1.0-rc0038", + "Monai.Deploy.Messaging": "0.1.0-rc0040", "Monai.Deploy.Storage": "0.1.0-rc0021" } } diff --git a/src/TaskManager/packages.lock.json b/src/TaskManager/packages.lock.json index 9f1d21fc8..675bf516c 100644 --- a/src/TaskManager/packages.lock.json +++ b/src/TaskManager/packages.lock.json @@ -60,9 +60,9 @@ }, "Monai.Deploy.Messaging": { "type": "Direct", - "requested": "[0.1.0-rc0038, )", - "resolved": "0.1.0-rc0038", - "contentHash": "j6RswItc5GE7b42T1KMMT9oXsOEizr6HKgso4WAV/evGmcwMnzuBdRJHNIfGWeDTTKO7Ld7+BJrSOEno5+udsg==", + "requested": "[0.1.0-rc0040, )", + "resolved": "0.1.0-rc0040", + "contentHash": "q5t6h24M+22JQqeBTD9wOOYYAvb1Vk29nkadcOJ05wT569jnF+MGmEherDn1moX45Ehj4TsCvQl3TD4HJlC1mA==", "dependencies": { "Ardalis.GuardClauses": "4.0.1", "Microsoft.Extensions.Configuration": "6.0.1", @@ -562,7 +562,7 @@ "dependencies": { "Microsoft.Extensions.Configuration": "6.0.1", "Microsoft.Extensions.Options": "6.0.0", - "Monai.Deploy.Messaging": "0.1.0-rc0038", + "Monai.Deploy.Messaging": "0.1.0-rc0040", "Monai.Deploy.Storage": "0.1.0-rc0021", "Newtonsoft.Json": "13.0.1", "System.IO.Abstractions": "16.1.25" @@ -573,7 +573,7 @@ "dependencies": { "AWSSDK.SecurityToken": "3.7.1.141", "Microsoft.Extensions.Configuration": "6.0.1", - "Monai.Deploy.Messaging": "0.1.0-rc0038", + "Monai.Deploy.Messaging": "0.1.0-rc0040", "MongoDB.Bson": "2.15.0", "Newtonsoft.Json": "13.0.1" } @@ -584,7 +584,7 @@ "Ardalis.GuardClauses": "4.0.1", "Microsoft.Extensions.Logging.Abstractions": "6.0.1", "Microsoft.Extensions.Options": "6.0.0", - "Monai.Deploy.Messaging": "0.1.0-rc0038", + "Monai.Deploy.Messaging": "0.1.0-rc0040", "Monai.Deploy.WorkflowManager.Contracts": "1.0.0", "Monai.Deploy.WorkflowManager.Logging": "1.0.0", "MongoDB.Bson": "2.15.0", @@ -600,7 +600,7 @@ "monai.deploy.workflowmanager.taskmanager.api": { "type": "Project", "dependencies": { - "Monai.Deploy.Messaging": "0.1.0-rc0038", + "Monai.Deploy.Messaging": "0.1.0-rc0040", "Monai.Deploy.Storage": "0.1.0-rc0021" } } diff --git a/src/WorkflowExecuter/Monai.Deploy.WorkloadManager.WorkfowExecuter.csproj b/src/WorkflowExecuter/Monai.Deploy.WorkloadManager.WorkfowExecuter.csproj index e4220bce1..365a3c77d 100644 --- a/src/WorkflowExecuter/Monai.Deploy.WorkloadManager.WorkfowExecuter.csproj +++ b/src/WorkflowExecuter/Monai.Deploy.WorkloadManager.WorkfowExecuter.csproj @@ -13,13 +13,14 @@ - + + diff --git a/src/WorkflowExecuter/Services/WorkflowExecuterService.cs b/src/WorkflowExecuter/Services/WorkflowExecuterService.cs index 4159513bc..ed9da7182 100644 --- a/src/WorkflowExecuter/Services/WorkflowExecuterService.cs +++ b/src/WorkflowExecuter/Services/WorkflowExecuterService.cs @@ -9,6 +9,7 @@ using Monai.Deploy.Messaging.Messages; using Monai.Deploy.Storage; using Monai.Deploy.Storage.Configuration; +using Monai.Deploy.WorkflowManager.Common.Extensions; using Monai.Deploy.WorkflowManager.Configuration; using Monai.Deploy.WorkflowManager.Contracts.Models; using Monai.Deploy.WorkflowManager.Database.Interfaces; @@ -166,8 +167,9 @@ public async Task ProcessTaskUpdate(TaskUpdateEvent message) return false; } + var artifactDict = message.Outputs.ToArtifactDictionary(); - var validOutputArtifacts = _storageService.VerifyObjectsExist(workflowInstance.BucketId, message.OutputArtifacts); + var validOutputArtifacts = _storageService.VerifyObjectsExist(workflowInstance.BucketId, artifactDict); workflowInstance.Tasks?.ForEach(t => { diff --git a/src/WorkflowExecuter/packages.lock.json b/src/WorkflowExecuter/packages.lock.json index 48ddd163a..f9aad89f4 100644 --- a/src/WorkflowExecuter/packages.lock.json +++ b/src/WorkflowExecuter/packages.lock.json @@ -22,9 +22,9 @@ }, "Monai.Deploy.Messaging": { "type": "Direct", - "requested": "[0.1.0-rc0038, )", - "resolved": "0.1.0-rc0038", - "contentHash": "j6RswItc5GE7b42T1KMMT9oXsOEizr6HKgso4WAV/evGmcwMnzuBdRJHNIfGWeDTTKO7Ld7+BJrSOEno5+udsg==", + "requested": "[0.1.0-rc0040, )", + "resolved": "0.1.0-rc0040", + "contentHash": "q5t6h24M+22JQqeBTD9wOOYYAvb1Vk29nkadcOJ05wT569jnF+MGmEherDn1moX45Ehj4TsCvQl3TD4HJlC1mA==", "dependencies": { "Ardalis.GuardClauses": "4.0.1", "Microsoft.Extensions.Configuration": "6.0.1", @@ -327,12 +327,21 @@ "resolved": "4.7.1", "contentHash": "6akRtHK/wab3246t4p5v3HQrtQk8LboOt5T4dtpNgsp3zvDeM4/Gx8V12t0h+c/W9/enUrilk8n6EQqdQorZAA==" }, + "monai.deploy.workflowmanager.common": { + "type": "Project", + "dependencies": { + "Ardalis.GuardClauses": "4.0.1", + "Microsoft.Extensions.DependencyInjection.Abstractions": "6.0.0", + "Monai.Deploy.WorkflowManager.Contracts": "1.0.0", + "Monai.Deploy.WorkflowManager.Database": "1.0.0" + } + }, "monai.deploy.workflowmanager.configuration": { "type": "Project", "dependencies": { "Microsoft.Extensions.Configuration": "6.0.1", "Microsoft.Extensions.Options": "6.0.0", - "Monai.Deploy.Messaging": "0.1.0-rc0038", + "Monai.Deploy.Messaging": "0.1.0-rc0040", "Monai.Deploy.Storage": "0.1.0-rc0021", "Newtonsoft.Json": "13.0.1", "System.IO.Abstractions": "16.1.25" @@ -343,7 +352,7 @@ "dependencies": { "AWSSDK.SecurityToken": "3.7.1.141", "Microsoft.Extensions.Configuration": "6.0.1", - "Monai.Deploy.Messaging": "0.1.0-rc0038", + "Monai.Deploy.Messaging": "0.1.0-rc0040", "MongoDB.Bson": "2.15.0", "Newtonsoft.Json": "13.0.1" } @@ -354,7 +363,7 @@ "Ardalis.GuardClauses": "4.0.1", "Microsoft.Extensions.Logging.Abstractions": "6.0.1", "Microsoft.Extensions.Options": "6.0.0", - "Monai.Deploy.Messaging": "0.1.0-rc0038", + "Monai.Deploy.Messaging": "0.1.0-rc0040", "Monai.Deploy.WorkflowManager.Contracts": "1.0.0", "Monai.Deploy.WorkflowManager.Logging": "1.0.0", "MongoDB.Bson": "2.15.0", diff --git a/src/WorkflowManager/Monai.Deploy.WorkflowManager.csproj b/src/WorkflowManager/Monai.Deploy.WorkflowManager.csproj index 95d178669..c95162514 100644 --- a/src/WorkflowManager/Monai.Deploy.WorkflowManager.csproj +++ b/src/WorkflowManager/Monai.Deploy.WorkflowManager.csproj @@ -34,7 +34,7 @@ SPDX-License-Identifier: Apache License 2.0 - + all diff --git a/src/WorkflowManager/packages.lock.json b/src/WorkflowManager/packages.lock.json index 586d6cd0d..8add31a5a 100644 --- a/src/WorkflowManager/packages.lock.json +++ b/src/WorkflowManager/packages.lock.json @@ -138,9 +138,9 @@ }, "Monai.Deploy.Messaging": { "type": "Direct", - "requested": "[0.1.0-rc0038, )", - "resolved": "0.1.0-rc0038", - "contentHash": "j6RswItc5GE7b42T1KMMT9oXsOEizr6HKgso4WAV/evGmcwMnzuBdRJHNIfGWeDTTKO7Ld7+BJrSOEno5+udsg==", + "requested": "[0.1.0-rc0040, )", + "resolved": "0.1.0-rc0040", + "contentHash": "q5t6h24M+22JQqeBTD9wOOYYAvb1Vk29nkadcOJ05wT569jnF+MGmEherDn1moX45Ehj4TsCvQl3TD4HJlC1mA==", "dependencies": { "Ardalis.GuardClauses": "4.0.1", "Microsoft.Extensions.Configuration": "6.0.1", @@ -721,7 +721,7 @@ "dependencies": { "Microsoft.Extensions.Configuration": "6.0.1", "Microsoft.Extensions.Options": "6.0.0", - "Monai.Deploy.Messaging": "0.1.0-rc0038", + "Monai.Deploy.Messaging": "0.1.0-rc0040", "Monai.Deploy.Storage": "0.1.0-rc0021", "Newtonsoft.Json": "13.0.1", "System.IO.Abstractions": "16.1.25" @@ -732,7 +732,7 @@ "dependencies": { "AWSSDK.SecurityToken": "3.7.1.141", "Microsoft.Extensions.Configuration": "6.0.1", - "Monai.Deploy.Messaging": "0.1.0-rc0038", + "Monai.Deploy.Messaging": "0.1.0-rc0040", "MongoDB.Bson": "2.15.0", "Newtonsoft.Json": "13.0.1" } @@ -743,7 +743,7 @@ "Ardalis.GuardClauses": "4.0.1", "Microsoft.Extensions.Logging.Abstractions": "6.0.1", "Microsoft.Extensions.Options": "6.0.0", - "Monai.Deploy.Messaging": "0.1.0-rc0038", + "Monai.Deploy.Messaging": "0.1.0-rc0040", "Monai.Deploy.WorkflowManager.Contracts": "1.0.0", "Monai.Deploy.WorkflowManager.Logging": "1.0.0", "MongoDB.Bson": "2.15.0", @@ -761,7 +761,7 @@ "dependencies": { "Ardalis.GuardClauses": "4.0.1", "Microsoft.Extensions.Hosting.Abstractions": "6.0.0", - "Monai.Deploy.Messaging": "0.1.0-rc0038", + "Monai.Deploy.Messaging": "0.1.0-rc0040", "Monai.Deploy.Storage": "0.1.0-rc0021", "Monai.Deploy.WorkflowManager.Common": "1.0.0", "Monai.Deploy.WorkflowManager.Configuration": "1.0.0", @@ -775,8 +775,9 @@ "dependencies": { "AWSSDK.SecurityToken": "3.7.1.141", "Ardalis.GuardClauses": "4.0.1", - "Monai.Deploy.Messaging": "0.1.0-rc0038", + "Monai.Deploy.Messaging": "0.1.0-rc0040", "Monai.Deploy.Storage": "0.1.0-rc0021", + "Monai.Deploy.WorkflowManager.Common": "1.0.0", "Monai.Deploy.WorkflowManager.Configuration": "1.0.0", "Monai.Deploy.WorkflowManager.Contracts": "1.0.0", "Monai.Deploy.WorkflowManager.Database": "1.0.0", diff --git a/tests/UnitTests/Configuration.Tests/Monai.Deploy.WorkflowManager.Configuration.Tests.csproj b/tests/UnitTests/Configuration.Tests/Monai.Deploy.WorkflowManager.Configuration.Tests.csproj index 576543024..eed9bac31 100644 --- a/tests/UnitTests/Configuration.Tests/Monai.Deploy.WorkflowManager.Configuration.Tests.csproj +++ b/tests/UnitTests/Configuration.Tests/Monai.Deploy.WorkflowManager.Configuration.Tests.csproj @@ -18,7 +18,7 @@ SPDX-License-Identifier: Apache License 2.0 runtime; build; native; contentfiles; analyzers; buildtransitive - + diff --git a/tests/UnitTests/PayloadListener.Tests/Monai.Deploy.WorkflowManager.PayloadListener.Tests.csproj b/tests/UnitTests/PayloadListener.Tests/Monai.Deploy.WorkflowManager.PayloadListener.Tests.csproj index 8ef7dc081..4aedd4ba3 100644 --- a/tests/UnitTests/PayloadListener.Tests/Monai.Deploy.WorkflowManager.PayloadListener.Tests.csproj +++ b/tests/UnitTests/PayloadListener.Tests/Monai.Deploy.WorkflowManager.PayloadListener.Tests.csproj @@ -18,7 +18,7 @@ SPDX-License-Identifier: Apache License 2.0 - + diff --git a/tests/UnitTests/WorkflowExecuter.Tests/Services/WorkflowExecuterServiceTests.cs b/tests/UnitTests/WorkflowExecuter.Tests/Services/WorkflowExecuterServiceTests.cs index 77afd3f98..b70a86abd 100644 --- a/tests/UnitTests/WorkflowExecuter.Tests/Services/WorkflowExecuterServiceTests.cs +++ b/tests/UnitTests/WorkflowExecuter.Tests/Services/WorkflowExecuterServiceTests.cs @@ -13,6 +13,7 @@ using Monai.Deploy.Messaging.Messages; using Monai.Deploy.Storage; using Monai.Deploy.Storage.Configuration; +using Monai.Deploy.WorkflowManager.Common.Extensions; using Monai.Deploy.WorkflowManager.Configuration; using Monai.Deploy.WorkflowManager.Contracts.Models; using Monai.Deploy.WorkflowManager.Database.Interfaces; @@ -756,9 +757,13 @@ public async Task ProcessTaskUpdate_ValidTaskUpdateEventWithOutputArtifacts_Retu Message = "This is a message", Metadata = new Dictionary(), CorrelationId = Guid.NewGuid().ToString(), - OutputArtifacts = new Dictionary + Outputs = new List { - { "artifact.txt", "path/to/artifact" } + new Messaging.Common.Storage + { + Name = "artifactname", + RelativeRootPath = "path/to/artifact" + } } }; @@ -832,11 +837,13 @@ public async Task ProcessTaskUpdate_ValidTaskUpdateEventWithOutputArtifacts_Retu } }; + var artifactDict = updateEvent.Outputs.ToArtifactDictionary(); + _workflowInstanceRepository.Setup(w => w.UpdateTaskStatusAsync(It.IsAny(), It.IsAny(), It.IsAny())).ReturnsAsync(true); _workflowInstanceRepository.Setup(w => w.GetByWorkflowInstanceIdAsync(workflowInstance.Id)).ReturnsAsync(workflowInstance); _workflowInstanceRepository.Setup(w => w.UpdateTasksAsync(workflowInstance.Id, It.IsAny>())).ReturnsAsync(true); _workflowRepository.Setup(w => w.GetByWorkflowIdAsync(workflowInstance.WorkflowId)).ReturnsAsync(workflow); - _storageService.Setup(w => w.VerifyObjectsExist(workflowInstance.BucketId, updateEvent.OutputArtifacts)).Returns(updateEvent.OutputArtifacts); + _storageService.Setup(w => w.VerifyObjectsExist(workflowInstance.BucketId, artifactDict)).Returns(artifactDict); var response = await WorkflowExecuterService.ProcessTaskUpdate(updateEvent); diff --git a/tests/UnitTests/WorkflowManager.Common.Tests/Monai.Deploy.WorkflowManager.Common.Tests.csproj b/tests/UnitTests/WorkflowManager.Common.Tests/Monai.Deploy.WorkflowManager.Common.Tests.csproj new file mode 100644 index 000000000..dd325396f --- /dev/null +++ b/tests/UnitTests/WorkflowManager.Common.Tests/Monai.Deploy.WorkflowManager.Common.Tests.csproj @@ -0,0 +1,28 @@ + + + + net6.0 + enable + enable + + false + + + + + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + + + + + + diff --git a/tests/UnitTests/WorkflowManager.Common.Tests/StorageListExtensionsTests.cs b/tests/UnitTests/WorkflowManager.Common.Tests/StorageListExtensionsTests.cs new file mode 100644 index 000000000..f174a5781 --- /dev/null +++ b/tests/UnitTests/WorkflowManager.Common.Tests/StorageListExtensionsTests.cs @@ -0,0 +1,66 @@ +using Monai.Deploy.WorkflowManager.Common.Extensions; +using Xunit; + +namespace Monai.Deploy.WorkflowManager.Common.Tests +{ + public class StorageListExtensionsTests + { + [Fact] + public void ToArtifactDictionary_ValidStorageList_GeneratesDictionary() + { + var storageList = new List + { + new Messaging.Common.Storage + { + Name = "test", + RelativeRootPath = "payloadid/dcm" + }, + new Messaging.Common.Storage + { + Name = "test2", + RelativeRootPath = "payloadid/dcm" + } + }; + + var expected = new Dictionary + { + { "test", "payloadid/dcm" }, + { "test2", "payloadid/dcm" } + }; + + var artifactDict = storageList.ToArtifactDictionary(); + + Assert.Equal(expected, artifactDict); + } + + [Fact] + public void ToArtifactDictionary_MissingFields_ReturnsEmpty() + { + var storageList = new List + { + new Messaging.Common.Storage + { + Endpoint = "test" + } + }; + + var expected = new Dictionary(); + + var artifactDict = storageList.ToArtifactDictionary(); + + Assert.Equal(expected, artifactDict); + } + + [Fact] + public void ToArtifactDictionary_EmptyList_ReturnsEmpty() + { + var storageList = new List(); + + var expected = new Dictionary(); + + var artifactDict = storageList.ToArtifactDictionary(); + + Assert.Equal(expected, artifactDict); + } + } +} diff --git a/tests/UnitTests/WorkflowManager.Tests/packages.lock.json b/tests/UnitTests/WorkflowManager.Tests/packages.lock.json index 391842334..8c7ccb8c9 100644 --- a/tests/UnitTests/WorkflowManager.Tests/packages.lock.json +++ b/tests/UnitTests/WorkflowManager.Tests/packages.lock.json @@ -592,8 +592,8 @@ }, "Monai.Deploy.Messaging": { "type": "Transitive", - "resolved": "0.1.0-rc0038", - "contentHash": "j6RswItc5GE7b42T1KMMT9oXsOEizr6HKgso4WAV/evGmcwMnzuBdRJHNIfGWeDTTKO7Ld7+BJrSOEno5+udsg==", + "resolved": "0.1.0-rc0040", + "contentHash": "q5t6h24M+22JQqeBTD9wOOYYAvb1Vk29nkadcOJ05wT569jnF+MGmEherDn1moX45Ehj4TsCvQl3TD4HJlC1mA==", "dependencies": { "Ardalis.GuardClauses": "4.0.1", "Microsoft.Extensions.Configuration": "6.0.1", @@ -1376,7 +1376,7 @@ "Microsoft.Extensions.Logging": "6.0.0", "Microsoft.Extensions.Logging.Console": "6.0.0", "Microsoft.Extensions.Options": "6.0.0", - "Monai.Deploy.Messaging": "0.1.0-rc0038", + "Monai.Deploy.Messaging": "0.1.0-rc0040", "Monai.Deploy.WorkflowManager.Common": "1.0.0", "Monai.Deploy.WorkflowManager.Configuration": "1.0.0", "Monai.Deploy.WorkflowManager.Contracts": "1.0.0", @@ -1401,7 +1401,7 @@ "dependencies": { "Microsoft.Extensions.Configuration": "6.0.1", "Microsoft.Extensions.Options": "6.0.0", - "Monai.Deploy.Messaging": "0.1.0-rc0038", + "Monai.Deploy.Messaging": "0.1.0-rc0040", "Monai.Deploy.Storage": "0.1.0-rc0021", "Newtonsoft.Json": "13.0.1", "System.IO.Abstractions": "16.1.25" @@ -1412,7 +1412,7 @@ "dependencies": { "AWSSDK.SecurityToken": "3.7.1.141", "Microsoft.Extensions.Configuration": "6.0.1", - "Monai.Deploy.Messaging": "0.1.0-rc0038", + "Monai.Deploy.Messaging": "0.1.0-rc0040", "MongoDB.Bson": "2.15.0", "Newtonsoft.Json": "13.0.1" } @@ -1423,7 +1423,7 @@ "Ardalis.GuardClauses": "4.0.1", "Microsoft.Extensions.Logging.Abstractions": "6.0.1", "Microsoft.Extensions.Options": "6.0.0", - "Monai.Deploy.Messaging": "0.1.0-rc0038", + "Monai.Deploy.Messaging": "0.1.0-rc0040", "Monai.Deploy.WorkflowManager.Contracts": "1.0.0", "Monai.Deploy.WorkflowManager.Logging": "1.0.0", "MongoDB.Bson": "2.15.0", @@ -1441,7 +1441,7 @@ "dependencies": { "Ardalis.GuardClauses": "4.0.1", "Microsoft.Extensions.Hosting.Abstractions": "6.0.0", - "Monai.Deploy.Messaging": "0.1.0-rc0038", + "Monai.Deploy.Messaging": "0.1.0-rc0040", "Monai.Deploy.Storage": "0.1.0-rc0021", "Monai.Deploy.WorkflowManager.Common": "1.0.0", "Monai.Deploy.WorkflowManager.Configuration": "1.0.0", @@ -1455,8 +1455,9 @@ "dependencies": { "AWSSDK.SecurityToken": "3.7.1.141", "Ardalis.GuardClauses": "4.0.1", - "Monai.Deploy.Messaging": "0.1.0-rc0038", + "Monai.Deploy.Messaging": "0.1.0-rc0040", "Monai.Deploy.Storage": "0.1.0-rc0021", + "Monai.Deploy.WorkflowManager.Common": "1.0.0", "Monai.Deploy.WorkflowManager.Configuration": "1.0.0", "Monai.Deploy.WorkflowManager.Contracts": "1.0.0", "Monai.Deploy.WorkflowManager.Database": "1.0.0", From 590204fdecce06c87153df5222d5c41c5ceb32a6 Mon Sep 17 00:00:00 2001 From: Jack Schofield Date: Mon, 6 Jun 2022 13:12:53 +0100 Subject: [PATCH 8/9] Address comments Signed-off-by: Jack Schofield --- src/Common/Extensions/StorageListExtensions.cs | 5 ++++- .../Services/WorkflowExecuterService.cs | 10 +++++----- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/Common/Extensions/StorageListExtensions.cs b/src/Common/Extensions/StorageListExtensions.cs index 6ebf3c333..442cad9b5 100644 --- a/src/Common/Extensions/StorageListExtensions.cs +++ b/src/Common/Extensions/StorageListExtensions.cs @@ -1,4 +1,7 @@ -using Ardalis.GuardClauses; +// SPDX-FileCopyrightText: © 2021-2022 MONAI Consortium +// SPDX-License-Identifier: Apache License 2.0 + +using Ardalis.GuardClauses; namespace Monai.Deploy.WorkflowManager.Common.Extensions { diff --git a/src/WorkflowExecuter/Services/WorkflowExecuterService.cs b/src/WorkflowExecuter/Services/WorkflowExecuterService.cs index ed9da7182..bb8761f06 100644 --- a/src/WorkflowExecuter/Services/WorkflowExecuterService.cs +++ b/src/WorkflowExecuter/Services/WorkflowExecuterService.cs @@ -81,7 +81,7 @@ await _workflowRepository.GetWorkflowsByAeTitleAsync(message.CalledAeTitle) as L var workflowInstances = new List(); - workflows.ForEach(async (workflow) => workflowInstances.Add(await CreateWorkflowInstance(message, workflow))); + workflows.ForEach(async (workflow) => workflowInstances.Add(await CreateWorkflowInstanceAsync(message, workflow))); var existingInstances = await _workflowInstanceRepository.GetByWorkflowsIdsAsync(workflowInstances.Select(w => w.WorkflowId).ToList()); workflowInstances.RemoveAll(i => existingInstances.ToList().Exists(e => e.WorkflowId == i.WorkflowId && e.PayloadId == i.PayloadId)); @@ -275,7 +275,7 @@ private async Task> HandleTaskDestinations(WorkflowInstance continue; } - newTaskExecutions.Add(await CreateTaskExecution(newTask, workflowInstance.Id, workflowInstance.BucketId, workflowInstance.PayloadId)); + newTaskExecutions.Add(await CreateTaskExecutionAsync(newTask, workflowInstance.Id, workflowInstance.BucketId, workflowInstance.PayloadId)); } return newTaskExecutions; @@ -291,7 +291,7 @@ private async Task DispatchTask(WorkflowInstance workflowInstance, TaskExe return await _workflowInstanceRepository.UpdateTaskStatusAsync(workflowInstance.Id, taskExec.TaskId, TaskExecutionStatus.Dispatched); } - private async Task CreateWorkflowInstance(WorkflowRequestEvent message, WorkflowRevision workflow) + private async Task CreateWorkflowInstanceAsync(WorkflowRequestEvent message, WorkflowRevision workflow) { Guard.Against.Null(message, nameof(message)); Guard.Against.Null(workflow, nameof(workflow)); @@ -327,7 +327,7 @@ private async Task CreateWorkflowInstance(WorkflowRequestEvent try { - tasks.Add(await CreateTaskExecution(firstTask, workflowInstance.Id, message.Bucket, message.PayloadId.ToString())); + tasks.Add(await CreateTaskExecutionAsync(firstTask, workflowInstance.Id, message.Bucket, message.PayloadId.ToString())); } catch (FileNotFoundException e) { @@ -340,7 +340,7 @@ private async Task CreateWorkflowInstance(WorkflowRequestEvent return workflowInstance; } - private async Task CreateTaskExecution(TaskObject task, string workflowInstanceId, string bucketName, string payloadId) + private async Task CreateTaskExecutionAsync(TaskObject task, string workflowInstanceId, string bucketName, string payloadId) { Guard.Against.Null(task, nameof(task)); Guard.Against.NullOrWhiteSpace(task.Type, nameof(task.Type)); From 58993d2817f8141e53290592ae42ad60f5d8512c Mon Sep 17 00:00:00 2001 From: Jack Schofield Date: Wed, 8 Jun 2022 09:55:25 +0100 Subject: [PATCH 9/9] address comments and code smells Signed-off-by: Jack Schofield --- src/Database/WorkflowInstanceRepository.cs | 12 ++++++++++-- .../Services/WorkflowExecuterService.cs | 9 +++++++-- .../Common/ArtifactMapperTests.cs | 2 -- 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/src/Database/WorkflowInstanceRepository.cs b/src/Database/WorkflowInstanceRepository.cs index 052989e21..0e11ce1c7 100644 --- a/src/Database/WorkflowInstanceRepository.cs +++ b/src/Database/WorkflowInstanceRepository.cs @@ -55,6 +55,7 @@ public async Task> GetByWorkflowsIdsAsync(List w catch (Exception e) { _logger.DbCallFailed(nameof(GetByWorkflowsIdsAsync), e); + return new List(); } } @@ -66,11 +67,13 @@ public async Task CreateAsync(IList workflowInstances) try { await _workflowInstanceCollection.InsertManyAsync(workflowInstances); + return true; } catch (Exception e) { _logger.DbCallFailed(nameof(CreateAsync), e); + return false; } } @@ -83,14 +86,16 @@ public async Task UpdateTaskStatusAsync(string workflowInstanceId, string try { - var result = await _workflowInstanceCollection.FindOneAndUpdateAsync( + await _workflowInstanceCollection.FindOneAndUpdateAsync( i => i.Id == workflowInstanceId && i.Tasks.Any(t => t.TaskId == taskId), Builders.Update.Set(w => w.Tasks[-1].Status, status)); + return true; } catch (Exception e) { _logger.DbCallFailed(nameof(UpdateTaskStatusAsync), e); + return false; } } @@ -103,14 +108,16 @@ public async Task UpdateTaskOutputArtifactsAsync(string workflowInstanceId try { - var result = await _workflowInstanceCollection.FindOneAndUpdateAsync( + await _workflowInstanceCollection.FindOneAndUpdateAsync( i => i.Id == workflowInstanceId && i.Tasks.Any(t => t.TaskId == taskId), Builders.Update.Set(w => w.Tasks[-1].OutputArtifacts, outputArtifacts)); + return true; } catch (Exception e) { _logger.DbCallFailed(nameof(UpdateTaskOutputArtifactsAsync), e); + return false; } } @@ -150,6 +157,7 @@ public async Task GetTaskByIdAsync(string workflowInstanceId, str catch (Exception e) { _logger.DbCallFailed(nameof(GetTaskByIdAsync), e); + return null; } } diff --git a/src/WorkflowExecuter/Services/WorkflowExecuterService.cs b/src/WorkflowExecuter/Services/WorkflowExecuterService.cs index bb8761f06..6b4a6b628 100644 --- a/src/WorkflowExecuter/Services/WorkflowExecuterService.cs +++ b/src/WorkflowExecuter/Services/WorkflowExecuterService.cs @@ -107,6 +107,11 @@ await _workflowRepository.GetWorkflowsByAeTitleAsync(message.CalledAeTitle) as L var task = workflowInstance.Tasks.FirstOrDefault(); + if (task is null) + { + continue; + } + if (task.Status != TaskExecutionStatus.Created) { _logger.TaskPreviouslyDispatched(workflowInstance.PayloadId, task.TaskId); @@ -191,7 +196,7 @@ public async Task ProcessTaskUpdate(TaskUpdateEvent message) return await _workflowInstanceRepository.UpdateTaskStatusAsync(message.WorkflowInstanceId, message.TaskId, message.Status); } - workflowInstance.Tasks.AddRange(newTaskExecutions); + workflowInstance.Tasks?.AddRange(newTaskExecutions); if (!await _workflowInstanceRepository.UpdateTasksAsync(message.WorkflowInstanceId, workflowInstance.Tasks)) { @@ -329,7 +334,7 @@ private async Task CreateWorkflowInstanceAsync(WorkflowRequest { tasks.Add(await CreateTaskExecutionAsync(firstTask, workflowInstance.Id, message.Bucket, message.PayloadId.ToString())); } - catch (FileNotFoundException e) + catch (FileNotFoundException) { workflowInstance.Status = Status.Failed; } diff --git a/tests/UnitTests/WorkflowExecuter.Tests/Common/ArtifactMapperTests.cs b/tests/UnitTests/WorkflowExecuter.Tests/Common/ArtifactMapperTests.cs index daac02dee..1cd643f7c 100644 --- a/tests/UnitTests/WorkflowExecuter.Tests/Common/ArtifactMapperTests.cs +++ b/tests/UnitTests/WorkflowExecuter.Tests/Common/ArtifactMapperTests.cs @@ -21,8 +21,6 @@ public class ArtifactMapperTests { private IArtifactMapper ArtifactMapper { get; set; } - private IStorageService StorageService { get; set; } - private readonly Mock _workflowInstanceRepository; private readonly Mock _storageService;