diff --git a/OnnxStack.Core/Extensions/TensorExtension.cs b/OnnxStack.Core/Extensions/TensorExtension.cs index ae99cb53..f701553e 100644 --- a/OnnxStack.Core/Extensions/TensorExtension.cs +++ b/OnnxStack.Core/Extensions/TensorExtension.cs @@ -5,6 +5,33 @@ namespace OnnxStack.Core { public static class TensorExtension { + /// + /// Repeats the specified Tensor along the 0 axis. + /// + /// The tensor1. + /// The count. + /// The axis. + /// + /// Only axis 0 is supported + public static DenseTensor Repeat(this DenseTensor tensor1, int count, int axis = 0) + { + if (axis != 0) + throw new NotImplementedException("Only axis 0 is supported"); + + var dimensions = tensor1.Dimensions.ToArray(); + dimensions[0] *= count; + + var length = (int)tensor1.Length; + var totalLength = length * count; + var buffer = new float[totalLength].AsMemory(); + for (int i = 0; i < count; i++) + { + tensor1.Buffer.CopyTo(buffer[(i * length)..]); + } + return new DenseTensor(buffer, dimensions); + } + + /// /// Concatenates the specified tensors along the specified axis. /// diff --git a/OnnxStack.FeatureExtractor/Common/FeatureExtractorModelSet.cs b/OnnxStack.FeatureExtractor/Common/FeatureExtractorModelSet.cs new file mode 100644 index 00000000..481f3e1f --- /dev/null +++ b/OnnxStack.FeatureExtractor/Common/FeatureExtractorModelSet.cs @@ -0,0 +1,18 @@ +using Microsoft.ML.OnnxRuntime; +using OnnxStack.Core.Config; +using System.Collections.Generic; + +namespace OnnxStack.FeatureExtractor.Common +{ + public record FeatureExtractorModelSet : IOnnxModelSetConfig + { + public string Name { get; set; } + public bool IsEnabled { get; set; } + public int DeviceId { get; set; } + public int InterOpNumThreads { get; set; } + public int IntraOpNumThreads { get; set; } + public ExecutionMode ExecutionMode { get; set; } + public ExecutionProvider ExecutionProvider { get; set; } + public List ModelConfigurations { get; set; } + } +} diff --git a/OnnxStack.FeatureExtractor/OnnxStack.FeatureExtractor.csproj b/OnnxStack.FeatureExtractor/OnnxStack.FeatureExtractor.csproj new file mode 100644 index 00000000..fdf79e8c --- /dev/null +++ b/OnnxStack.FeatureExtractor/OnnxStack.FeatureExtractor.csproj @@ -0,0 +1,49 @@ + + + + 0.17.0 + net7.0 + disable + disable + x64 + + OnnxStack.FeatureExtractor + Backyard Industries + + OnnxRuntime Image Feature Extractor Library for .NET Core + + Backyard Industries - 2023 + https://github.com/saddam213/OnnxStack + onnx;onnx-runtime; + sa_ddam213 + README.md + OnnxStack.FeatureExtractor + $(AssemblyName) + Apache-2.0 + True + True + OnnxStack - 128x128.png + Debug;Release + x64 + + + + + \ + True + + + + + + + + + + + \ + True + + + + diff --git a/OnnxStack.FeatureExtractor/README.md b/OnnxStack.FeatureExtractor/README.md new file mode 100644 index 00000000..189e799d --- /dev/null +++ b/OnnxStack.FeatureExtractor/README.md @@ -0,0 +1,13 @@ +# OnnxStack.FeatureExtractor + +## Canny +https://huggingface.co/axodoxian/controlnet_onnx/resolve/main/annotators/canny.onnx + +## Hed +https://huggingface.co/axodoxian/controlnet_onnx/resolve/main/annotators/hed.onnx + +## Depth +https://huggingface.co/axodoxian/controlnet_onnx/resolve/main/annotators/depth.onnx + +## OpenPose (TODO) +https://huggingface.co/axodoxian/controlnet_onnx/resolve/main/annotators/openpose.onnx \ No newline at end of file diff --git a/OnnxStack.FeatureExtractor/Services/FeatureExtractorService.cs b/OnnxStack.FeatureExtractor/Services/FeatureExtractorService.cs new file mode 100644 index 00000000..5cf509e6 --- /dev/null +++ b/OnnxStack.FeatureExtractor/Services/FeatureExtractorService.cs @@ -0,0 +1,162 @@ +using Microsoft.Extensions.Logging; +using Microsoft.ML.OnnxRuntime.Tensors; +using OnnxStack.Core; +using OnnxStack.Core.Config; +using OnnxStack.Core.Image; +using OnnxStack.Core.Model; +using OnnxStack.Core.Services; +using OnnxStack.FeatureExtractor.Common; +using System.Linq; +using System.Threading.Tasks; + +namespace OnnxStack.FeatureExtractor.Services +{ + /// + /// Service for handing images for input and output of the diffusion process + /// + /// + public class FeatureExtractorService : IFeatureExtractorService + { + private readonly ILogger _logger; + private readonly IOnnxModelService _onnxModelService; + + + /// + /// Initializes a new instance of the class. + /// + /// The onnx model service. + public FeatureExtractorService(IOnnxModelService onnxModelService, ILogger logger) + { + _logger = logger; + _onnxModelService = onnxModelService; + } + + + /// + /// Generates the canny image mask. + /// + /// The control net model. + /// The input image. + /// The height. + /// The width. + /// + public async Task CannyImage(FeatureExtractorModelSet controlNetModel, InputImage inputImage, int height, int width) + { + _logger.LogInformation($"[CannyImage] - Generating Canny image..."); + var controlImage = await inputImage.ToDenseTensorAsync(height, width, ImageNormalizeType.ZeroToOne); + var metadata = _onnxModelService.GetModelMetadata(controlNetModel, OnnxModelType.Annotation); + using (var inferenceParameters = new OnnxInferenceParameters(metadata)) + { + inferenceParameters.AddInputTensor(controlImage); + inferenceParameters.AddOutputBuffer(new[] { 1, 1, height, width }); + + var results = await _onnxModelService.RunInferenceAsync(controlNetModel, OnnxModelType.Annotation, inferenceParameters); + using (var result = results.First()) + { + var testImage = result.ToDenseTensor().Repeat(3); + var imageTensor = new DenseTensor(controlImage.Dimensions); + for (int i = 0; i < testImage.Length; i++) + imageTensor.SetValue(i, testImage.GetValue(i)); + + var maskImage = imageTensor.ToImageMask(); + //await maskImage.SaveAsPngAsync("D:\\Canny.png"); + _logger.LogInformation($"[CannyImage] - Canny image generation complete."); + return new InputImage(maskImage); + } + } + } + + + /// + /// Generates the hard edge image mask. + /// + /// The control net model. + /// The input image. + /// The height. + /// The width. + /// + public async Task HedImage(FeatureExtractorModelSet controlNetModel, InputImage inputImage, int height, int width) + { + _logger.LogInformation($"[HedImage] - Generating HardEdge image..."); + var controlImage = await inputImage.ToDenseTensorAsync(height, width, ImageNormalizeType.ZeroToOne); + var metadata = _onnxModelService.GetModelMetadata(controlNetModel, OnnxModelType.Annotation); + using (var inferenceParameters = new OnnxInferenceParameters(metadata)) + { + inferenceParameters.AddInputTensor(controlImage); + inferenceParameters.AddOutputBuffer(new[] { 1, 1, height, width }); + + var results = await _onnxModelService.RunInferenceAsync(controlNetModel, OnnxModelType.Annotation, inferenceParameters); + using (var result = results.First()) + { + var testImage = result.ToDenseTensor().Repeat(3); + var imageTensor = new DenseTensor(controlImage.Dimensions); + for (int i = 0; i < testImage.Length; i++) + imageTensor.SetValue(i, testImage.GetValue(i)); + + var maskImage = imageTensor.ToImageMask(); + //await maskImage.SaveAsPngAsync("D:\\Hed.png"); + _logger.LogInformation($"[HedImage] - HardEdge image generation complete."); + return new InputImage(maskImage); + } + } + } + + + /// + /// Generates the depth image mask. + /// + /// The control net model. + /// The input image. + /// The height. + /// The width. + /// + public async Task DepthImage(FeatureExtractorModelSet controlNetModel, InputImage inputImage, int height, int width) + { + _logger.LogInformation($"[DepthImage] - Generating Depth image..."); + var controlImage = await inputImage.ToDenseTensorAsync(height, width, ImageNormalizeType.ZeroToOne); + var metadata = _onnxModelService.GetModelMetadata(controlNetModel, OnnxModelType.Annotation); + using (var inferenceParameters = new OnnxInferenceParameters(metadata)) + { + inferenceParameters.AddInputTensor(controlImage); + inferenceParameters.AddOutputBuffer(new[] { 1, 1, height, width }); + + var results = await _onnxModelService.RunInferenceAsync(controlNetModel, OnnxModelType.Annotation, inferenceParameters); + using (var result = results.First()) + { + var testImage = result.ToDenseTensor().Repeat(3); + var imageTensor = new DenseTensor(controlImage.Dimensions); + for (int i = 0; i < testImage.Length; i++) + imageTensor.SetValue(i, testImage.GetValue(i)); + + NormalizeDepthTensor(imageTensor); + var maskImage = imageTensor.ToImageMask(); + //await maskImage.SaveAsPngAsync("D:\\Depth.png"); + _logger.LogInformation($"[DepthImage] - Depth image generation complete."); + return new InputImage(maskImage); + } + } + } + + + /// + /// Normalizes the depth tensor. + /// + /// The value. + public static void NormalizeDepthTensor(DenseTensor value) + { + var values = value.Buffer.Span; + float min = float.PositiveInfinity, max = float.NegativeInfinity; + foreach (var val in values) + { + if (min > val) min = val; + if (max < val) max = val; + } + + var range = max - min; + for (var i = 0; i < values.Length; i++) + { + values[i] = (values[i] - min) / range; + } + } + } +} diff --git a/OnnxStack.FeatureExtractor/Services/IFeatureExtractorService.cs b/OnnxStack.FeatureExtractor/Services/IFeatureExtractorService.cs new file mode 100644 index 00000000..04109210 --- /dev/null +++ b/OnnxStack.FeatureExtractor/Services/IFeatureExtractorService.cs @@ -0,0 +1,12 @@ +using OnnxStack.Core.Image; +using System.Threading.Tasks; + +namespace OnnxStack.FeatureExtractor.Common +{ + public interface IFeatureExtractorService + { + Task CannyImage(FeatureExtractorModelSet controlNetModel, InputImage inputImage, int height, int width); + Task HedImage(FeatureExtractorModelSet controlNetModel, InputImage inputImage, int height, int width); + Task DepthImage(FeatureExtractorModelSet controlNetModel, InputImage inputImage, int height, int width); + } +} \ No newline at end of file diff --git a/OnnxStack.StableDiffusion/Helpers/TensorHelper.cs b/OnnxStack.StableDiffusion/Helpers/TensorHelper.cs index b3c88ca1..942dd36e 100644 --- a/OnnxStack.StableDiffusion/Helpers/TensorHelper.cs +++ b/OnnxStack.StableDiffusion/Helpers/TensorHelper.cs @@ -253,31 +253,7 @@ public static DenseTensor Clip(this DenseTensor tensor, float minV - /// - /// Repeats the specified Tensor along the 0 axis. - /// - /// The tensor1. - /// The count. - /// The axis. - /// - /// Only axis 0 is supported - public static DenseTensor Repeat(this DenseTensor tensor1, int count, int axis = 0) - { - if (axis != 0) - throw new NotImplementedException("Only axis 0 is supported"); - - var dimensions = tensor1.Dimensions.ToArray(); - dimensions[0] *= count; - var length = (int)tensor1.Length; - var totalLength = length * count; - var buffer = new float[totalLength].AsMemory(); - for (int i = 0; i < count; i++) - { - tensor1.Buffer.CopyTo(buffer[(i * length)..]); - } - return new DenseTensor(buffer, dimensions); - } /// diff --git a/OnnxStack.sln b/OnnxStack.sln index 336f3039..1cc8f106 100644 --- a/OnnxStack.sln +++ b/OnnxStack.sln @@ -15,6 +15,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "OnnxStack.ImageUpscaler", " EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "OnnxStack.Adapter", "OnnxStack.Adapter\OnnxStack.Adapter.vcxproj", "{5A64FC26-6926-4A45-ACCE-1FE173166990}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "OnnxStack.FeatureExtractor", "OnnxStack.FeatureExtractor\OnnxStack.FeatureExtractor.csproj", "{0E8095A4-83EF-48AD-9BD1-0CC2893050F6}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|x64 = Debug|x64 @@ -30,41 +32,61 @@ Global {02404CEB-207F-4D19-894C-11D51394F1D5}.Debug-Cuda|x64.ActiveCfg = Debug|x64 {02404CEB-207F-4D19-894C-11D51394F1D5}.Debug-Cuda|x64.Build.0 = Debug|x64 {02404CEB-207F-4D19-894C-11D51394F1D5}.Debug-TensorRT|x64.ActiveCfg = Debug|x64 + {02404CEB-207F-4D19-894C-11D51394F1D5}.Debug-TensorRT|x64.Build.0 = Debug|x64 {02404CEB-207F-4D19-894C-11D51394F1D5}.Release|x64.ActiveCfg = Release|x64 + {02404CEB-207F-4D19-894C-11D51394F1D5}.Release|x64.Build.0 = Release|x64 {02404CEB-207F-4D19-894C-11D51394F1D5}.Release-Cuda|x64.ActiveCfg = Release|x64 + {02404CEB-207F-4D19-894C-11D51394F1D5}.Release-Cuda|x64.Build.0 = Release|x64 {02404CEB-207F-4D19-894C-11D51394F1D5}.Release-TensorRT|x64.ActiveCfg = Release|x64 + {02404CEB-207F-4D19-894C-11D51394F1D5}.Release-TensorRT|x64.Build.0 = Release|x64 {EA1F61D0-490B-42EC-96F5-7DCCAB94457A}.Debug|x64.ActiveCfg = Debug|x64 {EA1F61D0-490B-42EC-96F5-7DCCAB94457A}.Debug|x64.Build.0 = Debug|x64 {EA1F61D0-490B-42EC-96F5-7DCCAB94457A}.Debug-Cuda|x64.ActiveCfg = Debug|x64 {EA1F61D0-490B-42EC-96F5-7DCCAB94457A}.Debug-Cuda|x64.Build.0 = Debug|x64 {EA1F61D0-490B-42EC-96F5-7DCCAB94457A}.Debug-TensorRT|x64.ActiveCfg = Debug|x64 + {EA1F61D0-490B-42EC-96F5-7DCCAB94457A}.Debug-TensorRT|x64.Build.0 = Debug|x64 {EA1F61D0-490B-42EC-96F5-7DCCAB94457A}.Release|x64.ActiveCfg = Release|x64 + {EA1F61D0-490B-42EC-96F5-7DCCAB94457A}.Release|x64.Build.0 = Release|x64 {EA1F61D0-490B-42EC-96F5-7DCCAB94457A}.Release-Cuda|x64.ActiveCfg = Release|x64 + {EA1F61D0-490B-42EC-96F5-7DCCAB94457A}.Release-Cuda|x64.Build.0 = Release|x64 {EA1F61D0-490B-42EC-96F5-7DCCAB94457A}.Release-TensorRT|x64.ActiveCfg = Release|x64 + {EA1F61D0-490B-42EC-96F5-7DCCAB94457A}.Release-TensorRT|x64.Build.0 = Release|x64 {46A43C80-A440-4461-B7EB-81FA998FB24B}.Debug|x64.ActiveCfg = Debug|x64 {46A43C80-A440-4461-B7EB-81FA998FB24B}.Debug|x64.Build.0 = Debug|x64 {46A43C80-A440-4461-B7EB-81FA998FB24B}.Debug-Cuda|x64.ActiveCfg = Debug-Cuda|x64 {46A43C80-A440-4461-B7EB-81FA998FB24B}.Debug-Cuda|x64.Build.0 = Debug-Cuda|x64 {46A43C80-A440-4461-B7EB-81FA998FB24B}.Debug-TensorRT|x64.ActiveCfg = Debug-TensorRT|x64 + {46A43C80-A440-4461-B7EB-81FA998FB24B}.Debug-TensorRT|x64.Build.0 = Debug-TensorRT|x64 {46A43C80-A440-4461-B7EB-81FA998FB24B}.Release|x64.ActiveCfg = Release|x64 + {46A43C80-A440-4461-B7EB-81FA998FB24B}.Release|x64.Build.0 = Release|x64 {46A43C80-A440-4461-B7EB-81FA998FB24B}.Release-Cuda|x64.ActiveCfg = Release-Cuda|x64 + {46A43C80-A440-4461-B7EB-81FA998FB24B}.Release-Cuda|x64.Build.0 = Release-Cuda|x64 {46A43C80-A440-4461-B7EB-81FA998FB24B}.Release-TensorRT|x64.ActiveCfg = Release-TensorRT|x64 + {46A43C80-A440-4461-B7EB-81FA998FB24B}.Release-TensorRT|x64.Build.0 = Release-TensorRT|x64 {85BB1855-8C3B-4049-A2DD-1130FA6CD846}.Debug|x64.ActiveCfg = Debug|x64 {85BB1855-8C3B-4049-A2DD-1130FA6CD846}.Debug|x64.Build.0 = Debug|x64 {85BB1855-8C3B-4049-A2DD-1130FA6CD846}.Debug-Cuda|x64.ActiveCfg = Debug-Cuda|x64 {85BB1855-8C3B-4049-A2DD-1130FA6CD846}.Debug-Cuda|x64.Build.0 = Debug-Cuda|x64 {85BB1855-8C3B-4049-A2DD-1130FA6CD846}.Debug-TensorRT|x64.ActiveCfg = Debug-TensorRT|x64 + {85BB1855-8C3B-4049-A2DD-1130FA6CD846}.Debug-TensorRT|x64.Build.0 = Debug-TensorRT|x64 {85BB1855-8C3B-4049-A2DD-1130FA6CD846}.Release|x64.ActiveCfg = Release|x64 + {85BB1855-8C3B-4049-A2DD-1130FA6CD846}.Release|x64.Build.0 = Release|x64 {85BB1855-8C3B-4049-A2DD-1130FA6CD846}.Release-Cuda|x64.ActiveCfg = Release-Cuda|x64 + {85BB1855-8C3B-4049-A2DD-1130FA6CD846}.Release-Cuda|x64.Build.0 = Release-Cuda|x64 {85BB1855-8C3B-4049-A2DD-1130FA6CD846}.Release-TensorRT|x64.ActiveCfg = Release-TensorRT|x64 + {85BB1855-8C3B-4049-A2DD-1130FA6CD846}.Release-TensorRT|x64.Build.0 = Release-TensorRT|x64 {A33D08BF-7881-4910-8439-5AE46646C1DD}.Debug|x64.ActiveCfg = Debug|x64 {A33D08BF-7881-4910-8439-5AE46646C1DD}.Debug|x64.Build.0 = Debug|x64 {A33D08BF-7881-4910-8439-5AE46646C1DD}.Debug-Cuda|x64.ActiveCfg = Debug|x64 {A33D08BF-7881-4910-8439-5AE46646C1DD}.Debug-Cuda|x64.Build.0 = Debug|x64 {A33D08BF-7881-4910-8439-5AE46646C1DD}.Debug-TensorRT|x64.ActiveCfg = Debug|x64 + {A33D08BF-7881-4910-8439-5AE46646C1DD}.Debug-TensorRT|x64.Build.0 = Debug|x64 {A33D08BF-7881-4910-8439-5AE46646C1DD}.Release|x64.ActiveCfg = Release|x64 + {A33D08BF-7881-4910-8439-5AE46646C1DD}.Release|x64.Build.0 = Release|x64 {A33D08BF-7881-4910-8439-5AE46646C1DD}.Release-Cuda|x64.ActiveCfg = Release|x64 + {A33D08BF-7881-4910-8439-5AE46646C1DD}.Release-Cuda|x64.Build.0 = Release|x64 {A33D08BF-7881-4910-8439-5AE46646C1DD}.Release-TensorRT|x64.ActiveCfg = Release|x64 + {A33D08BF-7881-4910-8439-5AE46646C1DD}.Release-TensorRT|x64.Build.0 = Release|x64 {5A64FC26-6926-4A45-ACCE-1FE173166990}.Debug|x64.ActiveCfg = Debug|x64 {5A64FC26-6926-4A45-ACCE-1FE173166990}.Debug|x64.Build.0 = Debug|x64 {5A64FC26-6926-4A45-ACCE-1FE173166990}.Debug-Cuda|x64.ActiveCfg = Debug|x64 @@ -77,6 +99,18 @@ Global {5A64FC26-6926-4A45-ACCE-1FE173166990}.Release-Cuda|x64.Build.0 = Release|x64 {5A64FC26-6926-4A45-ACCE-1FE173166990}.Release-TensorRT|x64.ActiveCfg = Release|x64 {5A64FC26-6926-4A45-ACCE-1FE173166990}.Release-TensorRT|x64.Build.0 = Release|x64 + {0E8095A4-83EF-48AD-9BD1-0CC2893050F6}.Debug|x64.ActiveCfg = Debug|x64 + {0E8095A4-83EF-48AD-9BD1-0CC2893050F6}.Debug|x64.Build.0 = Debug|x64 + {0E8095A4-83EF-48AD-9BD1-0CC2893050F6}.Debug-Cuda|x64.ActiveCfg = Debug|x64 + {0E8095A4-83EF-48AD-9BD1-0CC2893050F6}.Debug-Cuda|x64.Build.0 = Debug|x64 + {0E8095A4-83EF-48AD-9BD1-0CC2893050F6}.Debug-TensorRT|x64.ActiveCfg = Debug|x64 + {0E8095A4-83EF-48AD-9BD1-0CC2893050F6}.Debug-TensorRT|x64.Build.0 = Debug|x64 + {0E8095A4-83EF-48AD-9BD1-0CC2893050F6}.Release|x64.ActiveCfg = Release|x64 + {0E8095A4-83EF-48AD-9BD1-0CC2893050F6}.Release|x64.Build.0 = Release|x64 + {0E8095A4-83EF-48AD-9BD1-0CC2893050F6}.Release-Cuda|x64.ActiveCfg = Release|x64 + {0E8095A4-83EF-48AD-9BD1-0CC2893050F6}.Release-Cuda|x64.Build.0 = Release|x64 + {0E8095A4-83EF-48AD-9BD1-0CC2893050F6}.Release-TensorRT|x64.ActiveCfg = Release|x64 + {0E8095A4-83EF-48AD-9BD1-0CC2893050F6}.Release-TensorRT|x64.Build.0 = Release|x64 EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE