diff --git a/OnnxStack.Core/Extensions/TensorExtension.cs b/OnnxStack.Core/Extensions/TensorExtension.cs
index ae99cb53..f701553e 100644
--- a/OnnxStack.Core/Extensions/TensorExtension.cs
+++ b/OnnxStack.Core/Extensions/TensorExtension.cs
@@ -5,6 +5,33 @@ namespace OnnxStack.Core
{
public static class TensorExtension
{
+ ///
+ /// Repeats the specified Tensor along the 0 axis.
+ ///
+ /// The tensor1.
+ /// The count.
+ /// The axis.
+ ///
+ /// Only axis 0 is supported
+ public static DenseTensor Repeat(this DenseTensor tensor1, int count, int axis = 0)
+ {
+ if (axis != 0)
+ throw new NotImplementedException("Only axis 0 is supported");
+
+ var dimensions = tensor1.Dimensions.ToArray();
+ dimensions[0] *= count;
+
+ var length = (int)tensor1.Length;
+ var totalLength = length * count;
+ var buffer = new float[totalLength].AsMemory();
+ for (int i = 0; i < count; i++)
+ {
+ tensor1.Buffer.CopyTo(buffer[(i * length)..]);
+ }
+ return new DenseTensor(buffer, dimensions);
+ }
+
+
///
/// Concatenates the specified tensors along the specified axis.
///
diff --git a/OnnxStack.FeatureExtractor/Common/FeatureExtractorModelSet.cs b/OnnxStack.FeatureExtractor/Common/FeatureExtractorModelSet.cs
new file mode 100644
index 00000000..481f3e1f
--- /dev/null
+++ b/OnnxStack.FeatureExtractor/Common/FeatureExtractorModelSet.cs
@@ -0,0 +1,18 @@
+using Microsoft.ML.OnnxRuntime;
+using OnnxStack.Core.Config;
+using System.Collections.Generic;
+
+namespace OnnxStack.FeatureExtractor.Common
+{
+ public record FeatureExtractorModelSet : IOnnxModelSetConfig
+ {
+ public string Name { get; set; }
+ public bool IsEnabled { get; set; }
+ public int DeviceId { get; set; }
+ public int InterOpNumThreads { get; set; }
+ public int IntraOpNumThreads { get; set; }
+ public ExecutionMode ExecutionMode { get; set; }
+ public ExecutionProvider ExecutionProvider { get; set; }
+ public List ModelConfigurations { get; set; }
+ }
+}
diff --git a/OnnxStack.FeatureExtractor/OnnxStack.FeatureExtractor.csproj b/OnnxStack.FeatureExtractor/OnnxStack.FeatureExtractor.csproj
new file mode 100644
index 00000000..fdf79e8c
--- /dev/null
+++ b/OnnxStack.FeatureExtractor/OnnxStack.FeatureExtractor.csproj
@@ -0,0 +1,49 @@
+
+
+
+ 0.17.0
+ net7.0
+ disable
+ disable
+ x64
+
+ OnnxStack.FeatureExtractor
+ Backyard Industries
+
+ OnnxRuntime Image Feature Extractor Library for .NET Core
+
+ Backyard Industries - 2023
+ https://github.com/saddam213/OnnxStack
+ onnx;onnx-runtime;
+ sa_ddam213
+ README.md
+ OnnxStack.FeatureExtractor
+ $(AssemblyName)
+ Apache-2.0
+ True
+ True
+ OnnxStack - 128x128.png
+ Debug;Release
+ x64
+
+
+
+
+ \
+ True
+
+
+
+
+
+
+
+
+
+
+ \
+ True
+
+
+
+
diff --git a/OnnxStack.FeatureExtractor/README.md b/OnnxStack.FeatureExtractor/README.md
new file mode 100644
index 00000000..189e799d
--- /dev/null
+++ b/OnnxStack.FeatureExtractor/README.md
@@ -0,0 +1,13 @@
+# OnnxStack.FeatureExtractor
+
+## Canny
+https://huggingface.co/axodoxian/controlnet_onnx/resolve/main/annotators/canny.onnx
+
+## Hed
+https://huggingface.co/axodoxian/controlnet_onnx/resolve/main/annotators/hed.onnx
+
+## Depth
+https://huggingface.co/axodoxian/controlnet_onnx/resolve/main/annotators/depth.onnx
+
+## OpenPose (TODO)
+https://huggingface.co/axodoxian/controlnet_onnx/resolve/main/annotators/openpose.onnx
\ No newline at end of file
diff --git a/OnnxStack.FeatureExtractor/Services/FeatureExtractorService.cs b/OnnxStack.FeatureExtractor/Services/FeatureExtractorService.cs
new file mode 100644
index 00000000..5cf509e6
--- /dev/null
+++ b/OnnxStack.FeatureExtractor/Services/FeatureExtractorService.cs
@@ -0,0 +1,162 @@
+using Microsoft.Extensions.Logging;
+using Microsoft.ML.OnnxRuntime.Tensors;
+using OnnxStack.Core;
+using OnnxStack.Core.Config;
+using OnnxStack.Core.Image;
+using OnnxStack.Core.Model;
+using OnnxStack.Core.Services;
+using OnnxStack.FeatureExtractor.Common;
+using System.Linq;
+using System.Threading.Tasks;
+
+namespace OnnxStack.FeatureExtractor.Services
+{
+ ///
+ /// Service for handing images for input and output of the diffusion process
+ ///
+ ///
+ public class FeatureExtractorService : IFeatureExtractorService
+ {
+ private readonly ILogger _logger;
+ private readonly IOnnxModelService _onnxModelService;
+
+
+ ///
+ /// Initializes a new instance of the class.
+ ///
+ /// The onnx model service.
+ public FeatureExtractorService(IOnnxModelService onnxModelService, ILogger logger)
+ {
+ _logger = logger;
+ _onnxModelService = onnxModelService;
+ }
+
+
+ ///
+ /// Generates the canny image mask.
+ ///
+ /// The control net model.
+ /// The input image.
+ /// The height.
+ /// The width.
+ ///
+ public async Task CannyImage(FeatureExtractorModelSet controlNetModel, InputImage inputImage, int height, int width)
+ {
+ _logger.LogInformation($"[CannyImage] - Generating Canny image...");
+ var controlImage = await inputImage.ToDenseTensorAsync(height, width, ImageNormalizeType.ZeroToOne);
+ var metadata = _onnxModelService.GetModelMetadata(controlNetModel, OnnxModelType.Annotation);
+ using (var inferenceParameters = new OnnxInferenceParameters(metadata))
+ {
+ inferenceParameters.AddInputTensor(controlImage);
+ inferenceParameters.AddOutputBuffer(new[] { 1, 1, height, width });
+
+ var results = await _onnxModelService.RunInferenceAsync(controlNetModel, OnnxModelType.Annotation, inferenceParameters);
+ using (var result = results.First())
+ {
+ var testImage = result.ToDenseTensor().Repeat(3);
+ var imageTensor = new DenseTensor(controlImage.Dimensions);
+ for (int i = 0; i < testImage.Length; i++)
+ imageTensor.SetValue(i, testImage.GetValue(i));
+
+ var maskImage = imageTensor.ToImageMask();
+ //await maskImage.SaveAsPngAsync("D:\\Canny.png");
+ _logger.LogInformation($"[CannyImage] - Canny image generation complete.");
+ return new InputImage(maskImage);
+ }
+ }
+ }
+
+
+ ///
+ /// Generates the hard edge image mask.
+ ///
+ /// The control net model.
+ /// The input image.
+ /// The height.
+ /// The width.
+ ///
+ public async Task HedImage(FeatureExtractorModelSet controlNetModel, InputImage inputImage, int height, int width)
+ {
+ _logger.LogInformation($"[HedImage] - Generating HardEdge image...");
+ var controlImage = await inputImage.ToDenseTensorAsync(height, width, ImageNormalizeType.ZeroToOne);
+ var metadata = _onnxModelService.GetModelMetadata(controlNetModel, OnnxModelType.Annotation);
+ using (var inferenceParameters = new OnnxInferenceParameters(metadata))
+ {
+ inferenceParameters.AddInputTensor(controlImage);
+ inferenceParameters.AddOutputBuffer(new[] { 1, 1, height, width });
+
+ var results = await _onnxModelService.RunInferenceAsync(controlNetModel, OnnxModelType.Annotation, inferenceParameters);
+ using (var result = results.First())
+ {
+ var testImage = result.ToDenseTensor().Repeat(3);
+ var imageTensor = new DenseTensor(controlImage.Dimensions);
+ for (int i = 0; i < testImage.Length; i++)
+ imageTensor.SetValue(i, testImage.GetValue(i));
+
+ var maskImage = imageTensor.ToImageMask();
+ //await maskImage.SaveAsPngAsync("D:\\Hed.png");
+ _logger.LogInformation($"[HedImage] - HardEdge image generation complete.");
+ return new InputImage(maskImage);
+ }
+ }
+ }
+
+
+ ///
+ /// Generates the depth image mask.
+ ///
+ /// The control net model.
+ /// The input image.
+ /// The height.
+ /// The width.
+ ///
+ public async Task DepthImage(FeatureExtractorModelSet controlNetModel, InputImage inputImage, int height, int width)
+ {
+ _logger.LogInformation($"[DepthImage] - Generating Depth image...");
+ var controlImage = await inputImage.ToDenseTensorAsync(height, width, ImageNormalizeType.ZeroToOne);
+ var metadata = _onnxModelService.GetModelMetadata(controlNetModel, OnnxModelType.Annotation);
+ using (var inferenceParameters = new OnnxInferenceParameters(metadata))
+ {
+ inferenceParameters.AddInputTensor(controlImage);
+ inferenceParameters.AddOutputBuffer(new[] { 1, 1, height, width });
+
+ var results = await _onnxModelService.RunInferenceAsync(controlNetModel, OnnxModelType.Annotation, inferenceParameters);
+ using (var result = results.First())
+ {
+ var testImage = result.ToDenseTensor().Repeat(3);
+ var imageTensor = new DenseTensor(controlImage.Dimensions);
+ for (int i = 0; i < testImage.Length; i++)
+ imageTensor.SetValue(i, testImage.GetValue(i));
+
+ NormalizeDepthTensor(imageTensor);
+ var maskImage = imageTensor.ToImageMask();
+ //await maskImage.SaveAsPngAsync("D:\\Depth.png");
+ _logger.LogInformation($"[DepthImage] - Depth image generation complete.");
+ return new InputImage(maskImage);
+ }
+ }
+ }
+
+
+ ///
+ /// Normalizes the depth tensor.
+ ///
+ /// The value.
+ public static void NormalizeDepthTensor(DenseTensor value)
+ {
+ var values = value.Buffer.Span;
+ float min = float.PositiveInfinity, max = float.NegativeInfinity;
+ foreach (var val in values)
+ {
+ if (min > val) min = val;
+ if (max < val) max = val;
+ }
+
+ var range = max - min;
+ for (var i = 0; i < values.Length; i++)
+ {
+ values[i] = (values[i] - min) / range;
+ }
+ }
+ }
+}
diff --git a/OnnxStack.FeatureExtractor/Services/IFeatureExtractorService.cs b/OnnxStack.FeatureExtractor/Services/IFeatureExtractorService.cs
new file mode 100644
index 00000000..04109210
--- /dev/null
+++ b/OnnxStack.FeatureExtractor/Services/IFeatureExtractorService.cs
@@ -0,0 +1,12 @@
+using OnnxStack.Core.Image;
+using System.Threading.Tasks;
+
+namespace OnnxStack.FeatureExtractor.Common
+{
+ public interface IFeatureExtractorService
+ {
+ Task CannyImage(FeatureExtractorModelSet controlNetModel, InputImage inputImage, int height, int width);
+ Task HedImage(FeatureExtractorModelSet controlNetModel, InputImage inputImage, int height, int width);
+ Task DepthImage(FeatureExtractorModelSet controlNetModel, InputImage inputImage, int height, int width);
+ }
+}
\ No newline at end of file
diff --git a/OnnxStack.StableDiffusion/Helpers/TensorHelper.cs b/OnnxStack.StableDiffusion/Helpers/TensorHelper.cs
index b3c88ca1..942dd36e 100644
--- a/OnnxStack.StableDiffusion/Helpers/TensorHelper.cs
+++ b/OnnxStack.StableDiffusion/Helpers/TensorHelper.cs
@@ -253,31 +253,7 @@ public static DenseTensor Clip(this DenseTensor tensor, float minV
- ///
- /// Repeats the specified Tensor along the 0 axis.
- ///
- /// The tensor1.
- /// The count.
- /// The axis.
- ///
- /// Only axis 0 is supported
- public static DenseTensor Repeat(this DenseTensor tensor1, int count, int axis = 0)
- {
- if (axis != 0)
- throw new NotImplementedException("Only axis 0 is supported");
-
- var dimensions = tensor1.Dimensions.ToArray();
- dimensions[0] *= count;
- var length = (int)tensor1.Length;
- var totalLength = length * count;
- var buffer = new float[totalLength].AsMemory();
- for (int i = 0; i < count; i++)
- {
- tensor1.Buffer.CopyTo(buffer[(i * length)..]);
- }
- return new DenseTensor(buffer, dimensions);
- }
///
diff --git a/OnnxStack.sln b/OnnxStack.sln
index 336f3039..1cc8f106 100644
--- a/OnnxStack.sln
+++ b/OnnxStack.sln
@@ -15,6 +15,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "OnnxStack.ImageUpscaler", "
EndProject
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "OnnxStack.Adapter", "OnnxStack.Adapter\OnnxStack.Adapter.vcxproj", "{5A64FC26-6926-4A45-ACCE-1FE173166990}"
EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "OnnxStack.FeatureExtractor", "OnnxStack.FeatureExtractor\OnnxStack.FeatureExtractor.csproj", "{0E8095A4-83EF-48AD-9BD1-0CC2893050F6}"
+EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|x64 = Debug|x64
@@ -30,41 +32,61 @@ Global
{02404CEB-207F-4D19-894C-11D51394F1D5}.Debug-Cuda|x64.ActiveCfg = Debug|x64
{02404CEB-207F-4D19-894C-11D51394F1D5}.Debug-Cuda|x64.Build.0 = Debug|x64
{02404CEB-207F-4D19-894C-11D51394F1D5}.Debug-TensorRT|x64.ActiveCfg = Debug|x64
+ {02404CEB-207F-4D19-894C-11D51394F1D5}.Debug-TensorRT|x64.Build.0 = Debug|x64
{02404CEB-207F-4D19-894C-11D51394F1D5}.Release|x64.ActiveCfg = Release|x64
+ {02404CEB-207F-4D19-894C-11D51394F1D5}.Release|x64.Build.0 = Release|x64
{02404CEB-207F-4D19-894C-11D51394F1D5}.Release-Cuda|x64.ActiveCfg = Release|x64
+ {02404CEB-207F-4D19-894C-11D51394F1D5}.Release-Cuda|x64.Build.0 = Release|x64
{02404CEB-207F-4D19-894C-11D51394F1D5}.Release-TensorRT|x64.ActiveCfg = Release|x64
+ {02404CEB-207F-4D19-894C-11D51394F1D5}.Release-TensorRT|x64.Build.0 = Release|x64
{EA1F61D0-490B-42EC-96F5-7DCCAB94457A}.Debug|x64.ActiveCfg = Debug|x64
{EA1F61D0-490B-42EC-96F5-7DCCAB94457A}.Debug|x64.Build.0 = Debug|x64
{EA1F61D0-490B-42EC-96F5-7DCCAB94457A}.Debug-Cuda|x64.ActiveCfg = Debug|x64
{EA1F61D0-490B-42EC-96F5-7DCCAB94457A}.Debug-Cuda|x64.Build.0 = Debug|x64
{EA1F61D0-490B-42EC-96F5-7DCCAB94457A}.Debug-TensorRT|x64.ActiveCfg = Debug|x64
+ {EA1F61D0-490B-42EC-96F5-7DCCAB94457A}.Debug-TensorRT|x64.Build.0 = Debug|x64
{EA1F61D0-490B-42EC-96F5-7DCCAB94457A}.Release|x64.ActiveCfg = Release|x64
+ {EA1F61D0-490B-42EC-96F5-7DCCAB94457A}.Release|x64.Build.0 = Release|x64
{EA1F61D0-490B-42EC-96F5-7DCCAB94457A}.Release-Cuda|x64.ActiveCfg = Release|x64
+ {EA1F61D0-490B-42EC-96F5-7DCCAB94457A}.Release-Cuda|x64.Build.0 = Release|x64
{EA1F61D0-490B-42EC-96F5-7DCCAB94457A}.Release-TensorRT|x64.ActiveCfg = Release|x64
+ {EA1F61D0-490B-42EC-96F5-7DCCAB94457A}.Release-TensorRT|x64.Build.0 = Release|x64
{46A43C80-A440-4461-B7EB-81FA998FB24B}.Debug|x64.ActiveCfg = Debug|x64
{46A43C80-A440-4461-B7EB-81FA998FB24B}.Debug|x64.Build.0 = Debug|x64
{46A43C80-A440-4461-B7EB-81FA998FB24B}.Debug-Cuda|x64.ActiveCfg = Debug-Cuda|x64
{46A43C80-A440-4461-B7EB-81FA998FB24B}.Debug-Cuda|x64.Build.0 = Debug-Cuda|x64
{46A43C80-A440-4461-B7EB-81FA998FB24B}.Debug-TensorRT|x64.ActiveCfg = Debug-TensorRT|x64
+ {46A43C80-A440-4461-B7EB-81FA998FB24B}.Debug-TensorRT|x64.Build.0 = Debug-TensorRT|x64
{46A43C80-A440-4461-B7EB-81FA998FB24B}.Release|x64.ActiveCfg = Release|x64
+ {46A43C80-A440-4461-B7EB-81FA998FB24B}.Release|x64.Build.0 = Release|x64
{46A43C80-A440-4461-B7EB-81FA998FB24B}.Release-Cuda|x64.ActiveCfg = Release-Cuda|x64
+ {46A43C80-A440-4461-B7EB-81FA998FB24B}.Release-Cuda|x64.Build.0 = Release-Cuda|x64
{46A43C80-A440-4461-B7EB-81FA998FB24B}.Release-TensorRT|x64.ActiveCfg = Release-TensorRT|x64
+ {46A43C80-A440-4461-B7EB-81FA998FB24B}.Release-TensorRT|x64.Build.0 = Release-TensorRT|x64
{85BB1855-8C3B-4049-A2DD-1130FA6CD846}.Debug|x64.ActiveCfg = Debug|x64
{85BB1855-8C3B-4049-A2DD-1130FA6CD846}.Debug|x64.Build.0 = Debug|x64
{85BB1855-8C3B-4049-A2DD-1130FA6CD846}.Debug-Cuda|x64.ActiveCfg = Debug-Cuda|x64
{85BB1855-8C3B-4049-A2DD-1130FA6CD846}.Debug-Cuda|x64.Build.0 = Debug-Cuda|x64
{85BB1855-8C3B-4049-A2DD-1130FA6CD846}.Debug-TensorRT|x64.ActiveCfg = Debug-TensorRT|x64
+ {85BB1855-8C3B-4049-A2DD-1130FA6CD846}.Debug-TensorRT|x64.Build.0 = Debug-TensorRT|x64
{85BB1855-8C3B-4049-A2DD-1130FA6CD846}.Release|x64.ActiveCfg = Release|x64
+ {85BB1855-8C3B-4049-A2DD-1130FA6CD846}.Release|x64.Build.0 = Release|x64
{85BB1855-8C3B-4049-A2DD-1130FA6CD846}.Release-Cuda|x64.ActiveCfg = Release-Cuda|x64
+ {85BB1855-8C3B-4049-A2DD-1130FA6CD846}.Release-Cuda|x64.Build.0 = Release-Cuda|x64
{85BB1855-8C3B-4049-A2DD-1130FA6CD846}.Release-TensorRT|x64.ActiveCfg = Release-TensorRT|x64
+ {85BB1855-8C3B-4049-A2DD-1130FA6CD846}.Release-TensorRT|x64.Build.0 = Release-TensorRT|x64
{A33D08BF-7881-4910-8439-5AE46646C1DD}.Debug|x64.ActiveCfg = Debug|x64
{A33D08BF-7881-4910-8439-5AE46646C1DD}.Debug|x64.Build.0 = Debug|x64
{A33D08BF-7881-4910-8439-5AE46646C1DD}.Debug-Cuda|x64.ActiveCfg = Debug|x64
{A33D08BF-7881-4910-8439-5AE46646C1DD}.Debug-Cuda|x64.Build.0 = Debug|x64
{A33D08BF-7881-4910-8439-5AE46646C1DD}.Debug-TensorRT|x64.ActiveCfg = Debug|x64
+ {A33D08BF-7881-4910-8439-5AE46646C1DD}.Debug-TensorRT|x64.Build.0 = Debug|x64
{A33D08BF-7881-4910-8439-5AE46646C1DD}.Release|x64.ActiveCfg = Release|x64
+ {A33D08BF-7881-4910-8439-5AE46646C1DD}.Release|x64.Build.0 = Release|x64
{A33D08BF-7881-4910-8439-5AE46646C1DD}.Release-Cuda|x64.ActiveCfg = Release|x64
+ {A33D08BF-7881-4910-8439-5AE46646C1DD}.Release-Cuda|x64.Build.0 = Release|x64
{A33D08BF-7881-4910-8439-5AE46646C1DD}.Release-TensorRT|x64.ActiveCfg = Release|x64
+ {A33D08BF-7881-4910-8439-5AE46646C1DD}.Release-TensorRT|x64.Build.0 = Release|x64
{5A64FC26-6926-4A45-ACCE-1FE173166990}.Debug|x64.ActiveCfg = Debug|x64
{5A64FC26-6926-4A45-ACCE-1FE173166990}.Debug|x64.Build.0 = Debug|x64
{5A64FC26-6926-4A45-ACCE-1FE173166990}.Debug-Cuda|x64.ActiveCfg = Debug|x64
@@ -77,6 +99,18 @@ Global
{5A64FC26-6926-4A45-ACCE-1FE173166990}.Release-Cuda|x64.Build.0 = Release|x64
{5A64FC26-6926-4A45-ACCE-1FE173166990}.Release-TensorRT|x64.ActiveCfg = Release|x64
{5A64FC26-6926-4A45-ACCE-1FE173166990}.Release-TensorRT|x64.Build.0 = Release|x64
+ {0E8095A4-83EF-48AD-9BD1-0CC2893050F6}.Debug|x64.ActiveCfg = Debug|x64
+ {0E8095A4-83EF-48AD-9BD1-0CC2893050F6}.Debug|x64.Build.0 = Debug|x64
+ {0E8095A4-83EF-48AD-9BD1-0CC2893050F6}.Debug-Cuda|x64.ActiveCfg = Debug|x64
+ {0E8095A4-83EF-48AD-9BD1-0CC2893050F6}.Debug-Cuda|x64.Build.0 = Debug|x64
+ {0E8095A4-83EF-48AD-9BD1-0CC2893050F6}.Debug-TensorRT|x64.ActiveCfg = Debug|x64
+ {0E8095A4-83EF-48AD-9BD1-0CC2893050F6}.Debug-TensorRT|x64.Build.0 = Debug|x64
+ {0E8095A4-83EF-48AD-9BD1-0CC2893050F6}.Release|x64.ActiveCfg = Release|x64
+ {0E8095A4-83EF-48AD-9BD1-0CC2893050F6}.Release|x64.Build.0 = Release|x64
+ {0E8095A4-83EF-48AD-9BD1-0CC2893050F6}.Release-Cuda|x64.ActiveCfg = Release|x64
+ {0E8095A4-83EF-48AD-9BD1-0CC2893050F6}.Release-Cuda|x64.Build.0 = Release|x64
+ {0E8095A4-83EF-48AD-9BD1-0CC2893050F6}.Release-TensorRT|x64.ActiveCfg = Release|x64
+ {0E8095A4-83EF-48AD-9BD1-0CC2893050F6}.Release-TensorRT|x64.Build.0 = Release|x64
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE