FeatureExtractor Project

OnnxStack.Core/Extensions/TensorExtension.cs

@@ -5,6 +5,33 @@ namespace OnnxStack.Core
 {
     public static class TensorExtension
     {
+        /// <summary>
+        /// Repeats the specified Tensor along the 0 axis.
+        /// </summary>
+        /// <param name="tensor1">The tensor1.</param>
+        /// <param name="count">The count.</param>
+        /// <param name="axis">The axis.</param>
+        /// <returns></returns>
+        /// <exception cref="System.NotImplementedException">Only axis 0 is supported</exception>
+        public static DenseTensor<float> Repeat(this DenseTensor<float> tensor1, int count, int axis = 0)
+        {
+            if (axis != 0)
+                throw new NotImplementedException("Only axis 0 is supported");
+
+            var dimensions = tensor1.Dimensions.ToArray();
+            dimensions[0] *= count;
+
+            var length = (int)tensor1.Length;
+            var totalLength = length * count;
+            var buffer = new float[totalLength].AsMemory();
+            for (int i = 0; i < count; i++)
+            {
+                tensor1.Buffer.CopyTo(buffer[(i * length)..]);
+            }
+            return new DenseTensor<float>(buffer, dimensions);
+        }
+
+
         /// <summary>
         /// Concatenates the specified tensors along the specified axis.
         /// </summary>

OnnxStack.FeatureExtractor/Common/FeatureExtractorModelSet.cs

@@ -0,0 +1,18 @@
+using Microsoft.ML.OnnxRuntime;
+using OnnxStack.Core.Config;
+using System.Collections.Generic;
+
+namespace OnnxStack.FeatureExtractor.Common
+{
+    public record FeatureExtractorModelSet : IOnnxModelSetConfig
+    {
+        public string Name { get; set; }
+        public bool IsEnabled { get; set; }
+        public int DeviceId { get; set; }
+        public int InterOpNumThreads { get; set; }
+        public int IntraOpNumThreads { get; set; }
+        public ExecutionMode ExecutionMode { get; set; }
+        public ExecutionProvider ExecutionProvider { get; set; }
+        public List<OnnxModelConfig> ModelConfigurations { get; set; }
+    }
+}

OnnxStack.FeatureExtractor/OnnxStack.FeatureExtractor.csproj

@@ -0,0 +1,49 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <Version>0.17.0</Version>
+    <TargetFramework>net7.0</TargetFramework>
+    <ImplicitUsings>disable</ImplicitUsings>
+    <Nullable>disable</Nullable>
+    <PlatformTarget>x64</PlatformTarget>
+
+    <Title>OnnxStack.FeatureExtractor</Title>
+    <Company>Backyard Industries</Company>
+    <Description>
+      OnnxRuntime Image Feature Extractor Library for .NET Core
+    </Description>
+    <Copyright>Backyard Industries - 2023</Copyright>
+    <RepositoryUrl>https://github.com/saddam213/OnnxStack</RepositoryUrl>
+    <PackageTags>onnx;onnx-runtime;</PackageTags>
+    <Authors>sa_ddam213</Authors>
+    <PackageReadmeFile>README.md</PackageReadmeFile>
+    <PackageId>OnnxStack.FeatureExtractor</PackageId>
+    <Product>$(AssemblyName)</Product>
+    <PackageLicenseExpression>Apache-2.0</PackageLicenseExpression>
+    <PackageRequireLicenseAcceptance>True</PackageRequireLicenseAcceptance>
+    <AllowUnsafeBlocks>True</AllowUnsafeBlocks>
+    <PackageIcon>OnnxStack - 128x128.png</PackageIcon>
+    <Configurations>Debug;Release</Configurations>
+    <Platforms>x64</Platforms>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <None Include="..\Assets\OnnxStack - 128x128.png" Link="OnnxStack - 128x128.png">
+      <PackagePath>\</PackagePath>
+      <Pack>True</Pack>
+    </None>
+  </ItemGroup>
+
+  <ItemGroup>
+    <PackageReference Include="OnnxStack.Core" Version="0.17.0" Condition=" '$(Configuration)' == 'Release' " />
+    <ProjectReference Include="..\OnnxStack.Core\OnnxStack.Core.csproj" Condition=" '$(Configuration)' == 'Debug' " />
+  </ItemGroup>
+
+  <ItemGroup>
+    <None Update="README.md">
+      <PackagePath>\</PackagePath>
+      <Pack>True</Pack>
+    </None>
+  </ItemGroup>
+
+</Project>

OnnxStack.FeatureExtractor/README.md

@@ -0,0 +1,13 @@
+# OnnxStack.FeatureExtractor
+
+## Canny
+https://huggingface.co/axodoxian/controlnet_onnx/resolve/main/annotators/canny.onnx
+
+## Hed
+https://huggingface.co/axodoxian/controlnet_onnx/resolve/main/annotators/hed.onnx
+
+## Depth
+https://huggingface.co/axodoxian/controlnet_onnx/resolve/main/annotators/depth.onnx
+
+## OpenPose (TODO)
+https://huggingface.co/axodoxian/controlnet_onnx/resolve/main/annotators/openpose.onnx

OnnxStack.FeatureExtractor/Services/FeatureExtractorService.cs

@@ -0,0 +1,162 @@
+using Microsoft.Extensions.Logging;
+using Microsoft.ML.OnnxRuntime.Tensors;
+using OnnxStack.Core;
+using OnnxStack.Core.Config;
+using OnnxStack.Core.Image;
+using OnnxStack.Core.Model;
+using OnnxStack.Core.Services;
+using OnnxStack.FeatureExtractor.Common;
+using System.Linq;
+using System.Threading.Tasks;
+
+namespace OnnxStack.FeatureExtractor.Services
+{
+    /// <summary>
+    /// Service for handing images for input and output of the diffusion process
+    /// </summary>
+    /// <seealso cref="OnnxStack.StableDiffusion.Common.IFeatureExtractor" />
+    public class FeatureExtractorService : IFeatureExtractorService
+    {
+        private readonly ILogger<FeatureExtractorService> _logger;
+        private readonly IOnnxModelService _onnxModelService;
+
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="FeatureExtractorService"/> class.
+        /// </summary>
+        /// <param name="onnxModelService">The onnx model service.</param>
+        public FeatureExtractorService(IOnnxModelService onnxModelService, ILogger<FeatureExtractorService> logger)
+        {
+            _logger = logger;
+            _onnxModelService = onnxModelService;
+        }
+
+
+        /// <summary>
+        /// Generates the canny image mask.
+        /// </summary>
+        /// <param name="controlNetModel">The control net model.</param>
+        /// <param name="inputImage">The input image.</param>
+        /// <param name="height">The height.</param>
+        /// <param name="width">The width.</param>
+        /// <returns></returns>
+        public async Task<InputImage> CannyImage(FeatureExtractorModelSet controlNetModel, InputImage inputImage, int height, int width)
+        {
+            _logger.LogInformation($"[CannyImage] - Generating Canny image...");
+            var controlImage = await inputImage.ToDenseTensorAsync(height, width, ImageNormalizeType.ZeroToOne);
+            var metadata = _onnxModelService.GetModelMetadata(controlNetModel, OnnxModelType.Annotation);
+            using (var inferenceParameters = new OnnxInferenceParameters(metadata))
+            {
+                inferenceParameters.AddInputTensor(controlImage);
+                inferenceParameters.AddOutputBuffer(new[] { 1, 1, height, width });
+
+                var results = await _onnxModelService.RunInferenceAsync(controlNetModel, OnnxModelType.Annotation, inferenceParameters);
+                using (var result = results.First())
+                {
+                    var testImage = result.ToDenseTensor().Repeat(3);
+                    var imageTensor = new DenseTensor<float>(controlImage.Dimensions);
+                    for (int i = 0; i < testImage.Length; i++)
+                        imageTensor.SetValue(i, testImage.GetValue(i));
+
+                    var maskImage = imageTensor.ToImageMask();
+                    //await maskImage.SaveAsPngAsync("D:\\Canny.png");
+                    _logger.LogInformation($"[CannyImage] - Canny image generation complete.");
+                    return new InputImage(maskImage);
+                }
+            }
+        }
+
+
+        /// <summary>
+        /// Generates the hard edge image mask.
+        /// </summary>
+        /// <param name="controlNetModel">The control net model.</param>
+        /// <param name="inputImage">The input image.</param>
+        /// <param name="height">The height.</param>
+        /// <param name="width">The width.</param>
+        /// <returns></returns>
+        public async Task<InputImage> HedImage(FeatureExtractorModelSet controlNetModel, InputImage inputImage, int height, int width)
+        {
+            _logger.LogInformation($"[HedImage] - Generating HardEdge image...");
+            var controlImage = await inputImage.ToDenseTensorAsync(height, width, ImageNormalizeType.ZeroToOne);
+            var metadata = _onnxModelService.GetModelMetadata(controlNetModel, OnnxModelType.Annotation);
+            using (var inferenceParameters = new OnnxInferenceParameters(metadata))
+            {
+                inferenceParameters.AddInputTensor(controlImage);
+                inferenceParameters.AddOutputBuffer(new[] { 1, 1, height, width });
+
+                var results = await _onnxModelService.RunInferenceAsync(controlNetModel, OnnxModelType.Annotation, inferenceParameters);
+                using (var result = results.First())
+                {
+                    var testImage = result.ToDenseTensor().Repeat(3);
+                    var imageTensor = new DenseTensor<float>(controlImage.Dimensions);
+                    for (int i = 0; i < testImage.Length; i++)
+                        imageTensor.SetValue(i, testImage.GetValue(i));
+
+                    var maskImage = imageTensor.ToImageMask();
+                    //await maskImage.SaveAsPngAsync("D:\\Hed.png");
+                    _logger.LogInformation($"[HedImage] - HardEdge image generation complete.");
+                    return new InputImage(maskImage);
+                }
+            }
+        }
+
+
+        /// <summary>
+        /// Generates the depth image mask.
+        /// </summary>
+        /// <param name="controlNetModel">The control net model.</param>
+        /// <param name="inputImage">The input image.</param>
+        /// <param name="height">The height.</param>
+        /// <param name="width">The width.</param>
+        /// <returns></returns>
+        public async Task<InputImage> DepthImage(FeatureExtractorModelSet controlNetModel, InputImage inputImage, int height, int width)
+        {
+            _logger.LogInformation($"[DepthImage] - Generating Depth image...");
+            var controlImage = await inputImage.ToDenseTensorAsync(height, width, ImageNormalizeType.ZeroToOne);
+            var metadata = _onnxModelService.GetModelMetadata(controlNetModel, OnnxModelType.Annotation);
+            using (var inferenceParameters = new OnnxInferenceParameters(metadata))
+            {
+                inferenceParameters.AddInputTensor(controlImage);
+                inferenceParameters.AddOutputBuffer(new[] { 1, 1, height, width });
+
+                var results = await _onnxModelService.RunInferenceAsync(controlNetModel, OnnxModelType.Annotation, inferenceParameters);
+                using (var result = results.First())
+                {
+                    var testImage = result.ToDenseTensor().Repeat(3);
+                    var imageTensor = new DenseTensor<float>(controlImage.Dimensions);
+                    for (int i = 0; i < testImage.Length; i++)
+                        imageTensor.SetValue(i, testImage.GetValue(i));
+
+                    NormalizeDepthTensor(imageTensor);
+                    var maskImage = imageTensor.ToImageMask();
+                    //await maskImage.SaveAsPngAsync("D:\\Depth.png");
+                    _logger.LogInformation($"[DepthImage] - Depth image generation complete.");
+                    return new InputImage(maskImage);
+                }
+            }
+        }
+
+
+        /// <summary>
+        /// Normalizes the depth tensor.
+        /// </summary>
+        /// <param name="value">The value.</param>
+        public static void NormalizeDepthTensor(DenseTensor<float> value)
+        {
+            var values = value.Buffer.Span;
+            float min = float.PositiveInfinity, max = float.NegativeInfinity;
+            foreach (var val in values)
+            {
+                if (min > val) min = val;
+                if (max < val) max = val;
+            }
+
+            var range = max - min;
+            for (var i = 0; i < values.Length; i++)
+            {
+                values[i] = (values[i] - min) / range;
+            }
+        }
+    }
+}

OnnxStack.FeatureExtractor/Services/IFeatureExtractorService.cs

@@ -0,0 +1,12 @@
+using OnnxStack.Core.Image;
+using System.Threading.Tasks;
+
+namespace OnnxStack.FeatureExtractor.Common
+{
+    public interface IFeatureExtractorService
+    {
+        Task<InputImage> CannyImage(FeatureExtractorModelSet controlNetModel, InputImage inputImage, int height, int width);
+        Task<InputImage> HedImage(FeatureExtractorModelSet controlNetModel, InputImage inputImage, int height, int width);
+        Task<InputImage> DepthImage(FeatureExtractorModelSet controlNetModel, InputImage inputImage, int height, int width);
+    }
+}

OnnxStack.StableDiffusion/Helpers/TensorHelper.cs

@@ -253,31 +253,7 @@ public static DenseTensor<float> Clip(this DenseTensor<float> tensor, float minV
 
 
 
-        /// <summary>
-        /// Repeats the specified Tensor along the 0 axis.
-        /// </summary>
-        /// <param name="tensor1">The tensor1.</param>
-        /// <param name="count">The count.</param>
-        /// <param name="axis">The axis.</param>
-        /// <returns></returns>
-        /// <exception cref="System.NotImplementedException">Only axis 0 is supported</exception>
-        public static DenseTensor<float> Repeat(this DenseTensor<float> tensor1, int count, int axis = 0)
-        {
-            if (axis != 0)
-                throw new NotImplementedException("Only axis 0 is supported");
-
-            var dimensions = tensor1.Dimensions.ToArray();
-            dimensions[0] *= count;
 
-            var length = (int)tensor1.Length;
-            var totalLength = length * count;
-            var buffer = new float[totalLength].AsMemory();
-            for (int i = 0; i < count; i++)
-            {
-                tensor1.Buffer.CopyTo(buffer[(i * length)..]);
-            }
-            return new DenseTensor<float>(buffer, dimensions);
-        }
 
 
         /// <summary>