Skip to content
This repository was archived by the owner on Nov 27, 2024. It is now read-only.

Commit 5d74009

Browse files
saddam213saddam213
authored
Merge pull request #104 from saddam213/FeatureExtractor
FeatureExtractor Project
2 parents 450e8bc + f3e6a00 commit 5d74009

File tree

8 files changed

+315
-24
lines changed

8 files changed

+315
-24
lines changed

‎OnnxStack.Core/Extensions/TensorExtension.cs

+27
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,33 @@ namespace OnnxStack.Core
55
{
66
public static class TensorExtension
77
{
8+
/// <summary>
9+
/// Repeats the specified Tensor along the 0 axis.
10+
/// </summary>
11+
/// <param name="tensor1">The tensor1.</param>
12+
/// <param name="count">The count.</param>
13+
/// <param name="axis">The axis.</param>
14+
/// <returns></returns>
15+
/// <exception cref="System.NotImplementedException">Only axis 0 is supported</exception>
16+
public static DenseTensor<float> Repeat(this DenseTensor<float> tensor1, int count, int axis = 0)
17+
{
18+
if (axis != 0)
19+
throw new NotImplementedException("Only axis 0 is supported");
20+
21+
var dimensions = tensor1.Dimensions.ToArray();
22+
dimensions[0] *= count;
23+
24+
var length = (int)tensor1.Length;
25+
var totalLength = length * count;
26+
var buffer = new float[totalLength].AsMemory();
27+
for (int i = 0; i < count; i++)
28+
{
29+
tensor1.Buffer.CopyTo(buffer[(i * length)..]);
30+
}
31+
return new DenseTensor<float>(buffer, dimensions);
32+
}
33+
34+
835
/// <summary>
936
/// Concatenates the specified tensors along the specified axis.
1037
/// </summary>

‎OnnxStack.FeatureExtractor/Common/FeatureExtractorModelSet.cs

+18
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
using Microsoft.ML.OnnxRuntime;
2+
using OnnxStack.Core.Config;
3+
using System.Collections.Generic;
4+
5+
namespace OnnxStack.FeatureExtractor.Common
6+
{
7+
public record FeatureExtractorModelSet : IOnnxModelSetConfig
8+
{
9+
public string Name { get; set; }
10+
public bool IsEnabled { get; set; }
11+
public int DeviceId { get; set; }
12+
public int InterOpNumThreads { get; set; }
13+
public int IntraOpNumThreads { get; set; }
14+
public ExecutionMode ExecutionMode { get; set; }
15+
public ExecutionProvider ExecutionProvider { get; set; }
16+
public List<OnnxModelConfig> ModelConfigurations { get; set; }
17+
}
18+
}

‎OnnxStack.FeatureExtractor/OnnxStack.FeatureExtractor.csproj

+49
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
<Project Sdk="Microsoft.NET.Sdk">
2+
3+
<PropertyGroup>
4+
<Version>0.17.0</Version>
5+
<TargetFramework>net7.0</TargetFramework>
6+
<ImplicitUsings>disable</ImplicitUsings>
7+
<Nullable>disable</Nullable>
8+
<PlatformTarget>x64</PlatformTarget>
9+
10+
<Title>OnnxStack.FeatureExtractor</Title>
11+
<Company>Backyard Industries</Company>
12+
<Description>
13+
OnnxRuntime Image Feature Extractor Library for .NET Core
14+
</Description>
15+
<Copyright>Backyard Industries - 2023</Copyright>
16+
<RepositoryUrl>https://github.com/saddam213/OnnxStack</RepositoryUrl>
17+
<PackageTags>onnx;onnx-runtime;</PackageTags>
18+
<Authors>sa_ddam213</Authors>
19+
<PackageReadmeFile>README.md</PackageReadmeFile>
20+
<PackageId>OnnxStack.FeatureExtractor</PackageId>
21+
<Product>$(AssemblyName)</Product>
22+
<PackageLicenseExpression>Apache-2.0</PackageLicenseExpression>
23+
<PackageRequireLicenseAcceptance>True</PackageRequireLicenseAcceptance>
24+
<AllowUnsafeBlocks>True</AllowUnsafeBlocks>
25+
<PackageIcon>OnnxStack - 128x128.png</PackageIcon>
26+
<Configurations>Debug;Release</Configurations>
27+
<Platforms>x64</Platforms>
28+
</PropertyGroup>
29+
30+
<ItemGroup>
31+
<None Include="..\Assets\OnnxStack - 128x128.png" Link="OnnxStack - 128x128.png">
32+
<PackagePath>\</PackagePath>
33+
<Pack>True</Pack>
34+
</None>
35+
</ItemGroup>
36+
37+
<ItemGroup>
38+
<PackageReference Include="OnnxStack.Core" Version="0.17.0" Condition=" '$(Configuration)' == 'Release' " />
39+
<ProjectReference Include="..\OnnxStack.Core\OnnxStack.Core.csproj" Condition=" '$(Configuration)' == 'Debug' " />
40+
</ItemGroup>
41+
42+
<ItemGroup>
43+
<None Update="README.md">
44+
<PackagePath>\</PackagePath>
45+
<Pack>True</Pack>
46+
</None>
47+
</ItemGroup>
48+
49+
</Project>

‎OnnxStack.FeatureExtractor/README.md

+13
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
# OnnxStack.FeatureExtractor
2+
3+
## Canny
4+
https://huggingface.co/axodoxian/controlnet_onnx/resolve/main/annotators/canny.onnx
5+
6+
## Hed
7+
https://huggingface.co/axodoxian/controlnet_onnx/resolve/main/annotators/hed.onnx
8+
9+
## Depth
10+
https://huggingface.co/axodoxian/controlnet_onnx/resolve/main/annotators/depth.onnx
11+
12+
## OpenPose (TODO)
13+
https://huggingface.co/axodoxian/controlnet_onnx/resolve/main/annotators/openpose.onnx

‎OnnxStack.FeatureExtractor/Services/FeatureExtractorService.cs

+162
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,162 @@
1+
using Microsoft.Extensions.Logging;
2+
using Microsoft.ML.OnnxRuntime.Tensors;
3+
using OnnxStack.Core;
4+
using OnnxStack.Core.Config;
5+
using OnnxStack.Core.Image;
6+
using OnnxStack.Core.Model;
7+
using OnnxStack.Core.Services;
8+
using OnnxStack.FeatureExtractor.Common;
9+
using System.Linq;
10+
using System.Threading.Tasks;
11+
12+
namespace OnnxStack.FeatureExtractor.Services
13+
{
14+
/// <summary>
15+
/// Service for handing images for input and output of the diffusion process
16+
/// </summary>
17+
/// <seealso cref="OnnxStack.StableDiffusion.Common.IFeatureExtractor" />
18+
public class FeatureExtractorService : IFeatureExtractorService
19+
{
20+
private readonly ILogger<FeatureExtractorService> _logger;
21+
private readonly IOnnxModelService _onnxModelService;
22+
23+
24+
/// <summary>
25+
/// Initializes a new instance of the <see cref="FeatureExtractorService"/> class.
26+
/// </summary>
27+
/// <param name="onnxModelService">The onnx model service.</param>
28+
public FeatureExtractorService(IOnnxModelService onnxModelService, ILogger<FeatureExtractorService> logger)
29+
{
30+
_logger = logger;
31+
_onnxModelService = onnxModelService;
32+
}
33+
34+
35+
/// <summary>
36+
/// Generates the canny image mask.
37+
/// </summary>
38+
/// <param name="controlNetModel">The control net model.</param>
39+
/// <param name="inputImage">The input image.</param>
40+
/// <param name="height">The height.</param>
41+
/// <param name="width">The width.</param>
42+
/// <returns></returns>
43+
public async Task<InputImage> CannyImage(FeatureExtractorModelSet controlNetModel, InputImage inputImage, int height, int width)
44+
{
45+
_logger.LogInformation($"[CannyImage] - Generating Canny image...");
46+
var controlImage = await inputImage.ToDenseTensorAsync(height, width, ImageNormalizeType.ZeroToOne);
47+
var metadata = _onnxModelService.GetModelMetadata(controlNetModel, OnnxModelType.Annotation);
48+
using (var inferenceParameters = new OnnxInferenceParameters(metadata))
49+
{
50+
inferenceParameters.AddInputTensor(controlImage);
51+
inferenceParameters.AddOutputBuffer(new[] { 1, 1, height, width });
52+
53+
var results = await _onnxModelService.RunInferenceAsync(controlNetModel, OnnxModelType.Annotation, inferenceParameters);
54+
using (var result = results.First())
55+
{
56+
var testImage = result.ToDenseTensor().Repeat(3);
57+
var imageTensor = new DenseTensor<float>(controlImage.Dimensions);
58+
for (int i = 0; i < testImage.Length; i++)
59+
imageTensor.SetValue(i, testImage.GetValue(i));
60+
61+
var maskImage = imageTensor.ToImageMask();
62+
//await maskImage.SaveAsPngAsync("D:\\Canny.png");
63+
_logger.LogInformation($"[CannyImage] - Canny image generation complete.");
64+
return new InputImage(maskImage);
65+
}
66+
}
67+
}
68+
69+
70+
/// <summary>
71+
/// Generates the hard edge image mask.
72+
/// </summary>
73+
/// <param name="controlNetModel">The control net model.</param>
74+
/// <param name="inputImage">The input image.</param>
75+
/// <param name="height">The height.</param>
76+
/// <param name="width">The width.</param>
77+
/// <returns></returns>
78+
public async Task<InputImage> HedImage(FeatureExtractorModelSet controlNetModel, InputImage inputImage, int height, int width)
79+
{
80+
_logger.LogInformation($"[HedImage] - Generating HardEdge image...");
81+
var controlImage = await inputImage.ToDenseTensorAsync(height, width, ImageNormalizeType.ZeroToOne);
82+
var metadata = _onnxModelService.GetModelMetadata(controlNetModel, OnnxModelType.Annotation);
83+
using (var inferenceParameters = new OnnxInferenceParameters(metadata))
84+
{
85+
inferenceParameters.AddInputTensor(controlImage);
86+
inferenceParameters.AddOutputBuffer(new[] { 1, 1, height, width });
87+
88+
var results = await _onnxModelService.RunInferenceAsync(controlNetModel, OnnxModelType.Annotation, inferenceParameters);
89+
using (var result = results.First())
90+
{
91+
var testImage = result.ToDenseTensor().Repeat(3);
92+
var imageTensor = new DenseTensor<float>(controlImage.Dimensions);
93+
for (int i = 0; i < testImage.Length; i++)
94+
imageTensor.SetValue(i, testImage.GetValue(i));
95+
96+
var maskImage = imageTensor.ToImageMask();
97+
//await maskImage.SaveAsPngAsync("D:\\Hed.png");
98+
_logger.LogInformation($"[HedImage] - HardEdge image generation complete.");
99+
return new InputImage(maskImage);
100+
}
101+
}
102+
}
103+
104+
105+
/// <summary>
106+
/// Generates the depth image mask.
107+
/// </summary>
108+
/// <param name="controlNetModel">The control net model.</param>
109+
/// <param name="inputImage">The input image.</param>
110+
/// <param name="height">The height.</param>
111+
/// <param name="width">The width.</param>
112+
/// <returns></returns>
113+
public async Task<InputImage> DepthImage(FeatureExtractorModelSet controlNetModel, InputImage inputImage, int height, int width)
114+
{
115+
_logger.LogInformation($"[DepthImage] - Generating Depth image...");
116+
var controlImage = await inputImage.ToDenseTensorAsync(height, width, ImageNormalizeType.ZeroToOne);
117+
var metadata = _onnxModelService.GetModelMetadata(controlNetModel, OnnxModelType.Annotation);
118+
using (var inferenceParameters = new OnnxInferenceParameters(metadata))
119+
{
120+
inferenceParameters.AddInputTensor(controlImage);
121+
inferenceParameters.AddOutputBuffer(new[] { 1, 1, height, width });
122+
123+
var results = await _onnxModelService.RunInferenceAsync(controlNetModel, OnnxModelType.Annotation, inferenceParameters);
124+
using (var result = results.First())
125+
{
126+
var testImage = result.ToDenseTensor().Repeat(3);
127+
var imageTensor = new DenseTensor<float>(controlImage.Dimensions);
128+
for (int i = 0; i < testImage.Length; i++)
129+
imageTensor.SetValue(i, testImage.GetValue(i));
130+
131+
NormalizeDepthTensor(imageTensor);
132+
var maskImage = imageTensor.ToImageMask();
133+
//await maskImage.SaveAsPngAsync("D:\\Depth.png");
134+
_logger.LogInformation($"[DepthImage] - Depth image generation complete.");
135+
return new InputImage(maskImage);
136+
}
137+
}
138+
}
139+
140+
141+
/// <summary>
142+
/// Normalizes the depth tensor.
143+
/// </summary>
144+
/// <param name="value">The value.</param>
145+
public static void NormalizeDepthTensor(DenseTensor<float> value)
146+
{
147+
var values = value.Buffer.Span;
148+
float min = float.PositiveInfinity, max = float.NegativeInfinity;
149+
foreach (var val in values)
150+
{
151+
if (min > val) min = val;
152+
if (max < val) max = val;
153+
}
154+
155+
var range = max - min;
156+
for (var i = 0; i < values.Length; i++)
157+
{
158+
values[i] = (values[i] - min) / range;
159+
}
160+
}
161+
}
162+
}

‎OnnxStack.FeatureExtractor/Services/IFeatureExtractorService.cs

+12
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
using OnnxStack.Core.Image;
2+
using System.Threading.Tasks;
3+
4+
namespace OnnxStack.FeatureExtractor.Common
5+
{
6+
public interface IFeatureExtractorService
7+
{
8+
Task<InputImage> CannyImage(FeatureExtractorModelSet controlNetModel, InputImage inputImage, int height, int width);
9+
Task<InputImage> HedImage(FeatureExtractorModelSet controlNetModel, InputImage inputImage, int height, int width);
10+
Task<InputImage> DepthImage(FeatureExtractorModelSet controlNetModel, InputImage inputImage, int height, int width);
11+
}
12+
}

‎OnnxStack.StableDiffusion/Helpers/TensorHelper.cs

-24
Original file line numberDiff line numberDiff line change
@@ -253,31 +253,7 @@ public static DenseTensor<float> Clip(this DenseTensor<float> tensor, float minV
253253

254254

255255

256-
/// <summary>
257-
/// Repeats the specified Tensor along the 0 axis.
258-
/// </summary>
259-
/// <param name="tensor1">The tensor1.</param>
260-
/// <param name="count">The count.</param>
261-
/// <param name="axis">The axis.</param>
262-
/// <returns></returns>
263-
/// <exception cref="System.NotImplementedException">Only axis 0 is supported</exception>
264-
public static DenseTensor<float> Repeat(this DenseTensor<float> tensor1, int count, int axis = 0)
265-
{
266-
if (axis != 0)
267-
throw new NotImplementedException("Only axis 0 is supported");
268-
269-
var dimensions = tensor1.Dimensions.ToArray();
270-
dimensions[0] *= count;
271256

272-
var length = (int)tensor1.Length;
273-
var totalLength = length * count;
274-
var buffer = new float[totalLength].AsMemory();
275-
for (int i = 0; i < count; i++)
276-
{
277-
tensor1.Buffer.CopyTo(buffer[(i * length)..]);
278-
}
279-
return new DenseTensor<float>(buffer, dimensions);
280-
}
281257

282258

283259
/// <summary>

0 commit comments

Comments
 (0)