Skip to content
This repository was archived by the owner on Nov 27, 2024. It is now read-only.

Commit a4fb47c

Browse files
authored
Merge pull request #122 from saddam213/Feature
Feature Extractor Example UI
2 parents 2928d00 + f1728c1 commit a4fb47c

17 files changed

+932
-30
lines changed

OnnxStack.Console/Examples/UpscaleStreamExample.cs

+1-1
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ public UpscaleStreamExample()
1515

1616
public int Index => 10;
1717

18-
public string Name => "Streaming Video Upscale Demo";
18+
public string Name => "Upscale Video Streaming Demo";
1919

2020
public string Description => "Upscales a video stream";
2121

OnnxStack.FeatureExtractor/Pipelines/FeatureExtractorPipeline.cs

+10
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,16 @@ public FeatureExtractorPipeline(string name, FeatureExtractorModel featureExtrac
4141
public string Name => _name;
4242

4343

44+
/// <summary>
45+
/// Loads the model.
46+
/// </summary>
47+
/// <returns></returns>
48+
public Task LoadAsync()
49+
{
50+
return _featureExtractorModel.LoadAsync();
51+
}
52+
53+
4454
/// <summary>
4555
/// Unloads the models.
4656
/// </summary>

OnnxStack.UI/App.xaml.cs

+1-1
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ public App()
4747
builder.Services.AddSingleton<IDeviceService, DeviceService>();
4848
builder.Services.AddSingleton<IStableDiffusionService, StableDiffusionService>();
4949
builder.Services.AddSingleton<IUpscaleService, UpscaleService>();
50-
50+
builder.Services.AddSingleton<IFeatureExtractorService, FeatureExtractorService>();
5151

5252
// Build App
5353
_applicationHost = builder.Build();

OnnxStack.UI/MainWindow.xaml

+13
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,19 @@
8787
<views:UpscaleView UISettings="{Binding UISettings}" Margin="0,6,0,0"/>
8888
</TabItem>
8989

90+
<!--Feature Extractor-->
91+
<TabItem>
92+
<TabItem.Header>
93+
<StackPanel Orientation="Horizontal" Margin="5">
94+
<StackPanel Orientation="Horizontal">
95+
<userControls:FontAwesome Icon="&#xf87c;" IconStyle="Light"/>
96+
</StackPanel>
97+
<TextBlock Text="Feature Extractor" Margin="5,0,0,0"/>
98+
</StackPanel>
99+
</TabItem.Header>
100+
<views:FeatureExtractorView UISettings="{Binding UISettings}" Margin="0,6,0,0"/>
101+
</TabItem>
102+
90103
<!--Log Window-->
91104
<TabItem DockPanel.Dock="Right" HorizontalAlignment="Right">
92105
<TabItem.Header>

OnnxStack.UI/MainWindow.xaml.cs

+22-2
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ public MainWindow(OnnxStackUIConfig uiSettings, ILogger<MainWindow> logger)
2727
UISettings = uiSettings;
2828
SaveImageCommand = new AsyncRelayCommand<UpscaleResult>(SaveUpscaleImageFile);
2929
SaveImageResultCommand = new AsyncRelayCommand<ImageResult>(SaveImageResultFile);
30+
SaveFeatureImageCommand = new AsyncRelayCommand<FeatureExtractorResult>(SaveFeatureImageFile);
3031
SaveBlueprintCommand = new AsyncRelayCommand<ImageResult>(SaveBlueprintFile);
3132
NavigateTextToImageCommand = new AsyncRelayCommand<ImageResult>(NavigateTextToImage);
3233
NavigateImageToImageCommand = new AsyncRelayCommand<ImageResult>(NavigateImageToImage);
@@ -39,14 +40,13 @@ public MainWindow(OnnxStackUIConfig uiSettings, ILogger<MainWindow> logger)
3940
InitializeComponent();
4041
}
4142

42-
43-
4443
public AsyncRelayCommand WindowMinimizeCommand { get; }
4544
public AsyncRelayCommand WindowRestoreCommand { get; }
4645
public AsyncRelayCommand WindowMaximizeCommand { get; }
4746
public AsyncRelayCommand WindowCloseCommand { get; }
4847
public AsyncRelayCommand<UpscaleResult> SaveImageCommand { get; }
4948
public AsyncRelayCommand<ImageResult> SaveImageResultCommand { get; }
49+
public AsyncRelayCommand<FeatureExtractorResult> SaveFeatureImageCommand { get; }
5050
public AsyncRelayCommand<ImageResult> SaveBlueprintCommand { get; }
5151
public AsyncRelayCommand<ImageResult> NavigateTextToImageCommand { get; }
5252
public AsyncRelayCommand<ImageResult> NavigateImageToImageCommand { get; }
@@ -151,6 +151,26 @@ private async Task SaveUpscaleImageFile(UpscaleResult imageResult)
151151
}
152152

153153

154+
private async Task SaveFeatureImageFile(FeatureExtractorResult imageResult)
155+
{
156+
try
157+
{
158+
var filename = GetSaveFilename($"feature-{Random.Shared.Next()}");
159+
if (string.IsNullOrEmpty(filename))
160+
return;
161+
162+
var result = await imageResult.Image.SaveImageFileAsync(filename);
163+
if (!result)
164+
_logger.LogError("Error saving image");
165+
166+
}
167+
catch (Exception ex)
168+
{
169+
_logger.LogError(ex, "Error saving image");
170+
}
171+
}
172+
173+
154174
private async Task SaveBlueprintFile(ImageResult imageResult)
155175
{
156176
try
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
using System.Windows.Media.Imaging;
2+
3+
namespace OnnxStack.UI.Models
4+
{
5+
public record FeatureExtractorResult(BitmapSource Image, double Elapsed);
6+
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
using Microsoft.Extensions.Logging;
2+
using OnnxStack.Core.Config;
3+
using OnnxStack.Core.Image;
4+
using OnnxStack.Core.Video;
5+
using OnnxStack.FeatureExtractor.Common;
6+
using OnnxStack.FeatureExtractor.Pipelines;
7+
using System;
8+
using System.Collections.Generic;
9+
using System.Threading;
10+
using System.Threading.Tasks;
11+
12+
namespace OnnxStack.UI.Services
13+
{
14+
public class FeatureExtractorService : IFeatureExtractorService
15+
{
16+
private readonly ILogger<FeatureExtractorService> _logger;
17+
private readonly Dictionary<IOnnxModel, FeatureExtractorPipeline> _pipelines;
18+
19+
/// <summary>
20+
/// Initializes a new instance of the <see cref="FeatureExtractorService"/> class.
21+
/// </summary>
22+
/// <param name="configuration">The configuration.</param>
23+
/// <param name="modelService">The model service.</param>
24+
/// <param name="imageService">The image service.</param>
25+
public FeatureExtractorService(ILogger<FeatureExtractorService> logger)
26+
{
27+
_logger = logger;
28+
_pipelines = new Dictionary<IOnnxModel, FeatureExtractorPipeline>();
29+
}
30+
31+
32+
/// <summary>
33+
/// Loads the model.
34+
/// </summary>
35+
/// <param name="model">The model.</param>
36+
/// <returns></returns>
37+
public async Task<bool> LoadModelAsync(FeatureExtractorModelSet model)
38+
{
39+
if (_pipelines.ContainsKey(model))
40+
return true;
41+
42+
var pipeline = CreatePipeline(model);
43+
await pipeline.LoadAsync();
44+
return _pipelines.TryAdd(model, pipeline);
45+
}
46+
47+
48+
/// <summary>
49+
/// Unloads the model.
50+
/// </summary>
51+
/// <param name="model">The model.</param>
52+
/// <returns></returns>
53+
public async Task<bool> UnloadModelAsync(FeatureExtractorModelSet model)
54+
{
55+
if (_pipelines.Remove(model, out var pipeline))
56+
{
57+
await pipeline?.UnloadAsync();
58+
}
59+
return true;
60+
}
61+
62+
63+
/// <summary>
64+
/// Determines whether [is model loaded] [the specified model options].
65+
/// </summary>
66+
/// <param name="model">The model.</param>
67+
/// <returns>
68+
/// <c>true</c> if [is model loaded] [the specified model options]; otherwise, <c>false</c>.
69+
/// </returns>
70+
/// <exception cref="System.NotImplementedException"></exception>
71+
public bool IsModelLoaded(FeatureExtractorModelSet model)
72+
{
73+
return _pipelines.ContainsKey(model);
74+
}
75+
76+
77+
/// <summary>
78+
/// Generates the feature image.
79+
/// </summary>
80+
/// <param name="model">The model.</param>
81+
/// <param name="inputImage">The input image.</param>
82+
/// <returns></returns>
83+
public async Task<OnnxImage> GenerateAsync(FeatureExtractorModelSet model, OnnxImage inputImage, CancellationToken cancellationToken = default)
84+
{
85+
if (!_pipelines.TryGetValue(model, out var pipeline))
86+
throw new Exception("Pipeline not found or is unsupported");
87+
88+
return await pipeline.RunAsync(inputImage, cancellationToken);
89+
}
90+
91+
92+
/// <summary>
93+
/// Generates the feature video.
94+
/// </summary>
95+
/// <param name="model">The model.</param>
96+
/// <param name="inputVideo">The input video.</param>
97+
/// <param name="cancellationToken">The cancellation token.</param>
98+
/// <returns></returns>
99+
/// <exception cref="System.Exception">Pipeline not found or is unsupported</exception>
100+
public async Task<OnnxVideo> GenerateAsync(FeatureExtractorModelSet model, OnnxVideo inputVideo, CancellationToken cancellationToken = default)
101+
{
102+
if (!_pipelines.TryGetValue(model, out var pipeline))
103+
throw new Exception("Pipeline not found or is unsupported");
104+
105+
return await pipeline.RunAsync(inputVideo, cancellationToken);
106+
}
107+
108+
109+
/// <summary>
110+
/// Creates the pipeline.
111+
/// </summary>
112+
/// <param name="modelSet">The model.</param>
113+
/// <returns></returns>
114+
private FeatureExtractorPipeline CreatePipeline(FeatureExtractorModelSet model)
115+
{
116+
return FeatureExtractorPipeline.CreatePipeline(model, _logger);
117+
}
118+
}
119+
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
using OnnxStack.Core.Image;
2+
using OnnxStack.Core.Video;
3+
using OnnxStack.FeatureExtractor.Common;
4+
using System.Threading;
5+
using System.Threading.Tasks;
6+
7+
namespace OnnxStack.UI.Services
8+
{
9+
public interface IFeatureExtractorService
10+
{
11+
12+
/// <summary>
13+
/// Loads the model.
14+
/// </summary>
15+
/// <param name="model">The model.</param>
16+
/// <returns></returns>
17+
Task<bool> LoadModelAsync(FeatureExtractorModelSet model);
18+
19+
/// <summary>
20+
/// Unloads the model.
21+
/// </summary>
22+
/// <param name="model">The model.</param>
23+
/// <returns></returns>
24+
Task<bool> UnloadModelAsync(FeatureExtractorModelSet model);
25+
26+
/// <summary>
27+
/// Determines whether [is model loaded] [the specified model options].
28+
/// </summary>
29+
/// <param name="model">The modelset.</param>
30+
/// <returns>
31+
/// <c>true</c> if [is model loaded] [the specified model options]; otherwise, <c>false</c>.
32+
/// </returns>
33+
bool IsModelLoaded(FeatureExtractorModelSet model);
34+
35+
/// <summary>
36+
/// Generates the feature image.
37+
/// </summary>
38+
/// <param name="model">The model.</param>
39+
/// <param name="inputImage">The input image.</param>
40+
/// <returns></returns>
41+
Task<OnnxImage> GenerateAsync(FeatureExtractorModelSet model, OnnxImage inputImage, CancellationToken cancellationToken = default);
42+
43+
/// <summary>
44+
/// Generates the feature video.
45+
/// </summary>
46+
/// <param name="model">The model.</param>
47+
/// <param name="inputVideo">The input video.</param>
48+
/// <returns></returns>
49+
Task<OnnxVideo> GenerateAsync(FeatureExtractorModelSet model, OnnxVideo inputVideo, CancellationToken cancellationToken = default);
50+
}
51+
}

OnnxStack.UI/Services/IUpscaleService.cs

+13-9
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,6 @@
1-
using Microsoft.ML.OnnxRuntime.Tensors;
2-
using OnnxStack.Core.Image;
1+
using OnnxStack.Core.Image;
32
using OnnxStack.Core.Video;
43
using OnnxStack.ImageUpscaler.Common;
5-
using SixLabors.ImageSharp;
6-
using SixLabors.ImageSharp.PixelFormats;
7-
using System.IO;
84
using System.Threading;
95
using System.Threading.Tasks;
106

@@ -30,18 +26,26 @@ public interface IUpscaleService
3026
/// <summary>
3127
/// Determines whether [is model loaded] [the specified model options].
3228
/// </summary>
33-
/// <param name="modelOptions">The model options.</param>
29+
/// <param name="model">The model.</param>
3430
/// <returns>
3531
/// <c>true</c> if [is model loaded] [the specified model options]; otherwise, <c>false</c>.
3632
/// </returns>
37-
bool IsModelLoaded(UpscaleModelSet modelOptions);
33+
bool IsModelLoaded(UpscaleModelSet model);
3834

3935
/// <summary>
4036
/// Generates the upscaled image.
4137
/// </summary>
42-
/// <param name="modelOptions">The model options.</param>
38+
/// <param name="model">The model.</param>
39+
/// <param name="inputImage">The input image.</param>
40+
/// <returns></returns>
41+
Task<OnnxImage> GenerateAsync(UpscaleModelSet model, OnnxImage inputImage, CancellationToken cancellationToken = default);
42+
43+
/// <summary>
44+
/// Generates the upscaled image.
45+
/// </summary>
46+
/// <param name="model">The model.</param>
4347
/// <param name="inputImage">The input image.</param>
4448
/// <returns></returns>
45-
Task<OnnxImage> GenerateAsync(UpscaleModelSet modelOptions, OnnxImage inputImage, CancellationToken cancellationToken = default);
49+
Task<OnnxVideo> GenerateAsync(UpscaleModelSet model, OnnxVideo inputVideo, CancellationToken cancellationToken = default);
4650
}
4751
}

OnnxStack.UI/Services/UpscaleService.cs

+12-12
Original file line numberDiff line numberDiff line change
@@ -63,26 +63,26 @@ public async Task<bool> UnloadModelAsync(UpscaleModelSet model)
6363
/// <summary>
6464
/// Determines whether [is model loaded] [the specified model options].
6565
/// </summary>
66-
/// <param name="modelOptions">The model options.</param>
66+
/// <param name="model">The model.</param>
6767
/// <returns>
6868
/// <c>true</c> if [is model loaded] [the specified model options]; otherwise, <c>false</c>.
6969
/// </returns>
7070
/// <exception cref="System.NotImplementedException"></exception>
71-
public bool IsModelLoaded(UpscaleModelSet modelOptions)
71+
public bool IsModelLoaded(UpscaleModelSet model)
7272
{
73-
return _pipelines.ContainsKey(modelOptions);
73+
return _pipelines.ContainsKey(model);
7474
}
7575

7676

7777
/// <summary>
7878
/// Generates the upscaled image.
7979
/// </summary>
80-
/// <param name="modelSet">The model options.</param>
80+
/// <param name="model">The model.</param>
8181
/// <param name="inputImage">The input image.</param>
8282
/// <returns></returns>
83-
public async Task<OnnxImage> GenerateAsync(UpscaleModelSet modelSet, OnnxImage inputImage, CancellationToken cancellationToken = default)
83+
public async Task<OnnxImage> GenerateAsync(UpscaleModelSet model, OnnxImage inputImage, CancellationToken cancellationToken = default)
8484
{
85-
if (!_pipelines.TryGetValue(modelSet, out var pipeline))
85+
if (!_pipelines.TryGetValue(model, out var pipeline))
8686
throw new Exception("Pipeline not found or is unsupported");
8787

8888
return await pipeline.RunAsync(inputImage, cancellationToken);
@@ -92,14 +92,14 @@ public async Task<OnnxImage> GenerateAsync(UpscaleModelSet modelSet, OnnxImage i
9292
/// <summary>
9393
/// Generates the upscaled video.
9494
/// </summary>
95-
/// <param name="modelSet">The model set.</param>
95+
/// <param name="model">The model.</param>
9696
/// <param name="inputVideo">The input video.</param>
9797
/// <param name="cancellationToken">The cancellation token.</param>
9898
/// <returns></returns>
9999
/// <exception cref="System.Exception">Pipeline not found or is unsupported</exception>
100-
public async Task<OnnxVideo> GenerateAsync(UpscaleModelSet modelSet, OnnxVideo inputVideo, CancellationToken cancellationToken = default)
100+
public async Task<OnnxVideo> GenerateAsync(UpscaleModelSet model, OnnxVideo inputVideo, CancellationToken cancellationToken = default)
101101
{
102-
if (!_pipelines.TryGetValue(modelSet, out var pipeline))
102+
if (!_pipelines.TryGetValue(model, out var pipeline))
103103
throw new Exception("Pipeline not found or is unsupported");
104104

105105
return await pipeline.RunAsync(inputVideo, cancellationToken);
@@ -109,11 +109,11 @@ public async Task<OnnxVideo> GenerateAsync(UpscaleModelSet modelSet, OnnxVideo i
109109
/// <summary>
110110
/// Creates the pipeline.
111111
/// </summary>
112-
/// <param name="modelSet">The model set.</param>
112+
/// <param name="model">The model.</param>
113113
/// <returns></returns>
114-
private ImageUpscalePipeline CreatePipeline(UpscaleModelSet modelSet)
114+
private ImageUpscalePipeline CreatePipeline(UpscaleModelSet model)
115115
{
116-
return ImageUpscalePipeline.CreatePipeline(modelSet, _logger);
116+
return ImageUpscalePipeline.CreatePipeline(model, _logger);
117117
}
118118
}
119119
}

0 commit comments

Comments
 (0)