Video Support

OnnxStack.Console/Examples/StableDiffusionBatch.cs

@@ -1,7 +1,9 @@
-using OnnxStack.StableDiffusion.Common;
+using OnnxStack.Core.Image;
+using OnnxStack.StableDiffusion.Common;
 using OnnxStack.StableDiffusion.Config;
 using OnnxStack.StableDiffusion.Enums;
 using OnnxStack.StableDiffusion.Helpers;
+using OnnxStack.StableDiffusion.Models;
 using SixLabors.ImageSharp;
 
 namespace OnnxStack.Console.Runner
@@ -58,13 +60,13 @@ public async Task RunAsync()
                     await _stableDiffusionService.LoadModelAsync(model);
 
                     var batchIndex = 0;
-                    var callback = (int batch, int batchCount, int step, int steps) =>
+                    var callback = (DiffusionProgress progress) =>
                     {
-                        batchIndex = batch;
-                        OutputHelpers.WriteConsole($"Image: {batch}/{batchCount} - Step: {step}/{steps}", ConsoleColor.Cyan);
+                        batchIndex = progress.BatchValue;
+                        OutputHelpers.WriteConsole($"Image: {progress.BatchValue}/{progress.BatchMax} - Step: {progress.StepValue}/{progress.StepMax}", ConsoleColor.Cyan);
                     };
 
-                    await foreach (var result in _stableDiffusionService.GenerateBatchAsync(model, promptOptions, schedulerOptions, batchOptions, callback))
+                    await foreach (var result in _stableDiffusionService.GenerateBatchAsync(model, promptOptions, schedulerOptions, batchOptions, default))
                     {
                         var outputFilename = Path.Combine(_outputDirectory, $"{batchIndex}_{result.SchedulerOptions.Seed}.png");
                         var image = result.ImageResult.ToImage();

OnnxStack.Console/OnnxStack.Console.csproj

@@ -6,7 +6,7 @@
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>disable</Nullable>
     <PlatformTarget>x64</PlatformTarget>
-    <Configurations>Debug;Release;Debug-DirectML;Debug-Cuda;Debug-TensorRT;Release-DirectML;Release-Cuda;Release-TensorRT</Configurations>
+    <Configurations>Debug;Release;Debug-Cuda;Debug-TensorRT;Release-Cuda;Release-TensorRT</Configurations>
   </PropertyGroup>
 
   <ItemGroup>
@@ -15,17 +15,16 @@
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="OnnxStack.StableDiffusion" Version="0.14.0" Condition=" '$(Configuration)' == 'Release' OR '$(Configuration)' == 'Release-DirectML' OR '$(Configuration)' == 'Release-Cuda' OR '$(Configuration)' == 'Release-TensorRT'" />
-    <PackageReference Include="OnnxStack.ImageUpscaler" Version="0.14.0" Condition=" '$(Configuration)' == 'Release' OR '$(Configuration)' == 'Release-DirectML' OR '$(Configuration)' == 'Release-Cuda' OR '$(Configuration)' == 'Release-TensorRT'" />
-    <ProjectReference Include="..\OnnxStack.StableDiffusion\OnnxStack.StableDiffusion.csproj" Condition=" '$(Configuration)' == 'Debug' OR '$(Configuration)' == 'Debug-DirectML' OR '$(Configuration)' == 'Debug-Cuda' OR '$(Configuration)' == 'Debug-TensorRT'" />
-    <ProjectReference Include="..\OnnxStack.ImageUpscaler\OnnxStack.ImageUpscaler.csproj" Condition=" '$(Configuration)' == 'Debug' OR '$(Configuration)' == 'Debug-DirectML' OR '$(Configuration)' == 'Debug-Cuda' OR '$(Configuration)' == 'Debug-TensorRT'" />
+    <PackageReference Include="OnnxStack.StableDiffusion" Version="0.14.0" Condition=" '$(Configuration)' == 'Release' OR '$(Configuration)' == 'Release-Cuda' OR '$(Configuration)' == 'Release-TensorRT'" />
+    <PackageReference Include="OnnxStack.ImageUpscaler" Version="0.14.0" Condition=" '$(Configuration)' == 'Release' OR '$(Configuration)' == 'Release-Cuda' OR '$(Configuration)' == 'Release-TensorRT'" />
+    <ProjectReference Include="..\OnnxStack.StableDiffusion\OnnxStack.StableDiffusion.csproj" Condition=" '$(Configuration)' == 'Debug' OR '$(Configuration)' == 'Debug-Cuda' OR '$(Configuration)' == 'Debug-TensorRT'" />
+    <ProjectReference Include="..\OnnxStack.ImageUpscaler\OnnxStack.ImageUpscaler.csproj" Condition=" '$(Configuration)' == 'Debug' OR '$(Configuration)' == 'Debug-Cuda' OR '$(Configuration)' == 'Debug-TensorRT'" />
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.ML.OnnxRuntime" Version="1.16.3" Condition=" '$(Configuration)' == 'Debug' OR '$(Configuration)' == 'Release' " />
+    <PackageReference Include="Microsoft.ML.OnnxRuntime.DirectML" Version="1.16.3" Condition=" '$(Configuration)' == 'Debug' OR '$(Configuration)' == 'Release' " />
     <PackageReference Include="Microsoft.ML.OnnxRuntime.Gpu" Version="1.16.3" Condition=" '$(Configuration)' == 'Debug-TensorRT' OR '$(Configuration)' == 'Release-TensorRT'" />
     <PackageReference Include="Microsoft.ML.OnnxRuntime.Gpu" Version="1.16.3" Condition=" '$(Configuration)' == 'Debug-Cuda' OR '$(Configuration)' == 'Release-Cuda'" />
-    <PackageReference Include="Microsoft.ML.OnnxRuntime.DirectML" Version="1.16.3" Condition=" '$(Configuration)' == 'Debug-DirectML' OR '$(Configuration)' == 'Release-DirectML'" />
   </ItemGroup>
 
   <ItemGroup>

OnnxStack.Core/Config/OnnxStackConfig.cs

@@ -4,6 +4,10 @@ namespace OnnxStack.Core.Config
 {
     public class OnnxStackConfig : IConfigSection
     {
+        public string TempPath { get; set; } = ".temp";
+        public string FFmpegPath { get; set; } = "ffmpeg.exe";
+        public string FFprobePath { get; set; } = "ffprobe.exe";
+
         public void Initialize()
         {
         }

OnnxStack.Core/Image/Extensions.cs

@@ -0,0 +1,73 @@
+using Microsoft.ML.OnnxRuntime.Tensors;
+using SixLabors.ImageSharp.PixelFormats;
+using SixLabors.ImageSharp;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using System.IO;
+
+namespace OnnxStack.Core.Image
+{
+    public static class Extensions
+    {
+        public static Image<Rgba32> ToImage(this DenseTensor<float> imageTensor)
+        {
+            var height = imageTensor.Dimensions[2];
+            var width = imageTensor.Dimensions[3];
+            var hasAlpha = imageTensor.Dimensions[1] == 4;
+            var result = new Image<Rgba32>(width, height);
+            for (var y = 0; y < height; y++)
+            {
+                for (var x = 0; x < width; x++)
+                {
+                    result[x, y] = new Rgba32(
+                        CalculateByte(imageTensor, 0, y, x),
+                        CalculateByte(imageTensor, 1, y, x),
+                        CalculateByte(imageTensor, 2, y, x),
+                        hasAlpha ? CalculateByte(imageTensor, 3, y, x) : byte.MaxValue
+                    );
+                }
+            }
+            return result;
+        }
+
+        /// <summary>
+        /// Converts to image byte array.
+        /// </summary>
+        /// <param name="imageTensor">The image tensor.</param>
+        /// <returns></returns>
+        public static byte[] ToImageBytes(this DenseTensor<float> imageTensor)
+        {
+            using (var image = imageTensor.ToImage())
+            using (var memoryStream = new MemoryStream())
+            {
+                image.SaveAsPng(memoryStream);
+                return memoryStream.ToArray();
+            }
+        }
+
+        /// <summary>
+        /// Converts to image byte array.
+        /// </summary>
+        /// <param name="imageTensor">The image tensor.</param>
+        /// <returns></returns>
+        public static async Task<byte[]> ToImageBytesAsync(this DenseTensor<float> imageTensor)
+        {
+            using (var image = imageTensor.ToImage())
+            using (var memoryStream = new MemoryStream())
+            {
+                await image.SaveAsPngAsync(memoryStream);
+                return memoryStream.ToArray();
+            }
+        }
+
+
+        private static byte CalculateByte(Tensor<float> imageTensor, int index, int y, int x)
+        {
+            return (byte)Math.Round(Math.Clamp(imageTensor[0, index, y, x] / 2 + 0.5, 0, 1) * 255);
+        }
+
+    }
+}

OnnxStack.Core/OnnxStack.Core.csproj

@@ -35,13 +35,15 @@
   </ItemGroup>
 
   <ItemGroup>
+    <PackageReference Include="FFMpegCore" Version="5.1.0" />
     <PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="8.0.0" />
     <PackageReference Include="Microsoft.Extensions.Hosting.Abstractions" Version="8.0.0" />
     <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="8.0.0" />
     <PackageReference Include="Microsoft.ML" Version="3.0.0" />
     <PackageReference Include="Microsoft.ML.OnnxRuntime.Extensions" Version="0.9.0" />
     <PackageReference Include="Microsoft.ML.OnnxRuntime.Managed" Version="1.16.3" />
     <PackageReference Include="SixLabors.ImageSharp" Version="3.1.1" />
+    <PackageReference Include="System.Linq.Async" Version="6.0.1" />
   </ItemGroup>
 
   <ItemGroup>

OnnxStack.Core/Services/IVideoService.cs

@@ -0,0 +1,109 @@
+using Microsoft.ML.OnnxRuntime.Tensors;
+using OnnxStack.Core.Video;
+using System.Collections.Generic;
+using System.IO;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace OnnxStack.Core.Services
+{
+    /// <summary>
+    /// Service with basic handling of video for use in OnnxStack, Frame->Video and Video->Frames
+    /// </summary>
+    public interface IVideoService
+    {
+        /// <summary>
+        /// Gets the video information asynchronous.
+        /// </summary>
+        /// <param name="videoBytes">The video bytes.</param>
+        /// <param name="cancellationToken">The cancellation token.</param>
+        /// <returns></returns>
+        Task<VideoInfo> GetVideoInfoAsync(byte[] videoBytes, CancellationToken cancellationToken = default);
+
+        /// <summary>
+        /// Gets the video information asynchronous.
+        /// </summary>
+        /// <param name="videoStream">The video stream.</param>
+        /// <param name="cancellationToken">The cancellation token.</param>
+        /// <returns></returns>
+        Task<VideoInfo> GetVideoInfoAsync(Stream videoStream, CancellationToken cancellationToken = default);
+
+        /// <summary>
+        /// Gets the video information, Size, FPS, Duration etc.
+        /// </summary>
+        /// <param name="videoInput">The video input.</param>
+        /// <param name="cancellationToken">The cancellation token.</param>
+        /// <returns></returns>
+        /// <exception cref="ArgumentException">No video data found</exception>
+        Task<VideoInfo> GetVideoInfoAsync(VideoInput videoInput, CancellationToken cancellationToken = default);
+
+
+        /// <summary>
+        /// Creates a collection of PNG frames from a video source
+        /// </summary>
+        /// <param name="videoBytes">The video bytes.</param>
+        /// <param name="videoFPS">The video FPS.</param>
+        /// <param name="cancellationToken">The cancellation token.</param>
+        /// <returns></returns>
+        Task<VideoFrames> CreateFramesAsync(byte[] videoBytes, float videoFPS, CancellationToken cancellationToken = default);
+
+
+        /// <summary>
+        /// Creates a collection of PNG frames from a video source
+        /// </summary>
+        /// <param name="videoStream">The video stream.</param>
+        /// <param name="videoFPS">The video FPS.</param>
+        /// <param name="cancellationToken">The cancellation token.</param>
+        /// <returns></returns>
+        Task<VideoFrames> CreateFramesAsync(Stream videoStream, float videoFPS, CancellationToken cancellationToken = default);
+
+
+        /// <summary>
+        /// Creates a collection of PNG frames from a video source
+        /// </summary>
+        /// <param name="videoInput">The video input.</param>
+        /// <param name="videoFPS">The video FPS.</param>
+        /// <param name="cancellationToken">The cancellation token.</param>
+        /// <returns></returns>
+        /// <exception cref="NotSupportedException">VideoTensor not supported</exception>
+        /// <exception cref="ArgumentException">No video data found</exception>
+        Task<VideoFrames> CreateFramesAsync(VideoInput videoInput, float videoFPS, CancellationToken cancellationToken = default);
+
+
+        /// <summary>
+        /// Creates and MP4 video from a collection of PNG images.
+        /// </summary>
+        /// <param name="videoFrames">The video frames.</param>
+        /// <param name="videoFPS">The video FPS.</param>
+        /// <param name="cancellationToken">The cancellation token.</param>
+        /// <returns></returns>
+        Task<VideoOutput> CreateVideoAsync(IEnumerable<byte[]> videoFrames, float videoFPS, CancellationToken cancellationToken = default);
+
+
+        /// <summary>
+        /// Creates and MP4 video from a collection of PNG images.
+        /// </summary>
+        /// <param name="videoFrames">The video frames.</param>
+        /// <param name="cancellationToken">The cancellation token.</param>
+        /// <returns></returns>
+        Task<VideoOutput> CreateVideoAsync(VideoFrames videoFrames, CancellationToken cancellationToken = default);
+
+        // <summary>
+        /// Creates and MP4 video from a collection of PNG images.
+        /// </summary>
+        /// <param name="videoTensor">The video frames.</param>
+        /// <param name="videoFPS">The video FPS.</param>
+        /// <param name="cancellationToken">The cancellation token.</param>
+        /// <returns></returns>
+        Task<VideoOutput> CreateVideoAsync(DenseTensor<float> videoTensor, float videoFPS, CancellationToken cancellationToken = default);
+
+        /// <summary>
+        /// Streams frames as PNG as they are processed from a video source
+        /// </summary>
+        /// <param name="videoBytes">The video bytes.</param>
+        /// <param name="targetFPS">The target FPS.</param>
+        /// <param name="cancellationToken">The cancellation token.</param>
+        /// <returns></returns>
+        IAsyncEnumerable<byte[]> StreamFramesAsync(byte[] videoBytes, float targetFPS, CancellationToken cancellationToken = default);
+    }
+}