Merge pull request #120 from saddam213/Stream

Video Stream Support

Author: saddam213

OnnxStack.Console/Examples/FeatureExtractorVideoExample.cs

@@ -0,0 +1,50 @@
+using OnnxStack.Core.Video;
+using OnnxStack.FeatureExtractor.Pipelines;
+using OnnxStack.StableDiffusion.Config;
+
+namespace OnnxStack.Console.Runner
+{
+    public sealed class FeatureExtractorVideoExample : IExampleRunner
+    {
+        private readonly string _outputDirectory;
+        private readonly StableDiffusionConfig _configuration;
+
+        public FeatureExtractorVideoExample(StableDiffusionConfig configuration)
+        {
+            _configuration = configuration;
+            _outputDirectory = Path.Combine(Directory.GetCurrentDirectory(), "Examples", nameof(FeatureExtractorVideoExample));
+            Directory.CreateDirectory(_outputDirectory);
+        }
+
+        public int Index => 13;
+
+        public string Name => "Feature Extractor Video Example";
+
+        public string Description => "Video exmaple using basic feature extractor";
+
+        /// <summary>
+        /// ControlNet Example
+        /// </summary>
+        public async Task RunAsync()
+        {
+            // Read Video
+            var videoFile = "C:\\Users\\Deven\\Pictures\\parrot.mp4";
+            var videoInfo = await VideoHelper.ReadVideoInfoAsync(videoFile);
+
+            // Create pipeline
+            var pipeline = FeatureExtractorPipeline.CreatePipeline("D:\\Repositories\\controlnet_onnx\\annotators\\canny.onnx");
+
+            // Create Video Stream
+            var videoStream = VideoHelper.ReadVideoStreamAsync(videoFile, videoInfo.FrameRate);
+
+            // Create Pipeline Stream
+            var pipelineStream = pipeline.RunAsync(videoStream);
+
+            // Write Video Stream
+            await VideoHelper.WriteVideoStreamAsync(videoInfo, pipelineStream, Path.Combine(_outputDirectory, $"Result.mp4"));
+
+            //Unload
+            await pipeline.UnloadAsync();
+        }
+    }
+}

OnnxStack.Console/Examples/UpscaleExample.cs

@@ -29,13 +29,10 @@ public async Task RunAsync()
 
             // Run pipeline
             var result = await pipeline.RunAsync(inputImage);
-
-            // Create Image from Tensor result
-            var image = new OnnxImage(result, ImageNormalizeType.ZeroToOne);
-
+         
             // Save Image File
             var outputFilename = Path.Combine(_outputDirectory, $"Upscaled.png");
-            await image.SaveAsync(outputFilename);
+            await result.SaveAsync(outputFilename);
 
             // Unload
             await pipeline.UnloadAsync();

OnnxStack.Console/Examples/UpscaleStreamExample.cs

@@ -0,0 +1,48 @@
+using OnnxStack.Core.Video;
+using OnnxStack.FeatureExtractor.Pipelines;
+
+namespace OnnxStack.Console.Runner
+{
+    public sealed class UpscaleStreamExample : IExampleRunner
+    {
+        private readonly string _outputDirectory;
+
+        public UpscaleStreamExample()
+        {
+            _outputDirectory = Path.Combine(Directory.GetCurrentDirectory(), "Examples", nameof(UpscaleStreamExample));
+            Directory.CreateDirectory(_outputDirectory);
+        }
+
+        public int Index => 10;
+
+        public string Name => "Streaming Video Upscale Demo";
+
+        public string Description => "Upscales a video stream";
+
+        public async Task RunAsync()
+        {
+            // Read Video
+            var videoFile = "C:\\Users\\Deven\\Pictures\\parrot.mp4";
+            var videoInfo = await VideoHelper.ReadVideoInfoAsync(videoFile);
+
+            // Create pipeline
+            var pipeline = ImageUpscalePipeline.CreatePipeline("D:\\Repositories\\upscaler\\SwinIR\\003_realSR_BSRGAN_DFO_s64w8_SwinIR-M_x4_GAN.onnx", 4);
+
+            // Load pipeline
+            await pipeline.LoadAsync();
+
+            // Create Video Stream
+            var videoStream = VideoHelper.ReadVideoStreamAsync(videoFile, videoInfo.FrameRate);
+
+            // Create Pipeline Stream
+            var pipelineStream = pipeline.RunAsync(videoStream);
+
+            // Write Video Stream
+            await VideoHelper.WriteVideoStreamAsync(videoInfo, pipelineStream, Path.Combine(_outputDirectory, $"Result.mp4"));
+
+            //Unload
+            await pipeline.UnloadAsync();
+        }
+
+    }
+}

OnnxStack.Console/Examples/VideoToVideoStreamExample.cs

@@ -0,0 +1,67 @@
+using OnnxStack.Core.Video;
+using OnnxStack.FeatureExtractor.Pipelines;
+using OnnxStack.StableDiffusion.Config;
+using OnnxStack.StableDiffusion.Enums;
+using OnnxStack.StableDiffusion.Pipelines;
+
+namespace OnnxStack.Console.Runner
+{
+    public sealed class VideoToVideoStreamExample : IExampleRunner
+    {
+        private readonly string _outputDirectory;
+        private readonly StableDiffusionConfig _configuration;
+
+        public VideoToVideoStreamExample(StableDiffusionConfig configuration)
+        {
+            _configuration = configuration;
+            _outputDirectory = Path.Combine(Directory.GetCurrentDirectory(), "Examples", nameof(VideoToVideoStreamExample));
+            Directory.CreateDirectory(_outputDirectory);
+        }
+
+        public int Index => 4;
+
+        public string Name => "Video To Video Stream Demo";
+
+        public string Description => "Video Stream Stable Diffusion Inference";
+
+        public async Task RunAsync()
+        {
+
+            // Read Video
+            var videoFile = "C:\\Users\\Deven\\Pictures\\gidsgphy.gif";
+            var videoInfo = await VideoHelper.ReadVideoInfoAsync(videoFile);
+
+            // Loop though the appsettings.json model sets
+            foreach (var modelSet in _configuration.ModelSets)
+            {
+                OutputHelpers.WriteConsole($"Loading Model `{modelSet.Name}`...", ConsoleColor.Cyan);
+
+                // Create Pipeline
+                var pipeline = PipelineBase.CreatePipeline(modelSet);
+
+                // Preload Models (optional)
+                await pipeline.LoadAsync();
+
+                // Add text and video to prompt
+                var promptOptions = new PromptOptions
+                {
+                    Prompt = "Iron Man",
+                    DiffuserType = DiffuserType.ImageToImage
+                };
+
+
+                // Create Video Stream
+                var videoStream = VideoHelper.ReadVideoStreamAsync(videoFile, videoInfo.FrameRate);
+
+                // Create Pipeline Stream
+                var pipelineStream = pipeline.GenerateVideoStreamAsync(videoStream, promptOptions, progressCallback:OutputHelpers.ProgressCallback);
+
+                // Write Video Stream
+                await VideoHelper.WriteVideoStreamAsync(videoInfo, pipelineStream, Path.Combine(_outputDirectory, $"{modelSet.PipelineType}.mp4"));
+
+                //Unload
+                await pipeline.UnloadAsync();
+            }
+        }
+    }
+}

OnnxStack.Core/Extensions/TensorExtension.cs

@@ -1,6 +1,7 @@
 using Microsoft.ML.OnnxRuntime.Tensors;
 using System;
 using System.Collections.Generic;
+using System.Linq;
 
 namespace OnnxStack.Core
 {
@@ -75,6 +76,33 @@ public static IEnumerable<DenseTensor<float>> SplitBatch(this DenseTensor<float>
         }
 
 
+        /// <summary>
+        /// Joins the tensors across the 0 axis.
+        /// </summary>
+        /// <param name="tensors">The tensors.</param>
+        /// <param name="axis">The axis.</param>
+        /// <returns></returns>
+        /// <exception cref="System.NotImplementedException">Only axis 0 is supported</exception>
+        public static DenseTensor<float> Join(this IList<DenseTensor<float>> tensors, int axis = 0)
+        {
+            if (axis != 0)
+                throw new NotImplementedException("Only axis 0 is supported");
+
+            var tensor = tensors.First();
+            var dimensions = tensor.Dimensions.ToArray();
+            dimensions[0] *= tensors.Count;
+
+            var newLength = (int)tensor.Length;
+            var buffer = new float[newLength * tensors.Count].AsMemory();
+            for (int i = 0; i < tensors.Count(); i++)
+            {
+                var start = i * newLength;
+                tensors[i].Buffer.CopyTo(buffer[start..]);
+            }
+            return new DenseTensor<float>(buffer, dimensions);
+        }
+
+
         /// <summary>
         /// Concatenates the specified tensors along the specified axis.
         /// </summary>

OnnxStack.Core/Image/OnnxImage.cs

@@ -144,6 +144,20 @@ public byte[] GetImageBytes()
         }
 
 
+        /// <summary>
+        /// Gets the image as bytes.
+        /// </summary>
+        /// <returns></returns>
+        public async Task<byte[]> GetImageBytesAsync()
+        {
+            using (var memoryStream = new MemoryStream())
+            {
+                await _imageData.SaveAsPngAsync(memoryStream);
+                return memoryStream.ToArray();
+            }
+        }
+
+
         /// <summary>
         /// Gets the image as stream.
         /// </summary>
@@ -156,6 +170,40 @@ public Stream GetImageStream()
         }
 
 
+        /// <summary>
+        /// Gets the image as stream.
+        /// </summary>
+        /// <returns></returns>
+        public async Task<Stream> GetImageStreamAsync()
+        {
+            var memoryStream = new MemoryStream();
+            await _imageData.SaveAsPngAsync(memoryStream);
+            return memoryStream;
+        }
+
+
+        /// <summary>
+        /// Copies the image to stream.
+        /// </summary>
+        /// <param name="destination">The destination.</param>
+        /// <returns></returns>
+        public void CopyToStream(Stream destination)
+        {
+            _imageData.SaveAsPng(destination);
+        }
+
+
+        /// <summary>
+        /// Copies the image to stream.
+        /// </summary>
+        /// <param name="destination">The destination.</param>
+        /// <returns></returns>
+        public Task CopyToStreamAsync(Stream destination)
+        {
+            return _imageData.SaveAsPngAsync(destination);
+        }
+
+
         /// <summary>
         /// Gets the image as tensor.
         /// </summary>

OnnxStack.Core/Video/OnnxVideo.cs

@@ -88,7 +88,7 @@ public OnnxVideo(VideoInfo info, IEnumerable<DenseTensor<float>> videoTensors)
         /// <summary>
         /// Gets the aspect ratio.
         /// </summary>
-        public double AspectRatio => (double)_info.Width / _info.Height;
+        public double AspectRatio => _info.AspectRatio;
 
         /// <summary>
         /// Gets a value indicating whether this instance has video.