VideoToVideo Example Runners And StableDiffusionService GenerateInputVideoFrames Bug Fix

OnnxStack.Console/Examples/VideoToVideoExample.cs

@@ -0,0 +1,74 @@
+using OnnxStack.Console;
+using OnnxStack.Core.Image;
+using OnnxStack.StableDiffusion.Common;
+using OnnxStack.StableDiffusion.Config;
+using OnnxStack.StableDiffusion.Enums;
+using SixLabors.ImageSharp.PixelFormats;
+using SixLabors.ImageSharp;
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Text;
+using OnnxStack.Core.Video;
+using OnnxStack.Core.Services;
+
+namespace OnnxStack.Console.Runner
+{
+    public sealed class VideoToVideoExample : IExampleRunner
+    {
+        private readonly string _outputDirectory;
+        private readonly StableDiffusionConfig _configuration;
+        private readonly IStableDiffusionService _stableDiffusionService;
+        private readonly IVideoService _videoService;
+
+        public VideoToVideoExample(StableDiffusionConfig configuration, IStableDiffusionService stableDiffusionService, IVideoService videoService)
+        {
+            _configuration = configuration;
+            _stableDiffusionService = stableDiffusionService;
+            _videoService = videoService;
+            _outputDirectory = Path.Combine(Directory.GetCurrentDirectory(), "Examples", nameof(UpscaleExample));
+            Directory.CreateDirectory(_outputDirectory);
+        }
+
+        public string Name => "Video To Video Demo";
+
+        public string Description => "Vidio Stable Diffusion Inference";
+
+        public async Task RunAsync()
+        {
+            var model = _configuration.ModelSets.FirstOrDefault(x => x.Name == "LCM-Dreamshaper-V7");
+            OutputHelpers.WriteConsole("Loading Model...", ConsoleColor.Cyan);
+            await _stableDiffusionService.LoadModelAsync(model);
+            OutputHelpers.WriteConsole("Model Loaded.", ConsoleColor.Cyan);
+            string inputVideoPath = "C:\\Users\\Hex\\Downloads\\doit.mp4";
+            string outputVideoPath = "C:\\Users\\Hex\\Downloads\\doitdiffused.mp4";
+
+
+            var prompt = "Iron Man";
+            var negativePrompt = "";
+
+            var inputVideo = File.ReadAllBytes(inputVideoPath);
+
+            var promptOptions = new PromptOptions
+            {
+                Prompt = prompt,
+                NegativePrompt = negativePrompt,
+                DiffuserType = DiffuserType.ImageToImage,
+                InputVideo = new VideoInput(inputVideo)
+            };
+
+            var schedulerOptions = new SchedulerOptions
+            {
+                SchedulerType = SchedulerType.LCM,
+                GuidanceScale = 1f,
+                InferenceSteps = 10,
+                Strength = 0.35f,
+                Height = 512,
+                Width = 512
+            };
+
+            var result = await _stableDiffusionService.GenerateAsBytesAsync(new ModelOptions(model), promptOptions, schedulerOptions);
+            File.WriteAllBytes(outputVideoPath, result);
+        }
+    }
+}

OnnxStack.Console/Examples/VideoToVideoExampleFromScratch.cs

@@ -0,0 +1,136 @@
+using OnnxStack.Core.Image;
+using OnnxStack.ImageUpscaler.Config;
+using OnnxStack.ImageUpscaler.Services;
+using OnnxStack.StableDiffusion.Common;
+using OnnxStack.StableDiffusion.Config;
+using SixLabors.ImageSharp.PixelFormats;
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Text;
+using SixLabors.ImageSharp;
+using SixLabors.ImageSharp.Processing;
+using OnnxStack.StableDiffusion.Enums;
+
+namespace OnnxStack.Console.Runner
+{
+    public sealed class VideoToVideoExampleFromScratch : IExampleRunner
+    {
+        private readonly string _outputDirectory;
+        private readonly StableDiffusionConfig _configuration;
+        private readonly IStableDiffusionService _stableDiffusionService;
+
+        public VideoToVideoExampleFromScratch(StableDiffusionConfig configuration, IStableDiffusionService stableDiffusionService)
+        {
+            _configuration = configuration;
+            _stableDiffusionService = stableDiffusionService;
+            _outputDirectory = Path.Combine(Directory.GetCurrentDirectory(), "Examples", nameof(UpscaleExample));
+            Directory.CreateDirectory(_outputDirectory);
+        }
+
+        public string Name => "Video To Video Demo From Scratch";
+
+        public string Description => "Vidio Stable Diffusion Inference From Scratch";
+
+        public async Task RunAsync()
+        {
+            var model = _configuration.ModelSets.FirstOrDefault(x => x.Name == "LCM-Dreamshaper-V7");
+            OutputHelpers.WriteConsole("Loading Model...", ConsoleColor.Cyan);
+            await _stableDiffusionService.LoadModelAsync(model);
+            OutputHelpers.WriteConsole("Model Loaded.", ConsoleColor.Cyan);
+            string inputVideoPath = "C:\\Users\\Hex\\Downloads\\doit.mp4";
+            string outputFramesPath = "C:\\Users\\Hex\\Desktop\\frames\\frame_%04d.png";
+            string ffmpegCommand = $"-i \"{inputVideoPath}\" -vf fps=30 -c:v png -y \"{outputFramesPath}\"";
+            string ffmpeg = @"C:\Users\Hex\Desktop\OnnxStack\ffmpeg.exe";
+
+            Process process = new Process
+            {
+                StartInfo = new ProcessStartInfo
+                {
+                    FileName = ffmpeg,
+                    Arguments = ffmpegCommand,
+                    RedirectStandardOutput = true,
+                    RedirectStandardError = true,
+                    UseShellExecute = false,
+                    CreateNoWindow = true
+                }
+            };
+
+            process.OutputDataReceived += (sender, e) => OutputHelpers.WriteConsole(e.Data, ConsoleColor.Cyan);
+            process.ErrorDataReceived += (sender, e) => OutputHelpers.WriteConsole(e.Data, ConsoleColor.Cyan);
+
+            process.Start();
+
+            process.BeginOutputReadLine();
+            process.BeginErrorReadLine();
+
+            process.WaitForExit();
+
+            outputFramesPath = outputFramesPath.Replace("\\frame_%04d.png", "");
+
+            string[] files = Directory.GetFiles(outputFramesPath);
+
+            var prompt = "Iron Man";
+            var negativePrompt = "";
+
+            var promptOptions = new PromptOptions
+            {
+                Prompt = prompt,
+                NegativePrompt = negativePrompt,
+                DiffuserType = DiffuserType.ImageToImage
+            };
+
+            var schedulerOptions = new SchedulerOptions
+            {
+                SchedulerType = SchedulerType.LCM,
+                GuidanceScale = 1f,
+                InferenceSteps = 10,
+                Strength = 0.35f,
+                Height = 512,
+                Width = 512
+            };
+
+            foreach (string filePath in files)
+            {
+                OutputHelpers.WriteConsole($"Defusing {filePath}", ConsoleColor.Cyan);
+                Image<Rgba32> frameDestination = new(schedulerOptions.Width, schedulerOptions.Height);
+                var frameSource = await Image.LoadAsync(filePath);
+                frameSource.Mutate(x => x.Resize(frameDestination.Size));
+                promptOptions.InputImage = new InputImage(frameSource.CloneAs<Rgba32>());
+                frameDestination = await _stableDiffusionService.GenerateAsImageAsync(new ModelOptions(model), promptOptions, schedulerOptions);
+                await frameDestination.SaveAsPngAsync(filePath);
+                OutputHelpers.WriteConsole($"{filePath} saved", ConsoleColor.Cyan);
+            }
+
+            string outputVideoPath = "C:\\Users\\Hex\\Downloads\\doitdefused.mp4";
+
+            ffmpegCommand = $"ffmpeg -framerate 30 -i \"{outputFramesPath}\\frame_%04d.png\" -c:v libx264 -pix_fmt yuv420p -y \"{outputVideoPath}\"";
+
+            // Create a process to run the FFmpeg command
+            process = new Process
+            {
+                StartInfo = new ProcessStartInfo
+                {
+                    FileName = ffmpeg,
+                    Arguments = ffmpegCommand,
+                    RedirectStandardOutput = true,
+                    RedirectStandardError = true,
+                    UseShellExecute = false,
+                    CreateNoWindow = true
+                }
+            };
+
+            process.OutputDataReceived += (sender, e) => OutputHelpers.WriteConsole(e.Data, ConsoleColor.Cyan);
+            process.ErrorDataReceived += (sender, e) => OutputHelpers.WriteConsole(e.Data, ConsoleColor.Cyan);
+
+            // Start the process
+            process.Start();
+
+            process.BeginOutputReadLine();
+            process.BeginErrorReadLine();
+
+            // Wait for the process to exit
+            process.WaitForExit();
+        }
+    }
+}

OnnxStack.StableDiffusion/Services/StableDiffusionService.cs

@@ -333,12 +333,19 @@ private async Task GenerateInputVideoFrames(PromptOptions promptOptions, Action<
             if (!promptOptions.HasInputVideo || promptOptions.InputVideo.VideoFrames is not null)
                 return;
 
-            // Already has VideoFrames
-            if (promptOptions.InputVideo.VideoFrames is not null)
-                return;
+            if (promptOptions.VideoInputFPS == 0 || promptOptions.VideoOutputFPS == 0)
+            {
+                var videoInfo = await _videoService.GetVideoInfoAsync(promptOptions.InputVideo);
+                if (promptOptions.VideoInputFPS == 0)
+                    promptOptions.VideoInputFPS = videoInfo.FPS;
+
+                if (promptOptions.VideoOutputFPS == 0)
+                    promptOptions.VideoOutputFPS = videoInfo.FPS;
+            }
 
+            var videoFrame = await _videoService.CreateFramesAsync(promptOptions.InputVideo, promptOptions.VideoInputFPS);
             progress?.Invoke(new DiffusionProgress($"Generating video frames @ {promptOptions.VideoInputFPS}fps"));
-            promptOptions.InputVideo.VideoFrames = await _videoService.CreateFramesAsync(promptOptions.InputVideo, promptOptions.VideoInputFPS);
+            promptOptions.InputVideo.VideoFrames = videoFrame;
         }
     }
 }