From 45a39fafc477a85a6da9cef3bd08673c6f49f13a Mon Sep 17 00:00:00 2001 From: TheyCallMeHex Date: Mon, 22 Jan 2024 13:23:44 +1300 Subject: [PATCH 1/2] If promptOptions.VideoInputFPS and promptOptions.VideoOutputFPS are not set, set them to the values of the InputVIdeo Removed bug checking promptOptions.InputVideo.VideoFrames is not null twice --- .../Services/StableDiffusionService.cs | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/OnnxStack.StableDiffusion/Services/StableDiffusionService.cs b/OnnxStack.StableDiffusion/Services/StableDiffusionService.cs index 50c495ab..80587324 100644 --- a/OnnxStack.StableDiffusion/Services/StableDiffusionService.cs +++ b/OnnxStack.StableDiffusion/Services/StableDiffusionService.cs @@ -333,12 +333,19 @@ private async Task GenerateInputVideoFrames(PromptOptions promptOptions, Action< if (!promptOptions.HasInputVideo || promptOptions.InputVideo.VideoFrames is not null) return; - // Already has VideoFrames - if (promptOptions.InputVideo.VideoFrames is not null) - return; + if (promptOptions.VideoInputFPS == 0 || promptOptions.VideoOutputFPS == 0) + { + var videoInfo = await _videoService.GetVideoInfoAsync(promptOptions.InputVideo); + if (promptOptions.VideoInputFPS == 0) + promptOptions.VideoInputFPS = videoInfo.FPS; + + if (promptOptions.VideoOutputFPS == 0) + promptOptions.VideoOutputFPS = videoInfo.FPS; + } + var videoFrame = await _videoService.CreateFramesAsync(promptOptions.InputVideo, promptOptions.VideoInputFPS); progress?.Invoke(new DiffusionProgress($"Generating video frames @ {promptOptions.VideoInputFPS}fps")); - promptOptions.InputVideo.VideoFrames = await _videoService.CreateFramesAsync(promptOptions.InputVideo, promptOptions.VideoInputFPS); + promptOptions.InputVideo.VideoFrames = videoFrame; } } } From 798d87e898d8033434800d0d9ada0307fc5c2e2b Mon Sep 17 00:00:00 2001 From: TheyCallMeHex Date: Mon, 22 Jan 2024 13:24:40 +1300 Subject: [PATCH 2/2] ExampleRunners for VideoToVideo demos, one from scratch, one using the video service. --- .../Examples/VideoToVideoExample.cs | 74 ++++++++++ .../VideoToVideoExampleFromScratch.cs | 136 ++++++++++++++++++ 2 files changed, 210 insertions(+) create mode 100644 OnnxStack.Console/Examples/VideoToVideoExample.cs create mode 100644 OnnxStack.Console/Examples/VideoToVideoExampleFromScratch.cs diff --git a/OnnxStack.Console/Examples/VideoToVideoExample.cs b/OnnxStack.Console/Examples/VideoToVideoExample.cs new file mode 100644 index 00000000..e1b31b3d --- /dev/null +++ b/OnnxStack.Console/Examples/VideoToVideoExample.cs @@ -0,0 +1,74 @@ +using OnnxStack.Console; +using OnnxStack.Core.Image; +using OnnxStack.StableDiffusion.Common; +using OnnxStack.StableDiffusion.Config; +using OnnxStack.StableDiffusion.Enums; +using SixLabors.ImageSharp.PixelFormats; +using SixLabors.ImageSharp; +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Text; +using OnnxStack.Core.Video; +using OnnxStack.Core.Services; + +namespace OnnxStack.Console.Runner +{ + public sealed class VideoToVideoExample : IExampleRunner + { + private readonly string _outputDirectory; + private readonly StableDiffusionConfig _configuration; + private readonly IStableDiffusionService _stableDiffusionService; + private readonly IVideoService _videoService; + + public VideoToVideoExample(StableDiffusionConfig configuration, IStableDiffusionService stableDiffusionService, IVideoService videoService) + { + _configuration = configuration; + _stableDiffusionService = stableDiffusionService; + _videoService = videoService; + _outputDirectory = Path.Combine(Directory.GetCurrentDirectory(), "Examples", nameof(UpscaleExample)); + Directory.CreateDirectory(_outputDirectory); + } + + public string Name => "Video To Video Demo"; + + public string Description => "Vidio Stable Diffusion Inference"; + + public async Task RunAsync() + { + var model = _configuration.ModelSets.FirstOrDefault(x => x.Name == "LCM-Dreamshaper-V7"); + OutputHelpers.WriteConsole("Loading Model...", ConsoleColor.Cyan); + await _stableDiffusionService.LoadModelAsync(model); + OutputHelpers.WriteConsole("Model Loaded.", ConsoleColor.Cyan); + string inputVideoPath = "C:\\Users\\Hex\\Downloads\\doit.mp4"; + string outputVideoPath = "C:\\Users\\Hex\\Downloads\\doitdiffused.mp4"; + + + var prompt = "Iron Man"; + var negativePrompt = ""; + + var inputVideo = File.ReadAllBytes(inputVideoPath); + + var promptOptions = new PromptOptions + { + Prompt = prompt, + NegativePrompt = negativePrompt, + DiffuserType = DiffuserType.ImageToImage, + InputVideo = new VideoInput(inputVideo) + }; + + var schedulerOptions = new SchedulerOptions + { + SchedulerType = SchedulerType.LCM, + GuidanceScale = 1f, + InferenceSteps = 10, + Strength = 0.35f, + Height = 512, + Width = 512 + }; + + var result = await _stableDiffusionService.GenerateAsBytesAsync(new ModelOptions(model), promptOptions, schedulerOptions); + File.WriteAllBytes(outputVideoPath, result); + } + } +} diff --git a/OnnxStack.Console/Examples/VideoToVideoExampleFromScratch.cs b/OnnxStack.Console/Examples/VideoToVideoExampleFromScratch.cs new file mode 100644 index 00000000..790d13a0 --- /dev/null +++ b/OnnxStack.Console/Examples/VideoToVideoExampleFromScratch.cs @@ -0,0 +1,136 @@ +using OnnxStack.Core.Image; +using OnnxStack.ImageUpscaler.Config; +using OnnxStack.ImageUpscaler.Services; +using OnnxStack.StableDiffusion.Common; +using OnnxStack.StableDiffusion.Config; +using SixLabors.ImageSharp.PixelFormats; +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Text; +using SixLabors.ImageSharp; +using SixLabors.ImageSharp.Processing; +using OnnxStack.StableDiffusion.Enums; + +namespace OnnxStack.Console.Runner +{ + public sealed class VideoToVideoExampleFromScratch : IExampleRunner + { + private readonly string _outputDirectory; + private readonly StableDiffusionConfig _configuration; + private readonly IStableDiffusionService _stableDiffusionService; + + public VideoToVideoExampleFromScratch(StableDiffusionConfig configuration, IStableDiffusionService stableDiffusionService) + { + _configuration = configuration; + _stableDiffusionService = stableDiffusionService; + _outputDirectory = Path.Combine(Directory.GetCurrentDirectory(), "Examples", nameof(UpscaleExample)); + Directory.CreateDirectory(_outputDirectory); + } + + public string Name => "Video To Video Demo From Scratch"; + + public string Description => "Vidio Stable Diffusion Inference From Scratch"; + + public async Task RunAsync() + { + var model = _configuration.ModelSets.FirstOrDefault(x => x.Name == "LCM-Dreamshaper-V7"); + OutputHelpers.WriteConsole("Loading Model...", ConsoleColor.Cyan); + await _stableDiffusionService.LoadModelAsync(model); + OutputHelpers.WriteConsole("Model Loaded.", ConsoleColor.Cyan); + string inputVideoPath = "C:\\Users\\Hex\\Downloads\\doit.mp4"; + string outputFramesPath = "C:\\Users\\Hex\\Desktop\\frames\\frame_%04d.png"; + string ffmpegCommand = $"-i \"{inputVideoPath}\" -vf fps=30 -c:v png -y \"{outputFramesPath}\""; + string ffmpeg = @"C:\Users\Hex\Desktop\OnnxStack\ffmpeg.exe"; + + Process process = new Process + { + StartInfo = new ProcessStartInfo + { + FileName = ffmpeg, + Arguments = ffmpegCommand, + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true + } + }; + + process.OutputDataReceived += (sender, e) => OutputHelpers.WriteConsole(e.Data, ConsoleColor.Cyan); + process.ErrorDataReceived += (sender, e) => OutputHelpers.WriteConsole(e.Data, ConsoleColor.Cyan); + + process.Start(); + + process.BeginOutputReadLine(); + process.BeginErrorReadLine(); + + process.WaitForExit(); + + outputFramesPath = outputFramesPath.Replace("\\frame_%04d.png", ""); + + string[] files = Directory.GetFiles(outputFramesPath); + + var prompt = "Iron Man"; + var negativePrompt = ""; + + var promptOptions = new PromptOptions + { + Prompt = prompt, + NegativePrompt = negativePrompt, + DiffuserType = DiffuserType.ImageToImage + }; + + var schedulerOptions = new SchedulerOptions + { + SchedulerType = SchedulerType.LCM, + GuidanceScale = 1f, + InferenceSteps = 10, + Strength = 0.35f, + Height = 512, + Width = 512 + }; + + foreach (string filePath in files) + { + OutputHelpers.WriteConsole($"Defusing {filePath}", ConsoleColor.Cyan); + Image frameDestination = new(schedulerOptions.Width, schedulerOptions.Height); + var frameSource = await Image.LoadAsync(filePath); + frameSource.Mutate(x => x.Resize(frameDestination.Size)); + promptOptions.InputImage = new InputImage(frameSource.CloneAs()); + frameDestination = await _stableDiffusionService.GenerateAsImageAsync(new ModelOptions(model), promptOptions, schedulerOptions); + await frameDestination.SaveAsPngAsync(filePath); + OutputHelpers.WriteConsole($"{filePath} saved", ConsoleColor.Cyan); + } + + string outputVideoPath = "C:\\Users\\Hex\\Downloads\\doitdefused.mp4"; + + ffmpegCommand = $"ffmpeg -framerate 30 -i \"{outputFramesPath}\\frame_%04d.png\" -c:v libx264 -pix_fmt yuv420p -y \"{outputVideoPath}\""; + + // Create a process to run the FFmpeg command + process = new Process + { + StartInfo = new ProcessStartInfo + { + FileName = ffmpeg, + Arguments = ffmpegCommand, + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true + } + }; + + process.OutputDataReceived += (sender, e) => OutputHelpers.WriteConsole(e.Data, ConsoleColor.Cyan); + process.ErrorDataReceived += (sender, e) => OutputHelpers.WriteConsole(e.Data, ConsoleColor.Cyan); + + // Start the process + process.Start(); + + process.BeginOutputReadLine(); + process.BeginErrorReadLine(); + + // Wait for the process to exit + process.WaitForExit(); + } + } +} \ No newline at end of file