From 28267ce6d0a5b37836e2eb74a1aa137880040b9d Mon Sep 17 00:00:00 2001 From: sa_ddam213 Date: Wed, 27 Dec 2023 10:49:05 +1300 Subject: [PATCH] Make timeids compatible with more model types --- .../InpaintLegacyDiffuser.cs | 5 ++-- .../InpaintLegacyDiffuser.cs | 5 ++-- .../StableDiffusionXLDiffuser.cs | 28 +++++-------------- 3 files changed, 13 insertions(+), 25 deletions(-) diff --git a/OnnxStack.StableDiffusion/Diffusers/LatentConsistencyXL/InpaintLegacyDiffuser.cs b/OnnxStack.StableDiffusion/Diffusers/LatentConsistencyXL/InpaintLegacyDiffuser.cs index 4a27c9da..6a66f947 100644 --- a/OnnxStack.StableDiffusion/Diffusers/LatentConsistencyXL/InpaintLegacyDiffuser.cs +++ b/OnnxStack.StableDiffusion/Diffusers/LatentConsistencyXL/InpaintLegacyDiffuser.cs @@ -72,7 +72,7 @@ protected override async Task> SchedulerStepAsync(StableDiffu var metadata = _onnxModelService.GetModelMetadata(modelOptions, OnnxModelType.Unet); // Get Time ids - var addTimeIds = GetAddTimeIds(modelOptions, schedulerOptions, performGuidance); + var addTimeIds = GetAddTimeIds(modelOptions, schedulerOptions); // Loop though the timesteps var step = 0; @@ -86,6 +86,7 @@ protected override async Task> SchedulerStepAsync(StableDiffu var inputLatent = performGuidance ? latents.Repeat(2) : latents; var inputTensor = scheduler.ScaleInput(inputLatent, timestep); var timestepTensor = CreateTimestepTensor(timestep); + var timeids = performGuidance ? addTimeIds.Repeat(2) : addTimeIds; var outputChannels = performGuidance ? 2 : 1; var outputDimension = schedulerOptions.GetScaledDimension(outputChannels); @@ -95,7 +96,7 @@ protected override async Task> SchedulerStepAsync(StableDiffu inferenceParameters.AddInputTensor(timestepTensor); inferenceParameters.AddInputTensor(promptEmbeddings.PromptEmbeds); inferenceParameters.AddInputTensor(promptEmbeddings.PooledPromptEmbeds); - inferenceParameters.AddInputTensor(addTimeIds); + inferenceParameters.AddInputTensor(timeids); inferenceParameters.AddOutputBuffer(outputDimension); var results = await _onnxModelService.RunInferenceAsync(modelOptions, OnnxModelType.Unet, inferenceParameters); diff --git a/OnnxStack.StableDiffusion/Diffusers/StableDiffusionXL/InpaintLegacyDiffuser.cs b/OnnxStack.StableDiffusion/Diffusers/StableDiffusionXL/InpaintLegacyDiffuser.cs index 206d1bde..f8b5f5c7 100644 --- a/OnnxStack.StableDiffusion/Diffusers/StableDiffusionXL/InpaintLegacyDiffuser.cs +++ b/OnnxStack.StableDiffusion/Diffusers/StableDiffusionXL/InpaintLegacyDiffuser.cs @@ -72,7 +72,7 @@ protected override async Task> SchedulerStepAsync(StableDiffu var metadata = _onnxModelService.GetModelMetadata(modelOptions, OnnxModelType.Unet); // Get Time ids - var addTimeIds = GetAddTimeIds(modelOptions, schedulerOptions, performGuidance); + var addTimeIds = GetAddTimeIds(modelOptions, schedulerOptions); // Loop though the timesteps var step = 0; @@ -86,6 +86,7 @@ protected override async Task> SchedulerStepAsync(StableDiffu var inputLatent = performGuidance ? latents.Repeat(2) : latents; var inputTensor = scheduler.ScaleInput(inputLatent, timestep); var timestepTensor = CreateTimestepTensor(timestep); + var timeids = performGuidance ? addTimeIds.Repeat(2) : addTimeIds; var outputChannels = performGuidance ? 2 : 1; var outputDimension = schedulerOptions.GetScaledDimension(outputChannels); @@ -95,7 +96,7 @@ protected override async Task> SchedulerStepAsync(StableDiffu inferenceParameters.AddInputTensor(timestepTensor); inferenceParameters.AddInputTensor(promptEmbeddings.PromptEmbeds); inferenceParameters.AddInputTensor(promptEmbeddings.PooledPromptEmbeds); - inferenceParameters.AddInputTensor(addTimeIds); + inferenceParameters.AddInputTensor(timeids); inferenceParameters.AddOutputBuffer(outputDimension); var results = await _onnxModelService.RunInferenceAsync(modelOptions, OnnxModelType.Unet, inferenceParameters); diff --git a/OnnxStack.StableDiffusion/Diffusers/StableDiffusionXL/StableDiffusionXLDiffuser.cs b/OnnxStack.StableDiffusion/Diffusers/StableDiffusionXL/StableDiffusionXLDiffuser.cs index f7359d37..72a3dad5 100644 --- a/OnnxStack.StableDiffusion/Diffusers/StableDiffusionXL/StableDiffusionXLDiffuser.cs +++ b/OnnxStack.StableDiffusion/Diffusers/StableDiffusionXL/StableDiffusionXLDiffuser.cs @@ -60,7 +60,7 @@ protected override async Task> SchedulerStepAsync(StableDiffu var metadata = _onnxModelService.GetModelMetadata(modelOptions, OnnxModelType.Unet); // Get Time ids - var addTimeIds = GetAddTimeIds(modelOptions, schedulerOptions, performGuidance); + var addTimeIds = GetAddTimeIds(modelOptions, schedulerOptions); // Loop though the timesteps var step = 0; @@ -74,6 +74,7 @@ protected override async Task> SchedulerStepAsync(StableDiffu var inputLatent = performGuidance ? latents.Repeat(2) : latents; var inputTensor = scheduler.ScaleInput(inputLatent, timestep); var timestepTensor = CreateTimestepTensor(timestep); + var timeids = performGuidance ? addTimeIds.Repeat(2) : addTimeIds; var outputChannels = performGuidance ? 2 : 1; var outputDimension = schedulerOptions.GetScaledDimension(outputChannels); @@ -83,7 +84,7 @@ protected override async Task> SchedulerStepAsync(StableDiffu inferenceParameters.AddInputTensor(timestepTensor); inferenceParameters.AddInputTensor(promptEmbeddings.PromptEmbeds); inferenceParameters.AddInputTensor(promptEmbeddings.PooledPromptEmbeds); - inferenceParameters.AddInputTensor(addTimeIds); + inferenceParameters.AddInputTensor(timeids); inferenceParameters.AddOutputBuffer(outputDimension); var results = await _onnxModelService.RunInferenceAsync(modelOptions, OnnxModelType.Unet, inferenceParameters); @@ -115,26 +116,11 @@ protected override async Task> SchedulerStepAsync(StableDiffu /// /// The scheduler options. /// - protected DenseTensor GetAddTimeIds(StableDiffusionModelSet model, SchedulerOptions schedulerOptions, bool performGuidance) + protected DenseTensor GetAddTimeIds(StableDiffusionModelSet model, SchedulerOptions schedulerOptions) { - float[] result; - if (model.ModelType == ModelType.Refiner) - { - //original_size + crops_coords_top_left + aesthetic_score - //original_size + crops_coords_top_left + negative_aesthetic_score - result = !performGuidance - ? new float[] { schedulerOptions.Height, schedulerOptions.Width, 0, 0, schedulerOptions.AestheticScore } - : new float[] { schedulerOptions.Height, schedulerOptions.Width, 0, 0, schedulerOptions.AestheticNegativeScore, schedulerOptions.Height, schedulerOptions.Width, 0, 0, schedulerOptions.AestheticScore }; - } - else - { - //original_size + crops_coords_top_left + target_size - //original_size + crops_coords_top_left + negative_target_size - result = !performGuidance - ? new float[] { schedulerOptions.Height, schedulerOptions.Width, 0, 0, schedulerOptions.Height, schedulerOptions.Width } - : new float[] { schedulerOptions.Height, schedulerOptions.Width, 0, 0, schedulerOptions.Height, schedulerOptions.Width, schedulerOptions.Height, schedulerOptions.Width, 0, 0, schedulerOptions.Height, schedulerOptions.Width }; - } - + float[] result = model.ModelType == ModelType.Refiner + ? new float[] { schedulerOptions.Height, schedulerOptions.Width, 0, 0, schedulerOptions.AestheticScore } + : new float[] { schedulerOptions.Height, schedulerOptions.Width, 0, 0, schedulerOptions.Height, schedulerOptions.Width }; return TensorHelper.CreateTensor(result, new[] { 1, result.Length }); }