diff --git a/ci/licenses_golden/licenses_flutter b/ci/licenses_golden/licenses_flutter index fc4a499cb44e6..a364fa26954c1 100644 --- a/ci/licenses_golden/licenses_flutter +++ b/ci/licenses_golden/licenses_flutter @@ -1212,6 +1212,8 @@ FILE: ../../../flutter/impeller/entity/contents/filters/morphology_filter_conten FILE: ../../../flutter/impeller/entity/contents/filters/morphology_filter_contents.h FILE: ../../../flutter/impeller/entity/contents/filters/srgb_to_linear_filter_contents.cc FILE: ../../../flutter/impeller/entity/contents/filters/srgb_to_linear_filter_contents.h +FILE: ../../../flutter/impeller/entity/contents/filters/yuv_to_rgb_filter_contents.cc +FILE: ../../../flutter/impeller/entity/contents/filters/yuv_to_rgb_filter_contents.h FILE: ../../../flutter/impeller/entity/contents/gradient_generator.cc FILE: ../../../flutter/impeller/entity/contents/gradient_generator.h FILE: ../../../flutter/impeller/entity/contents/linear_gradient_contents.cc @@ -1301,6 +1303,8 @@ FILE: ../../../flutter/impeller/entity/shaders/texture_fill.vert FILE: ../../../flutter/impeller/entity/shaders/tiled_texture_fill.frag FILE: ../../../flutter/impeller/entity/shaders/tiled_texture_fill.vert FILE: ../../../flutter/impeller/entity/shaders/vertices.frag +FILE: ../../../flutter/impeller/entity/shaders/yuv_to_rgb_filter.frag +FILE: ../../../flutter/impeller/entity/shaders/yuv_to_rgb_filter.vert FILE: ../../../flutter/impeller/geometry/color.cc FILE: ../../../flutter/impeller/geometry/color.h FILE: ../../../flutter/impeller/geometry/constants.cc diff --git a/common/graphics/texture.h b/common/graphics/texture.h index 585b3c14bb12d..16a97658d9fed 100644 --- a/common/graphics/texture.h +++ b/common/graphics/texture.h @@ -16,6 +16,10 @@ class GrDirectContext; +namespace impeller { +class AiksContext; +}; + namespace flutter { class ContextListener { @@ -39,6 +43,7 @@ class Texture : public ContextListener { SkCanvas* canvas = nullptr; DisplayListBuilder* builder = nullptr; GrDirectContext* gr_context = nullptr; + impeller::AiksContext* aiks_context = nullptr; const SkPaint* sk_paint = nullptr; const DlPaint* dl_paint = nullptr; }; diff --git a/flow/compositor_context.cc b/flow/compositor_context.cc index 891a1cd6a0839..90dcc6ac42282 100644 --- a/flow/compositor_context.cc +++ b/flow/compositor_context.cc @@ -83,11 +83,12 @@ std::unique_ptr CompositorContext::AcquireFrame( bool surface_supports_readback, fml::RefPtr raster_thread_merger, // NOLINT(performance-unnecessary-value-param) - DisplayListBuilder* display_list_builder) { + DisplayListBuilder* display_list_builder, + impeller::AiksContext* aiks_context) { return std::make_unique( *this, gr_context, canvas, view_embedder, root_surface_transformation, instrumentation_enabled, surface_supports_readback, raster_thread_merger, - display_list_builder); + display_list_builder, aiks_context); } CompositorContext::ScopedFrame::ScopedFrame( @@ -99,11 +100,13 @@ CompositorContext::ScopedFrame::ScopedFrame( bool instrumentation_enabled, bool surface_supports_readback, fml::RefPtr raster_thread_merger, - DisplayListBuilder* display_list_builder) + DisplayListBuilder* display_list_builder, + impeller::AiksContext* aiks_context) : context_(context), gr_context_(gr_context), canvas_(canvas), display_list_builder_(display_list_builder), + aiks_context_(aiks_context), view_embedder_(view_embedder), root_surface_transformation_(root_surface_transformation), instrumentation_enabled_(instrumentation_enabled), diff --git a/flow/compositor_context.h b/flow/compositor_context.h index 68535a6194061..3e9552bd94f7f 100644 --- a/flow/compositor_context.h +++ b/flow/compositor_context.h @@ -114,7 +114,8 @@ class CompositorContext { bool instrumentation_enabled, bool surface_supports_readback, fml::RefPtr raster_thread_merger, - DisplayListBuilder* display_list_builder); + DisplayListBuilder* display_list_builder, + impeller::AiksContext* aiks_context); virtual ~ScopedFrame(); @@ -136,6 +137,8 @@ class CompositorContext { GrDirectContext* gr_context() const { return gr_context_; } + impeller::AiksContext* aiks_context() const { return aiks_context_; } + virtual RasterStatus Raster(LayerTree& layer_tree, bool ignore_raster_cache, FrameDamage* frame_damage); @@ -145,6 +148,7 @@ class CompositorContext { GrDirectContext* gr_context_; SkCanvas* canvas_; DisplayListBuilder* display_list_builder_; + impeller::AiksContext* aiks_context_; ExternalViewEmbedder* view_embedder_; const SkMatrix& root_surface_transformation_; const bool instrumentation_enabled_; @@ -168,7 +172,8 @@ class CompositorContext { bool instrumentation_enabled, bool surface_supports_readback, fml::RefPtr raster_thread_merger, - DisplayListBuilder* display_list_builder); + DisplayListBuilder* display_list_builder, + impeller::AiksContext* aiks_context); void OnGrContextCreated(); diff --git a/flow/layers/layer.h b/flow/layers/layer.h index 09f47b5de4d50..0e7100f45255b 100644 --- a/flow/layers/layer.h +++ b/flow/layers/layer.h @@ -154,6 +154,7 @@ struct PaintContext { SkScalar inherited_opacity = SK_Scalar1; DisplayListBuilder* leaf_nodes_builder = nullptr; DisplayListBuilderMultiplexer* builder_multiplexer = nullptr; + impeller::AiksContext* aiks_context; }; // Represents a single composited layer. Created on the UI thread but then diff --git a/flow/layers/layer_tree.cc b/flow/layers/layer_tree.cc index 0e5aacc98fe92..6eccffefda3ad 100644 --- a/flow/layers/layer_tree.cc +++ b/flow/layers/layer_tree.cc @@ -158,6 +158,7 @@ void LayerTree::Paint(CompositorContext::ScopedFrame& frame, .inherited_opacity = SK_Scalar1, .leaf_nodes_builder = builder, .builder_multiplexer = builder ? &builder_multiplexer : nullptr, + .aiks_context = frame.aiks_context(), // clang-format on }; diff --git a/flow/layers/layer_tree_unittests.cc b/flow/layers/layer_tree_unittests.cc index 950e542c3a685..882e2039a47a1 100644 --- a/flow/layers/layer_tree_unittests.cc +++ b/flow/layers/layer_tree_unittests.cc @@ -28,6 +28,7 @@ class LayerTreeTest : public CanvasTest { false, true, nullptr, + nullptr, nullptr)) {} LayerTree& layer_tree() { return layer_tree_; } diff --git a/flow/layers/texture_layer.cc b/flow/layers/texture_layer.cc index 00361d1606132..c141820232441 100644 --- a/flow/layers/texture_layer.cc +++ b/flow/layers/texture_layer.cc @@ -63,6 +63,7 @@ void TextureLayer::Paint(PaintContext& context) const { .canvas = context.leaf_nodes_canvas, .builder = context.leaf_nodes_builder, .gr_context = context.gr_context, + .aiks_context = context.aiks_context, .sk_paint = cache_paint.sk_paint(), .dl_paint = cache_paint.dl_paint(), }; diff --git a/impeller/aiks/aiks_context.cc b/impeller/aiks/aiks_context.cc index a1c8b32308e17..fd504cbe03ca3 100644 --- a/impeller/aiks/aiks_context.cc +++ b/impeller/aiks/aiks_context.cc @@ -32,6 +32,10 @@ std::shared_ptr AiksContext::GetContext() const { return context_; } +const ContentContext& AiksContext::GetContentContext() const { + return *content_context_; +} + bool AiksContext::Render(const Picture& picture, RenderTarget& render_target) { if (!IsValid()) { return false; diff --git a/impeller/aiks/aiks_context.h b/impeller/aiks/aiks_context.h index 7f7780ffb69ab..d748f3156fcba 100644 --- a/impeller/aiks/aiks_context.h +++ b/impeller/aiks/aiks_context.h @@ -26,6 +26,8 @@ class AiksContext { std::shared_ptr GetContext() const; + const ContentContext& GetContentContext() const; + bool Render(const Picture& picture, RenderTarget& render_target); private: diff --git a/impeller/display_list/display_list_image_impeller.cc b/impeller/display_list/display_list_image_impeller.cc index f964010b9ae90..ae71d94132578 100644 --- a/impeller/display_list/display_list_image_impeller.cc +++ b/impeller/display_list/display_list_image_impeller.cc @@ -4,6 +4,9 @@ #include "impeller/display_list/display_list_image_impeller.h" +#include "impeller/aiks/aiks_context.h" +#include "impeller/entity/contents/filters/filter_contents.h" + namespace impeller { sk_sp DlImageImpeller::Make(std::shared_ptr texture) { @@ -13,6 +16,23 @@ sk_sp DlImageImpeller::Make(std::shared_ptr texture) { return sk_sp(new DlImageImpeller(std::move(texture))); } +sk_sp DlImageImpeller::MakeFromYUVTextures( + AiksContext* aiks_context, + std::shared_ptr y_texture, + std::shared_ptr uv_texture, + YUVColorSpace yuv_color_space) { + if (!aiks_context || !y_texture || !uv_texture) { + return nullptr; + } + auto yuv_to_rgb_filter_contents = FilterContents::MakeYUVToRGBFilter( + std::move(y_texture), std::move(uv_texture), yuv_color_space); + impeller::Entity entity; + entity.SetBlendMode(impeller::BlendMode::kSource); + auto snapshot = yuv_to_rgb_filter_contents->RenderToSnapshot( + aiks_context->GetContentContext(), entity); + return impeller::DlImageImpeller::Make(snapshot->texture); +} + DlImageImpeller::DlImageImpeller(std::shared_ptr texture) : texture_(std::move(texture)) {} diff --git a/impeller/display_list/display_list_image_impeller.h b/impeller/display_list/display_list_image_impeller.h index cf7ae7501eb69..65b9c2892bc2a 100644 --- a/impeller/display_list/display_list_image_impeller.h +++ b/impeller/display_list/display_list_image_impeller.h @@ -10,10 +10,18 @@ namespace impeller { +class AiksContext; + class DlImageImpeller final : public flutter::DlImage { public: static sk_sp Make(std::shared_ptr texture); + static sk_sp MakeFromYUVTextures( + AiksContext* aiks_context, + std::shared_ptr y_texture, + std::shared_ptr uv_texture, + YUVColorSpace yuv_color_space); + // |DlImage| ~DlImageImpeller() override; diff --git a/impeller/entity/BUILD.gn b/impeller/entity/BUILD.gn index 4dce88ea473f6..1175f5daef889 100644 --- a/impeller/entity/BUILD.gn +++ b/impeller/entity/BUILD.gn @@ -61,6 +61,8 @@ impeller_shaders("entity_shaders") { "shaders/tiled_texture_fill.frag", "shaders/tiled_texture_fill.vert", "shaders/vertices.frag", + "shaders/yuv_to_rgb_filter.frag", + "shaders/yuv_to_rgb_filter.vert", ] } @@ -106,6 +108,8 @@ impeller_component("entity") { "contents/filters/morphology_filter_contents.h", "contents/filters/srgb_to_linear_filter_contents.cc", "contents/filters/srgb_to_linear_filter_contents.h", + "contents/filters/yuv_to_rgb_filter_contents.cc", + "contents/filters/yuv_to_rgb_filter_contents.h", "contents/gradient_generator.cc", "contents/gradient_generator.h", "contents/linear_gradient_contents.cc", diff --git a/impeller/entity/contents/content_context.cc b/impeller/entity/contents/content_context.cc index 6e7918f14edd3..d8950c3554fce 100644 --- a/impeller/entity/contents/content_context.cc +++ b/impeller/entity/contents/content_context.cc @@ -214,6 +214,8 @@ ContentContext::ContentContext(std::shared_ptr context) geometry_position_pipelines_[{}] = CreateDefaultPipeline(*context_); atlas_pipelines_[{}] = CreateDefaultPipeline(*context_); + yuv_to_rgb_filter_pipelines_[{}] = + CreateDefaultPipeline(*context_); // Pipelines that are variants of the base pipelines with custom descriptors. // TODO(98684): Rework this API to allow fetching the descriptor without diff --git a/impeller/entity/contents/content_context.h b/impeller/entity/contents/content_context.h index 6a40e6a9543e3..f4c39b89ea00b 100644 --- a/impeller/entity/contents/content_context.h +++ b/impeller/entity/contents/content_context.h @@ -61,6 +61,8 @@ #include "impeller/entity/tiled_texture_fill.frag.h" #include "impeller/entity/tiled_texture_fill.vert.h" #include "impeller/entity/vertices.frag.h" +#include "impeller/entity/yuv_to_rgb_filter.frag.h" +#include "impeller/entity/yuv_to_rgb_filter.vert.h" #include "impeller/renderer/formats.h" #include "impeller/renderer/pipeline.h" @@ -160,6 +162,8 @@ using GeometryPositionPipeline = RenderPipelineT; using GeometryColorPipeline = RenderPipelineT; +using YUVToRGBFilterPipeline = + RenderPipelineT; struct ContentContextOptions { SampleCount sample_count = SampleCount::kCount1; @@ -299,6 +303,11 @@ class ContentContext { return GetPipeline(atlas_pipelines_, opts); } + std::shared_ptr> GetYUVToRGBFilterPipeline( + ContentContextOptions opts) const { + return GetPipeline(yuv_to_rgb_filter_pipelines_, opts); + } + // Advanced blends. std::shared_ptr> GetBlendColorPipeline( @@ -422,6 +431,7 @@ class ContentContext { mutable Variants atlas_pipelines_; mutable Variants geometry_position_pipelines_; mutable Variants geometry_color_pipelines_; + mutable Variants yuv_to_rgb_filter_pipelines_; // Advanced blends. mutable Variants blend_color_pipelines_; mutable Variants blend_colorburn_pipelines_; diff --git a/impeller/entity/contents/filters/filter_contents.cc b/impeller/entity/contents/filters/filter_contents.cc index b472dcec737cd..d43e4298c8845 100644 --- a/impeller/entity/contents/filters/filter_contents.cc +++ b/impeller/entity/contents/filters/filter_contents.cc @@ -20,6 +20,7 @@ #include "impeller/entity/contents/filters/local_matrix_filter_contents.h" #include "impeller/entity/contents/filters/matrix_filter_contents.h" #include "impeller/entity/contents/filters/morphology_filter_contents.h" +#include "impeller/entity/contents/filters/yuv_to_rgb_filter_contents.h" #include "impeller/entity/contents/texture_contents.h" #include "impeller/entity/entity.h" #include "impeller/geometry/path_builder.h" @@ -129,6 +130,17 @@ std::shared_ptr FilterContents::MakeLocalMatrixFilter( return filter; } +std::shared_ptr FilterContents::MakeYUVToRGBFilter( + std::shared_ptr y_texture, + std::shared_ptr uv_texture, + YUVColorSpace yuv_color_space) { + auto filter = std::make_shared(); + filter->SetInputs({impeller::FilterInput::Make(y_texture), + impeller::FilterInput::Make(uv_texture)}); + filter->SetYUVColorSpace(yuv_color_space); + return filter; +} + FilterContents::FilterContents() = default; FilterContents::~FilterContents() = default; diff --git a/impeller/entity/contents/filters/filter_contents.h b/impeller/entity/contents/filters/filter_contents.h index 48e12d338958c..9387e7ce7d014 100644 --- a/impeller/entity/contents/filters/filter_contents.h +++ b/impeller/entity/contents/filters/filter_contents.h @@ -84,6 +84,11 @@ class FilterContents : public Contents { FilterInput::Ref input, const Matrix& matrix); + static std::shared_ptr MakeYUVToRGBFilter( + std::shared_ptr y_texture, + std::shared_ptr uv_texture, + YUVColorSpace yuv_color_space); + FilterContents(); ~FilterContents() override; diff --git a/impeller/entity/contents/filters/yuv_to_rgb_filter_contents.cc b/impeller/entity/contents/filters/yuv_to_rgb_filter_contents.cc new file mode 100644 index 0000000000000..b00a4b92c8228 --- /dev/null +++ b/impeller/entity/contents/filters/yuv_to_rgb_filter_contents.cc @@ -0,0 +1,124 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "impeller/entity/contents/filters/yuv_to_rgb_filter_contents.h" + +#include "impeller/entity/contents/content_context.h" +#include "impeller/geometry/matrix.h" +#include "impeller/renderer/formats.h" +#include "impeller/renderer/render_pass.h" +#include "impeller/renderer/sampler_library.h" + +namespace impeller { + +// clang-format off +constexpr Matrix kMatrixBT601LimitedRange = { + 1.164, 1.164, 1.164, 0.0, + 0.0, -0.392, 2.017, 0.0, + 1.596, -0.813, 0.0, 0.0, + 0.0, 0.0, 0.0, 1.0 +}; + +constexpr Matrix kMatrixBT601FullRange = { + 1.0, 1.0, 1.0, 0.0, + 0.0, -0.344, 1.772, 0.0, + 1.402, -0.714, 0.0, 0.0, + 0.0, 0.0, 0.0, 1.0 +}; +// clang-format on + +YUVToRGBFilterContents::YUVToRGBFilterContents() = default; + +YUVToRGBFilterContents::~YUVToRGBFilterContents() = default; + +void YUVToRGBFilterContents::SetYUVColorSpace(YUVColorSpace yuv_color_space) { + yuv_color_space_ = yuv_color_space; +} + +std::optional YUVToRGBFilterContents::RenderFilter( + const FilterInput::Vector& inputs, + const ContentContext& renderer, + const Entity& entity, + const Matrix& effect_transform, + const Rect& coverage) const { + if (inputs.size() < 2) { + return std::nullopt; + } + + using VS = YUVToRGBFilterPipeline::VertexShader; + using FS = YUVToRGBFilterPipeline::FragmentShader; + + auto y_input_snapshot = inputs[0]->GetSnapshot(renderer, entity); + auto uv_input_snapshot = inputs[1]->GetSnapshot(renderer, entity); + if (!y_input_snapshot.has_value() || !uv_input_snapshot.has_value()) { + return std::nullopt; + } + + if (y_input_snapshot->texture->GetTextureDescriptor().format != + PixelFormat::kR8UNormInt || + uv_input_snapshot->texture->GetTextureDescriptor().format != + PixelFormat::kR8G8UNormInt) { + return std::nullopt; + } + + ContentContext::SubpassCallback callback = [&](const ContentContext& renderer, + RenderPass& pass) { + Command cmd; + cmd.label = "YUV to RGB Filter"; + + auto options = OptionsFromPass(pass); + options.blend_mode = BlendMode::kSource; + cmd.pipeline = renderer.GetYUVToRGBFilterPipeline(options); + + VertexBufferBuilder vtx_builder; + vtx_builder.AddVertices({ + {Point(0, 0)}, + {Point(1, 0)}, + {Point(1, 1)}, + {Point(0, 0)}, + {Point(1, 1)}, + {Point(0, 1)}, + }); + + auto& host_buffer = pass.GetTransientsBuffer(); + auto vtx_buffer = vtx_builder.CreateVertexBuffer(host_buffer); + cmd.BindVertices(vtx_buffer); + + VS::FrameInfo frame_info; + frame_info.mvp = Matrix::MakeOrthographic(ISize(1, 1)); + + FS::FragInfo frag_info; + frag_info.texture_sampler_y_coord_scale = + y_input_snapshot->texture->GetYCoordScale(); + frag_info.yuv_color_space = static_cast(yuv_color_space_); + switch (yuv_color_space_) { + case YUVColorSpace::kBT601LimitedRange: + frag_info.matrix = kMatrixBT601LimitedRange; + break; + case YUVColorSpace::kBT601FullRange: + frag_info.matrix = kMatrixBT601FullRange; + break; + } + + auto sampler = renderer.GetContext()->GetSamplerLibrary()->GetSampler({}); + FS::BindYTexture(cmd, y_input_snapshot->texture, sampler); + FS::BindUvTexture(cmd, uv_input_snapshot->texture, sampler); + + FS::BindFragInfo(cmd, host_buffer.EmplaceUniform(frag_info)); + VS::BindFrameInfo(cmd, host_buffer.EmplaceUniform(frame_info)); + + return pass.AddCommand(std::move(cmd)); + }; + + auto out_texture = + renderer.MakeSubpass(y_input_snapshot->texture->GetSize(), callback); + if (!out_texture) { + return std::nullopt; + } + out_texture->SetLabel("YUVToRGB Texture"); + + return Snapshot{.texture = out_texture}; +} + +} // namespace impeller diff --git a/impeller/entity/contents/filters/yuv_to_rgb_filter_contents.h b/impeller/entity/contents/filters/yuv_to_rgb_filter_contents.h new file mode 100644 index 0000000000000..7e15cb5efce97 --- /dev/null +++ b/impeller/entity/contents/filters/yuv_to_rgb_filter_contents.h @@ -0,0 +1,33 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#pragma once + +#include "impeller/entity/contents/filters/filter_contents.h" + +namespace impeller { + +class YUVToRGBFilterContents final : public FilterContents { + public: + YUVToRGBFilterContents(); + + ~YUVToRGBFilterContents() override; + + void SetYUVColorSpace(YUVColorSpace yuv_color_space); + + private: + // |FilterContents| + std::optional RenderFilter( + const FilterInput::Vector& input_textures, + const ContentContext& renderer, + const Entity& entity, + const Matrix& effect_transform, + const Rect& coverage) const override; + + YUVColorSpace yuv_color_space_ = YUVColorSpace::kBT601LimitedRange; + + FML_DISALLOW_COPY_AND_ASSIGN(YUVToRGBFilterContents); +}; + +} // namespace impeller diff --git a/impeller/entity/entity_unittests.cc b/impeller/entity/entity_unittests.cc index c4bdf04e8aace..f8c2b07c5fb85 100644 --- a/impeller/entity/entity_unittests.cc +++ b/impeller/entity/entity_unittests.cc @@ -2036,6 +2036,107 @@ TEST_P(EntityTest, SdfText) { ASSERT_TRUE(OpenPlaygroundHere(callback)); } +static Vector3 RGBToYUV(Vector3 rgb, YUVColorSpace yuv_color_space) { + Vector3 yuv; + switch (yuv_color_space) { + case YUVColorSpace::kBT601FullRange: + yuv.x = rgb.x * 0.299 + rgb.y * 0.587 + rgb.z * 0.114; + yuv.y = rgb.x * -0.169 + rgb.y * -0.331 + rgb.z * 0.5 + 0.5; + yuv.z = rgb.x * 0.5 + rgb.y * -0.419 + rgb.z * -0.081 + 0.5; + break; + case YUVColorSpace::kBT601LimitedRange: + yuv.x = rgb.x * 0.257 + rgb.y * 0.516 + rgb.z * 0.100 + 0.063; + yuv.y = rgb.x * -0.145 + rgb.y * -0.291 + rgb.z * 0.439 + 0.5; + yuv.z = rgb.x * 0.429 + rgb.y * -0.368 + rgb.z * -0.071 + 0.5; + break; + } + return yuv; +} + +static std::vector> CreateTestYUVTextures( + Context* context, + YUVColorSpace yuv_color_space) { + Vector3 red = {244.0 / 255.0, 67.0 / 255.0, 54.0 / 255.0}; + Vector3 green = {76.0 / 255.0, 175.0 / 255.0, 80.0 / 255.0}; + Vector3 blue = {33.0 / 255.0, 150.0 / 255.0, 243.0 / 255.0}; + Vector3 white = {1.0, 1.0, 1.0}; + Vector3 red_yuv = RGBToYUV(red, yuv_color_space); + Vector3 green_yuv = RGBToYUV(green, yuv_color_space); + Vector3 blue_yuv = RGBToYUV(blue, yuv_color_space); + Vector3 white_yuv = RGBToYUV(white, yuv_color_space); + std::vector yuvs{red_yuv, green_yuv, blue_yuv, white_yuv}; + std::vector y_data; + std::vector uv_data; + for (int i = 0; i < 4; i++) { + auto yuv = yuvs[i]; + uint8_t y = std::round(yuv.x * 255.0); + uint8_t u = std::round(yuv.y * 255.0); + uint8_t v = std::round(yuv.z * 255.0); + for (int j = 0; j < 16; j++) { + y_data.push_back(y); + } + for (int j = 0; j < 8; j++) { + uv_data.push_back(j % 2 == 0 ? u : v); + } + } + impeller::TextureDescriptor y_texture_descriptor; + y_texture_descriptor.storage_mode = impeller::StorageMode::kHostVisible; + y_texture_descriptor.format = PixelFormat::kR8UNormInt; + y_texture_descriptor.size = {8, 8}; + auto y_texture = + context->GetResourceAllocator()->CreateTexture(y_texture_descriptor); + auto y_mapping = std::make_shared(y_data); + if (!y_texture->SetContents(y_mapping)) { + FML_DLOG(ERROR) << "Could not copy contents into Y texture."; + } + + impeller::TextureDescriptor uv_texture_descriptor; + uv_texture_descriptor.storage_mode = impeller::StorageMode::kHostVisible; + uv_texture_descriptor.format = PixelFormat::kR8G8UNormInt; + uv_texture_descriptor.size = {4, 4}; + auto uv_texture = + context->GetResourceAllocator()->CreateTexture(uv_texture_descriptor); + auto uv_mapping = std::make_shared(uv_data); + if (!uv_texture->SetContents(uv_mapping)) { + FML_DLOG(ERROR) << "Could not copy contents into UV texture."; + } + + return {y_texture, uv_texture}; +} + +TEST_P(EntityTest, YUVToRGBFilter) { + if (GetParam() == PlaygroundBackend::kOpenGLES) { + // TODO(114588) : Support YUV to RGB filter on OpenGLES backend. + GTEST_SKIP_("YUV to RGB filter is not supported on OpenGLES backend yet."); + } + + auto callback = [&](ContentContext& context, RenderPass& pass) -> bool { + YUVColorSpace yuv_color_space_array[2]{YUVColorSpace::kBT601FullRange, + YUVColorSpace::kBT601LimitedRange}; + for (int i = 0; i < 2; i++) { + auto yuv_color_space = yuv_color_space_array[i]; + auto textures = + CreateTestYUVTextures(GetContext().get(), yuv_color_space); + auto filter_contents = FilterContents::MakeYUVToRGBFilter( + textures[0], textures[1], yuv_color_space); + Entity filter_entity; + filter_entity.SetContents(filter_contents); + auto snapshot = filter_contents->RenderToSnapshot(context, filter_entity); + + Entity entity; + auto contents = TextureContents::MakeRect(Rect::MakeLTRB(0, 0, 256, 256)); + contents->SetTexture(snapshot->texture); + contents->SetSourceRect(Rect::MakeSize(snapshot->texture->GetSize())); + entity.SetContents(contents); + entity.SetTransformation( + Matrix::MakeTranslation({static_cast(100 + 400 * i), 300})); + entity.Render(context, pass); + } + return true; + }; + ASSERT_TRUE(OpenPlaygroundHere(callback)); +} + TEST_P(EntityTest, RuntimeEffect) { if (GetParam() != PlaygroundBackend::kMetal) { GTEST_SKIP_("This test only has a Metal fixture at the moment."); diff --git a/impeller/entity/shaders/yuv_to_rgb_filter.frag b/impeller/entity/shaders/yuv_to_rgb_filter.frag new file mode 100644 index 0000000000000..b734b486024ab --- /dev/null +++ b/impeller/entity/shaders/yuv_to_rgb_filter.frag @@ -0,0 +1,35 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include +#include + +uniform sampler2D y_texture; +uniform sampler2D uv_texture; + +// These values must correspond to the order of the items in the +// 'YUVColorSpace' enum class. +const float kBT601LimitedRange = 0; +const float kBT601FullRange = 1; + +uniform FragInfo { + float texture_sampler_y_coord_scale; + mat4 matrix; + float yuv_color_space; +} frag_info; + +in vec2 v_position; +out vec4 frag_color; + +void main() { + vec3 yuv; + vec3 yuv_offset = vec3(0.0, 0.5, 0.5); + if (frag_info.yuv_color_space == kBT601LimitedRange) { + yuv_offset.x = 16.0 / 255.0; + } + + yuv.x = IPSample(y_texture, v_position, frag_info.texture_sampler_y_coord_scale).r; + yuv.yz = IPSample(uv_texture, v_position, frag_info.texture_sampler_y_coord_scale).rg; + frag_color = frag_info.matrix * vec4(yuv - yuv_offset, 1); +} diff --git a/impeller/entity/shaders/yuv_to_rgb_filter.vert b/impeller/entity/shaders/yuv_to_rgb_filter.vert new file mode 100644 index 0000000000000..b741b2744ec60 --- /dev/null +++ b/impeller/entity/shaders/yuv_to_rgb_filter.vert @@ -0,0 +1,15 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +uniform FrameInfo { + mat4 mvp; +} frame_info; + +in vec2 position; +out vec2 v_position; + +void main() { + v_position = position; + gl_Position = frame_info.mvp * vec4(position, 0.0, 1.0); +} diff --git a/impeller/geometry/color.h b/impeller/geometry/color.h index 64c1d6b9840cb..6966a3ccc0f6d 100644 --- a/impeller/geometry/color.h +++ b/impeller/geometry/color.h @@ -15,6 +15,8 @@ namespace impeller { struct ColorHSB; struct Vector4; +enum class YUVColorSpace { kBT601LimitedRange, kBT601FullRange }; + /// All blend modes assume that both the source (fragment output) and /// destination (first color attachment) have colors with premultiplied alpha. enum class BlendMode { diff --git a/impeller/renderer/backend/gles/texture_gles.cc b/impeller/renderer/backend/gles/texture_gles.cc index 0c18dab461979..e8a8f9b455721 100644 --- a/impeller/renderer/backend/gles/texture_gles.cc +++ b/impeller/renderer/backend/gles/texture_gles.cc @@ -109,6 +109,8 @@ struct TexImage2DData { break; case PixelFormat::kUnknown: case PixelFormat::kS8UInt: + case PixelFormat::kR8UNormInt: + case PixelFormat::kR8G8UNormInt: return; } is_valid_ = true; @@ -141,6 +143,10 @@ struct TexImage2DData { return; case PixelFormat::kS8UInt: return; + case PixelFormat::kR8UNormInt: + return; + case PixelFormat::kR8G8UNormInt: + return; } is_valid_ = true; } @@ -275,6 +281,8 @@ static std::optional ToRenderBufferFormat(PixelFormat format) { return GL_STENCIL_INDEX8; case PixelFormat::kUnknown: case PixelFormat::kA8UNormInt: + case PixelFormat::kR8UNormInt: + case PixelFormat::kR8G8UNormInt: case PixelFormat::kR8G8B8A8UNormIntSRGB: case PixelFormat::kB8G8R8A8UNormIntSRGB: return std::nullopt; diff --git a/impeller/renderer/backend/metal/formats_mtl.h b/impeller/renderer/backend/metal/formats_mtl.h index c9857f9a3b108..591ecb8d24cc8 100644 --- a/impeller/renderer/backend/metal/formats_mtl.h +++ b/impeller/renderer/backend/metal/formats_mtl.h @@ -43,6 +43,10 @@ constexpr MTLPixelFormat ToMTLPixelFormat(PixelFormat format) { return MTLPixelFormatInvalid; case PixelFormat::kA8UNormInt: return MTLPixelFormatA8Unorm; + case PixelFormat::kR8UNormInt: + return MTLPixelFormatR8Unorm; + case PixelFormat::kR8G8UNormInt: + return MTLPixelFormatRG8Unorm; case PixelFormat::kB8G8R8A8UNormInt: return MTLPixelFormatBGRA8Unorm; case PixelFormat::kB8G8R8A8UNormIntSRGB: diff --git a/impeller/renderer/backend/vulkan/formats_vk.h b/impeller/renderer/backend/vulkan/formats_vk.h index 716f785d69b9f..5f91080644808 100644 --- a/impeller/renderer/backend/vulkan/formats_vk.h +++ b/impeller/renderer/backend/vulkan/formats_vk.h @@ -150,6 +150,10 @@ constexpr vk::Format ToVKImageFormat(PixelFormat format) { return vk::Format::eB8G8R8A8Srgb; case PixelFormat::kS8UInt: return vk::Format::eS8Uint; + case PixelFormat::kR8UNormInt: + return vk::Format::eR8Unorm; + case PixelFormat::kR8G8UNormInt: + return vk::Format::eR8G8Unorm; } } @@ -176,6 +180,12 @@ constexpr PixelFormat ToPixelFormat(vk::Format format) { case vk::Format::eS8Uint: return PixelFormat::kS8UInt; + case vk::Format::eR8Unorm: + return PixelFormat::kR8UNormInt; + + case vk::Format::eR8G8Unorm: + return PixelFormat::kR8G8UNormInt; + default: return PixelFormat::kUnknown; } diff --git a/impeller/renderer/formats.h b/impeller/renderer/formats.h index 118138c03bac1..945b7f87752ed 100644 --- a/impeller/renderer/formats.h +++ b/impeller/renderer/formats.h @@ -81,6 +81,8 @@ enum class StorageMode { enum class PixelFormat { kUnknown, kA8UNormInt, + kR8UNormInt, + kR8G8UNormInt, kR8G8B8A8UNormInt, kR8G8B8A8UNormIntSRGB, kB8G8R8A8UNormInt, @@ -273,8 +275,11 @@ constexpr size_t BytesPerPixelForPixelFormat(PixelFormat format) { case PixelFormat::kUnknown: return 0u; case PixelFormat::kA8UNormInt: + case PixelFormat::kR8UNormInt: case PixelFormat::kS8UInt: return 1u; + case PixelFormat::kR8G8UNormInt: + return 2u; case PixelFormat::kR8G8B8A8UNormInt: case PixelFormat::kR8G8B8A8UNormIntSRGB: case PixelFormat::kB8G8R8A8UNormInt: diff --git a/shell/common/rasterizer.cc b/shell/common/rasterizer.cc index 7f66f93ff8830..cf124ad80476e 100644 --- a/shell/common/rasterizer.cc +++ b/shell/common/rasterizer.cc @@ -528,9 +528,10 @@ RasterStatus Rasterizer::DrawToSurfaceUnsafe( root_surface_transformation, // root surface transformation true, // instrumentation enabled frame->framebuffer_info() - .supports_readback, // surface supports pixel reads - raster_thread_merger_, // thread merger - frame->GetDisplayListBuilder().get() // display list builder + .supports_readback, // surface supports pixel reads + raster_thread_merger_, // thread merger + frame->GetDisplayListBuilder().get(), // display list builder + surface_->GetAiksContext() // aiks context ); if (compositor_frame) { compositor_context_->raster_cache().BeginFrame(); @@ -630,7 +631,7 @@ static sk_sp ScreenshotLayerTreeAsPicture( // https://github.com/flutter/flutter/issues/23435 auto frame = compositor_context.AcquireFrame( nullptr, recorder.getRecordingCanvas(), nullptr, - root_surface_transformation, false, true, nullptr, nullptr); + root_surface_transformation, false, true, nullptr, nullptr, nullptr); frame->Raster(*tree, true, nullptr); #if defined(OS_FUCHSIA) @@ -686,7 +687,8 @@ sk_sp Rasterizer::ScreenshotLayerTreeAsImage( false, // instrumentation enabled true, // render buffer readback supported nullptr, // thread merger - nullptr // display list builder + nullptr, // display list builder + nullptr // aiks context ); canvas->clear(SK_ColorTRANSPARENT); frame->Raster(*tree, true, nullptr); diff --git a/shell/platform/darwin/graphics/FlutterDarwinContextMetalImpeller.h b/shell/platform/darwin/graphics/FlutterDarwinContextMetalImpeller.h index f705a4f9a4c57..55fc5ac9b79c3 100644 --- a/shell/platform/darwin/graphics/FlutterDarwinContextMetalImpeller.h +++ b/shell/platform/darwin/graphics/FlutterDarwinContextMetalImpeller.h @@ -32,6 +32,7 @@ NS_ASSUME_NONNULL_BEGIN - (FlutterDarwinExternalTextureMetal*) createExternalTextureWithIdentifier:(int64_t)textureID texture:(NSObject*)texture; + /** * Impeller context; */ diff --git a/shell/platform/darwin/graphics/FlutterDarwinExternalTextureMetal.mm b/shell/platform/darwin/graphics/FlutterDarwinExternalTextureMetal.mm index 42d4d3d9c1dff..63552ed4f8bf8 100644 --- a/shell/platform/darwin/graphics/FlutterDarwinExternalTextureMetal.mm +++ b/shell/platform/darwin/graphics/FlutterDarwinExternalTextureMetal.mm @@ -154,12 +154,6 @@ - (void)onTextureUnregistered { - (sk_sp)wrapNV12ExternalPixelBuffer:(CVPixelBufferRef)pixelBuffer context:(flutter::Texture::PaintContext&)context { - if (_enableImpeller) { - // TODO(113688): Support YUV external textures. - VALIDATION_LOG << "YUV external texture support is not implemented yet."; - return nullptr; - } - SkISize textureSize = SkISize::Make(CVPixelBufferGetWidth(pixelBuffer), CVPixelBufferGetHeight(pixelBuffer)); CVMetalTextureRef yMetalTexture = nullptr; @@ -206,6 +200,32 @@ - (void)onTextureUnregistered { id uvTex = CVMetalTextureGetTexture(uvMetalTexture); CVBufferRelease(uvMetalTexture); + if (_enableImpeller) { + impeller::TextureDescriptor yDesc; + yDesc.storage_mode = impeller::StorageMode::kHostVisible; + yDesc.format = impeller::PixelFormat::kR8UNormInt; + yDesc.size = {textureSize.width(), textureSize.height()}; + yDesc.mip_count = 1; + auto yTexture = impeller::TextureMTL::Wrapper(yDesc, yTex); + yTexture->SetIntent(impeller::TextureIntent::kUploadFromHost); + + impeller::TextureDescriptor uvDesc; + uvDesc.storage_mode = impeller::StorageMode::kHostVisible; + uvDesc.format = impeller::PixelFormat::kR8G8UNormInt; + uvDesc.size = {textureSize.width() / 2, textureSize.height() / 2}; + uvDesc.mip_count = 1; + auto uvTexture = impeller::TextureMTL::Wrapper(uvDesc, uvTex); + uvTexture->SetIntent(impeller::TextureIntent::kUploadFromHost); + + impeller::YUVColorSpace yuvColorSpace = + _pixelFormat == kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange + ? impeller::YUVColorSpace::kBT601LimitedRange + : impeller::YUVColorSpace::kBT601FullRange; + + return impeller::DlImageImpeller::MakeFromYUVTextures(context.aiks_context, yTexture, uvTexture, + yuvColorSpace); + } + SkYUVColorSpace colorSpace = _pixelFormat == kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange ? kRec601_Limited_SkYUVColorSpace : kJPEG_Full_SkYUVColorSpace;