diff --git a/ci/licenses_golden/licenses_flutter b/ci/licenses_golden/licenses_flutter index 9983e59d6c974..47ed3e9222b42 100644 --- a/ci/licenses_golden/licenses_flutter +++ b/ci/licenses_golden/licenses_flutter @@ -1500,6 +1500,8 @@ ORIGIN: ../../../flutter/impeller/renderer/backend/metal/vertex_descriptor_mtl.h ORIGIN: ../../../flutter/impeller/renderer/backend/metal/vertex_descriptor_mtl.mm + ../../../flutter/LICENSE ORIGIN: ../../../flutter/impeller/renderer/backend/vulkan/allocator_vk.cc + ../../../flutter/LICENSE ORIGIN: ../../../flutter/impeller/renderer/backend/vulkan/allocator_vk.h + ../../../flutter/LICENSE +ORIGIN: ../../../flutter/impeller/renderer/backend/vulkan/android_hardware_buffer_texture_source_vk.cc + ../../../flutter/LICENSE +ORIGIN: ../../../flutter/impeller/renderer/backend/vulkan/android_hardware_buffer_texture_source_vk.h + ../../../flutter/LICENSE ORIGIN: ../../../flutter/impeller/renderer/backend/vulkan/barrier_vk.cc + ../../../flutter/LICENSE ORIGIN: ../../../flutter/impeller/renderer/backend/vulkan/barrier_vk.h + ../../../flutter/LICENSE ORIGIN: ../../../flutter/impeller/renderer/backend/vulkan/blit_command_vk.cc + ../../../flutter/LICENSE @@ -2338,8 +2340,10 @@ ORIGIN: ../../../flutter/shell/platform/android/external_view_embedder/surface_p ORIGIN: ../../../flutter/shell/platform/android/external_view_embedder/surface_pool.h + ../../../flutter/LICENSE ORIGIN: ../../../flutter/shell/platform/android/flutter_main.cc + ../../../flutter/LICENSE ORIGIN: ../../../flutter/shell/platform/android/flutter_main.h + ../../../flutter/LICENSE +ORIGIN: ../../../flutter/shell/platform/android/hardware_buffer_external_texture.h + ../../../flutter/LICENSE ORIGIN: ../../../flutter/shell/platform/android/hardware_buffer_external_texture_gl.cc + ../../../flutter/LICENSE ORIGIN: ../../../flutter/shell/platform/android/hardware_buffer_external_texture_gl.h + ../../../flutter/LICENSE +ORIGIN: ../../../flutter/shell/platform/android/hardware_buffer_external_texture_vk.h + ../../../flutter/LICENSE ORIGIN: ../../../flutter/shell/platform/android/io/flutter/FlutterInjector.java + ../../../flutter/LICENSE ORIGIN: ../../../flutter/shell/platform/android/io/flutter/Log.java + ../../../flutter/LICENSE ORIGIN: ../../../flutter/shell/platform/android/io/flutter/app/FlutterActivity.java + ../../../flutter/LICENSE @@ -4218,6 +4222,8 @@ FILE: ../../../flutter/impeller/renderer/backend/metal/vertex_descriptor_mtl.h FILE: ../../../flutter/impeller/renderer/backend/metal/vertex_descriptor_mtl.mm FILE: ../../../flutter/impeller/renderer/backend/vulkan/allocator_vk.cc FILE: ../../../flutter/impeller/renderer/backend/vulkan/allocator_vk.h +FILE: ../../../flutter/impeller/renderer/backend/vulkan/android_hardware_buffer_texture_source_vk.cc +FILE: ../../../flutter/impeller/renderer/backend/vulkan/android_hardware_buffer_texture_source_vk.h FILE: ../../../flutter/impeller/renderer/backend/vulkan/barrier_vk.cc FILE: ../../../flutter/impeller/renderer/backend/vulkan/barrier_vk.h FILE: ../../../flutter/impeller/renderer/backend/vulkan/blit_command_vk.cc @@ -5061,8 +5067,12 @@ FILE: ../../../flutter/shell/platform/android/external_view_embedder/surface_poo FILE: ../../../flutter/shell/platform/android/external_view_embedder/surface_pool.h FILE: ../../../flutter/shell/platform/android/flutter_main.cc FILE: ../../../flutter/shell/platform/android/flutter_main.h +FILE: ../../../flutter/shell/platform/android/hardware_buffer_external_texture.cc +FILE: ../../../flutter/shell/platform/android/hardware_buffer_external_texture.h FILE: ../../../flutter/shell/platform/android/hardware_buffer_external_texture_gl.cc FILE: ../../../flutter/shell/platform/android/hardware_buffer_external_texture_gl.h +FILE: ../../../flutter/shell/platform/android/hardware_buffer_external_texture_vk.cc +FILE: ../../../flutter/shell/platform/android/hardware_buffer_external_texture_vk.h FILE: ../../../flutter/shell/platform/android/io/flutter/FlutterInjector.java FILE: ../../../flutter/shell/platform/android/io/flutter/Log.java FILE: ../../../flutter/shell/platform/android/io/flutter/app/FlutterActivity.java diff --git a/impeller/core/platform.h b/impeller/core/platform.h index 416ea0eb41531..417a0f90c1a4d 100644 --- a/impeller/core/platform.h +++ b/impeller/core/platform.h @@ -14,10 +14,8 @@ namespace impeller { constexpr size_t DefaultUniformAlignment() { #if FML_OS_IOS && !TARGET_OS_SIMULATOR return 16u; -#elif FML_OS_MACOSX - return 256u; #else - return 0x40; + return 256u; #endif } diff --git a/impeller/renderer/backend/vulkan/BUILD.gn b/impeller/renderer/backend/vulkan/BUILD.gn index c54170f97d530..a0e44503e20d8 100644 --- a/impeller/renderer/backend/vulkan/BUILD.gn +++ b/impeller/renderer/backend/vulkan/BUILD.gn @@ -24,6 +24,8 @@ impeller_component("vulkan") { sources = [ "allocator_vk.cc", "allocator_vk.h", + "android_hardware_buffer_texture_source_vk.cc", + "android_hardware_buffer_texture_source_vk.h", "barrier_vk.cc", "barrier_vk.h", "blit_command_vk.cc", diff --git a/impeller/renderer/backend/vulkan/android_hardware_buffer_texture_source_vk.cc b/impeller/renderer/backend/vulkan/android_hardware_buffer_texture_source_vk.cc new file mode 100644 index 0000000000000..164d2b5276ac5 --- /dev/null +++ b/impeller/renderer/backend/vulkan/android_hardware_buffer_texture_source_vk.cc @@ -0,0 +1,203 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "impeller/renderer/backend/vulkan/android_hardware_buffer_texture_source_vk.h" +#include "impeller/renderer/backend/vulkan/texture_source_vk.h" + +#ifdef FML_OS_ANDROID + +namespace impeller { + +namespace { + +bool GetHardwareBufferProperties( + const vk::Device& device, + struct AHardwareBuffer* hardware_buffer, + ::impeller::vk::AndroidHardwareBufferPropertiesANDROID* ahb_props, + ::impeller::vk::AndroidHardwareBufferFormatPropertiesANDROID* + ahb_format_props) { + FML_CHECK(ahb_format_props != nullptr); + FML_CHECK(ahb_props != nullptr); + ahb_props->pNext = ahb_format_props; + ::impeller::vk::Result result = + device.getAndroidHardwareBufferPropertiesANDROID(hardware_buffer, + ahb_props); + if (result != impeller::vk::Result::eSuccess) { + return false; + } + return true; +} + +vk::ExternalFormatANDROID MakeExternalFormat( + const vk::AndroidHardwareBufferFormatPropertiesANDROID& format_props) { + vk::ExternalFormatANDROID external_format; + external_format.pNext = nullptr; + external_format.externalFormat = 0; + if (format_props.format == vk::Format::eUndefined) { + external_format.externalFormat = format_props.externalFormat; + } + return external_format; +} + +// Returns -1 if not found. +int FindMemoryTypeIndex( + const vk::AndroidHardwareBufferPropertiesANDROID& props) { + uint32_t memory_type_bits = props.memoryTypeBits; + int32_t type_index = -1; + for (uint32_t i = 0; memory_type_bits; + memory_type_bits = memory_type_bits >> 0x1, ++i) { + if (memory_type_bits & 0x1) { + type_index = i; + break; + } + } + return type_index; +} + +} // namespace + +AndroidHardwareBufferTextureSourceVK::AndroidHardwareBufferTextureSourceVK( + TextureDescriptor desc, + const vk::Device& device, + struct AHardwareBuffer* hardware_buffer, + const AHardwareBuffer_Desc& hardware_buffer_desc) + : TextureSourceVK(desc), device_(device) { + vk::AndroidHardwareBufferFormatPropertiesANDROID ahb_format_props; + vk::AndroidHardwareBufferPropertiesANDROID ahb_props; + if (!GetHardwareBufferProperties(device, hardware_buffer, &ahb_props, + &ahb_format_props)) { + return; + } + vk::ExternalFormatANDROID external_format = + MakeExternalFormat(ahb_format_props); + vk::ExternalMemoryImageCreateInfo external_memory_image_info; + external_memory_image_info.pNext = &external_format; + external_memory_image_info.handleTypes = + vk::ExternalMemoryHandleTypeFlagBits::eAndroidHardwareBufferANDROID; + const int memory_type_index = FindMemoryTypeIndex(ahb_props); + if (memory_type_index < 0) { + FML_LOG(ERROR) << "Could not find memory type."; + return; + } + + vk::ImageCreateFlags image_create_flags; + vk::ImageUsageFlags image_usage_flags; + if (hardware_buffer_desc.usage & AHARDWAREBUFFER_USAGE_GPU_SAMPLED_IMAGE) { + image_usage_flags |= impeller::vk::ImageUsageFlagBits::eSampled | + impeller::vk::ImageUsageFlagBits::eInputAttachment; + } + if (hardware_buffer_desc.usage & AHARDWAREBUFFER_USAGE_GPU_COLOR_OUTPUT) { + image_usage_flags |= impeller::vk::ImageUsageFlagBits::eColorAttachment; + } + if (hardware_buffer_desc.usage & AHARDWAREBUFFER_USAGE_PROTECTED_CONTENT) { + image_create_flags |= impeller::vk::ImageCreateFlagBits::eProtected; + } + + vk::ImageCreateInfo image_create_info; + image_create_info.pNext = &external_memory_image_info; + image_create_info.imageType = vk::ImageType::e2D; + image_create_info.format = ahb_format_props.format; + image_create_info.extent.width = hardware_buffer_desc.width; + image_create_info.extent.height = hardware_buffer_desc.height; + image_create_info.extent.depth = 1; + image_create_info.mipLevels = 1; + image_create_info.arrayLayers = 1; + image_create_info.samples = vk::SampleCountFlagBits::e1; + image_create_info.tiling = vk::ImageTiling::eOptimal; + image_create_info.usage = image_usage_flags; + image_create_info.flags = image_create_flags; + image_create_info.sharingMode = vk::SharingMode::eExclusive; + image_create_info.initialLayout = vk::ImageLayout::eUndefined; + + vk::ResultValue maybe_image = + device.createImage(image_create_info); + if (maybe_image.result != vk::Result::eSuccess) { + FML_LOG(ERROR) << "device.createImage failed: " + << static_cast(maybe_image.result); + return; + } + vk::Image image = maybe_image.value; + + vk::ImportAndroidHardwareBufferInfoANDROID ahb_import_info; + ahb_import_info.pNext = nullptr; + ahb_import_info.buffer = hardware_buffer; + + vk::MemoryDedicatedAllocateInfo dedicated_alloc_info; + dedicated_alloc_info.pNext = &ahb_import_info; + dedicated_alloc_info.image = image; + dedicated_alloc_info.buffer = VK_NULL_HANDLE; + + vk::MemoryAllocateInfo mem_alloc_info; + mem_alloc_info.pNext = &dedicated_alloc_info; + mem_alloc_info.allocationSize = ahb_props.allocationSize; + mem_alloc_info.memoryTypeIndex = memory_type_index; + + vk::ResultValue allocate_result = + device.allocateMemory(mem_alloc_info); + if (allocate_result.result != vk::Result::eSuccess) { + FML_LOG(ERROR) << "vkAllocateMemory failed : " + << static_cast(allocate_result.result); + device.destroyImage(image); + return; + } + vk::DeviceMemory device_memory = allocate_result.value; + + // Bind memory to the image object. + vk::Result bind_image_result = + device.bindImageMemory(image, device_memory, 0); + if (bind_image_result != vk::Result::eSuccess) { + FML_LOG(ERROR) << "vkBindImageMemory failed : " + << static_cast(bind_image_result); + device.destroyImage(image); + device.freeMemory(device_memory); + return; + } + image_ = image; + device_memory_ = device_memory; + + // Create image view. + vk::ImageViewCreateInfo view_info; + view_info.image = image_; + view_info.viewType = vk::ImageViewType::e2D; + view_info.format = ToVKImageFormat(desc.format); + view_info.subresourceRange.aspectMask = vk::ImageAspectFlagBits::eColor; + view_info.subresourceRange.baseMipLevel = 0u; + view_info.subresourceRange.baseArrayLayer = 0u; + view_info.subresourceRange.levelCount = desc.mip_count; + view_info.subresourceRange.layerCount = ToArrayLayerCount(desc.type); + auto [view_result, view] = device.createImageViewUnique(view_info); + if (view_result != vk::Result::eSuccess) { + FML_LOG(ERROR) << "createImageViewUnique failed : " + << static_cast(view_result); + return; + } + image_view_ = std::move(view); + is_valid_ = true; +} + +// |TextureSourceVK| +AndroidHardwareBufferTextureSourceVK::~AndroidHardwareBufferTextureSourceVK() { + device_.destroyImage(image_); + device_.freeMemory(device_memory_); +} + +bool AndroidHardwareBufferTextureSourceVK::IsValid() const { + return is_valid_; +} + +// |TextureSourceVK| +vk::Image AndroidHardwareBufferTextureSourceVK::GetImage() const { + FML_CHECK(IsValid()); + return image_; +} + +// |TextureSourceVK| +vk::ImageView AndroidHardwareBufferTextureSourceVK::GetImageView() const { + FML_CHECK(IsValid()); + return image_view_.get(); +} + +} // namespace impeller + +#endif diff --git a/impeller/renderer/backend/vulkan/android_hardware_buffer_texture_source_vk.h b/impeller/renderer/backend/vulkan/android_hardware_buffer_texture_source_vk.h new file mode 100644 index 0000000000000..10d77d8c594a3 --- /dev/null +++ b/impeller/renderer/backend/vulkan/android_hardware_buffer_texture_source_vk.h @@ -0,0 +1,55 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#pragma once + +#include "flutter/fml/build_config.h" +#include "vulkan/vulkan_core.h" + +#ifdef FML_OS_ANDROID + +#include "flutter/fml/macros.h" +#include "impeller/geometry/size.h" +#include "impeller/renderer/backend/vulkan/formats_vk.h" +#include "impeller/renderer/backend/vulkan/texture_source_vk.h" +#include "impeller/renderer/backend/vulkan/vk.h" + +#include +#include + +namespace impeller { + +class AndroidHardwareBufferTextureSourceVK final : public TextureSourceVK { + public: + AndroidHardwareBufferTextureSourceVK( + TextureDescriptor desc, + const vk::Device& device, + struct AHardwareBuffer* hardware_buffer, + const AHardwareBuffer_Desc& hardware_buffer_desc); + + // |TextureSourceVK| + ~AndroidHardwareBufferTextureSourceVK() override; + + // |TextureSourceVK| + vk::Image GetImage() const override; + + // |TextureSourceVK| + vk::ImageView GetImageView() const override; + + bool IsValid() const; + + private: + const vk::Device& device_; + vk::Image image_ = VK_NULL_HANDLE; + vk::UniqueImageView image_view_ = {}; + vk::DeviceMemory device_memory_ = VK_NULL_HANDLE; + + bool is_valid_ = false; + + FML_DISALLOW_COPY_AND_ASSIGN(AndroidHardwareBufferTextureSourceVK); +}; + +} // namespace impeller + +#endif diff --git a/impeller/renderer/backend/vulkan/capabilities_vk.cc b/impeller/renderer/backend/vulkan/capabilities_vk.cc index 33e3ca7ceb748..73e7ad08deb6f 100644 --- a/impeller/renderer/backend/vulkan/capabilities_vk.cc +++ b/impeller/renderer/backend/vulkan/capabilities_vk.cc @@ -206,6 +206,18 @@ CapabilitiesVK::GetEnabledDeviceExtensions( enabled.push_back("VK_KHR_portability_subset"); } +#ifdef FML_OS_ANDROID + if (exts->find("VK_ANDROID_external_memory_android_hardware_buffer") == + exts->end()) { + VALIDATION_LOG + << "Device does not support " + "VK_ANDROID_external_memory_android_hardware_buffer extension."; + return std::nullopt; + } + enabled.push_back("VK_ANDROID_external_memory_android_hardware_buffer"); + enabled.push_back("VK_EXT_queue_family_foreign"); +#endif + // Enable all optional extensions if the device supports it. IterateOptionalDeviceExtensions([&](auto ext) { auto ext_name = GetDeviceExtensionName(ext); diff --git a/shell/platform/android/BUILD.gn b/shell/platform/android/BUILD.gn index d2e201ea9f184..0895c3dc02286 100644 --- a/shell/platform/android/BUILD.gn +++ b/shell/platform/android/BUILD.gn @@ -98,8 +98,12 @@ source_set("flutter_shell_native_src") { "apk_asset_provider.h", "flutter_main.cc", "flutter_main.h", + "hardware_buffer_external_texture.cc", + "hardware_buffer_external_texture.h", "hardware_buffer_external_texture_gl.cc", "hardware_buffer_external_texture_gl.h", + "hardware_buffer_external_texture_vk.cc", + "hardware_buffer_external_texture_vk.h", "library_loader.cc", "ndk_helpers.cc", "ndk_helpers.h", diff --git a/shell/platform/android/hardware_buffer_external_texture.cc b/shell/platform/android/hardware_buffer_external_texture.cc new file mode 100644 index 0000000000000..f9cc5e15e8c86 --- /dev/null +++ b/shell/platform/android/hardware_buffer_external_texture.cc @@ -0,0 +1,106 @@ + +#include "flutter/shell/platform/android/hardware_buffer_external_texture.h" + +#include +#include +#include "shell/platform/android/ndk_helpers.h" + +namespace flutter { + +HardwareBufferExternalTexture::HardwareBufferExternalTexture( + int64_t id, + const fml::jni::ScopedJavaGlobalRef& image_texture_entry, + const std::shared_ptr& jni_facade) + : Texture(id), + image_texture_entry_(image_texture_entry), + jni_facade_(jni_facade) {} + +// Implementing flutter::Texture. +void HardwareBufferExternalTexture::Paint(PaintContext& context, + const SkRect& bounds, + bool freeze, + const DlImageSampling sampling) { + if (state_ == AttachmentState::kDetached) { + return; + } + const bool should_process_frame = + (!freeze && new_frame_ready_) || dl_image_ == nullptr; + if (should_process_frame) { + ProcessFrame(context, bounds); + new_frame_ready_ = false; + } + if (dl_image_) { + context.canvas->DrawImageRect( + dl_image_, // image + SkRect::Make(dl_image_->bounds()), // source rect + bounds, // destination rect + sampling, // sampling + context.paint, // paint + flutter::DlCanvas::SrcRectConstraint::kStrict // enforce edges + ); + } else { + FML_LOG(ERROR) << "No DlImage available."; + } +} + +// Implementing flutter::Texture. +void HardwareBufferExternalTexture::MarkNewFrameAvailable() { + new_frame_ready_ = true; +} + +// Implementing flutter::Texture. +void HardwareBufferExternalTexture::OnTextureUnregistered() {} + +// Implementing flutter::ContextListener. +void HardwareBufferExternalTexture::OnGrContextCreated() { + state_ = AttachmentState::kUninitialized; +} + +AHardwareBuffer* HardwareBufferExternalTexture::GetLatestHardwareBuffer() { + JNIEnv* env = fml::jni::AttachCurrentThread(); + FML_CHECK(env != nullptr); + + // ImageTextureEntry.acquireLatestImage. + JavaLocalRef image_java = jni_facade_->ImageTextureEntryAcquireLatestImage( + JavaLocalRef(image_texture_entry_)); + if (image_java.obj() == nullptr) { + return nullptr; + } + + // Image.getHardwareBuffer. + JavaLocalRef hardware_buffer_java = + jni_facade_->ImageGetHardwareBuffer(image_java); + if (hardware_buffer_java.obj() == nullptr) { + jni_facade_->ImageClose(image_java); + return nullptr; + } + + // Convert into NDK HardwareBuffer. + AHardwareBuffer* latest_hardware_buffer = + NDKHelpers::AHardwareBuffer_fromHardwareBuffer( + env, hardware_buffer_java.obj()); + if (latest_hardware_buffer == nullptr) { + return nullptr; + } + + // Keep hardware buffer alive. + NDKHelpers::AHardwareBuffer_acquire(latest_hardware_buffer); + + // Now that we have referenced the native hardware buffer, close the Java + // Image and HardwareBuffer objects. + jni_facade_->HardwareBufferClose(hardware_buffer_java); + jni_facade_->ImageClose(image_java); + + return latest_hardware_buffer; +} + +// Implementing flutter::ContextListener. +void HardwareBufferExternalTexture::OnGrContextDestroyed() { + if (state_ == AttachmentState::kAttached) { + dl_image_.reset(); + Detach(); + } + state_ = AttachmentState::kDetached; +} + +} // namespace flutter diff --git a/shell/platform/android/hardware_buffer_external_texture.h b/shell/platform/android/hardware_buffer_external_texture.h new file mode 100644 index 0000000000000..8f879d67a939f --- /dev/null +++ b/shell/platform/android/hardware_buffer_external_texture.h @@ -0,0 +1,63 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef FLUTTER_SHELL_PLATFORM_ANDROID_HARDWARE_BUFFER_EXTERNAL_TEXTURE_H_ +#define FLUTTER_SHELL_PLATFORM_ANDROID_HARDWARE_BUFFER_EXTERNAL_TEXTURE_H_ + +#include "flutter/common/graphics/texture.h" +#include "flutter/fml/logging.h" +#include "flutter/shell/platform/android/platform_view_android_jni_impl.h" + +#include +#include + +namespace flutter { + +class HardwareBufferExternalTexture : public flutter::Texture { + public: + explicit HardwareBufferExternalTexture( + int64_t id, + const fml::jni::ScopedJavaGlobalRef& + hardware_buffer_texture_entry, + const std::shared_ptr& jni_facade); + + // |flutter::Texture|. + void Paint(PaintContext& context, + const SkRect& bounds, + bool freeze, + const DlImageSampling sampling) override; + + // |flutter::Texture|. + void MarkNewFrameAvailable() override; + + // |flutter::Texture| + void OnTextureUnregistered() override; + + // |flutter::ContextListener| + void OnGrContextCreated() override; + + // |flutter::ContextListener| + void OnGrContextDestroyed() override; + + protected: + virtual void ProcessFrame(PaintContext& context, const SkRect& bounds) = 0; + virtual void Detach() = 0; + + AHardwareBuffer* GetLatestHardwareBuffer(); + + fml::jni::ScopedJavaGlobalRef image_texture_entry_; + std::shared_ptr jni_facade_; + + enum class AttachmentState { kUninitialized, kAttached, kDetached }; + AttachmentState state_ = AttachmentState::kUninitialized; + bool new_frame_ready_ = false; + + sk_sp dl_image_; + + FML_DISALLOW_COPY_AND_ASSIGN(HardwareBufferExternalTexture); +}; + +} // namespace flutter + +#endif // FLUTTER_SHELL_PLATFORM_ANDROID_HARDWARE_BUFFER_EXTERNAL_TEXTURE_H_ diff --git a/shell/platform/android/hardware_buffer_external_texture_gl.cc b/shell/platform/android/hardware_buffer_external_texture_gl.cc index 8cb74dccae675..3f3f6f220f6de 100644 --- a/shell/platform/android/hardware_buffer_external_texture_gl.cc +++ b/shell/platform/android/hardware_buffer_external_texture_gl.cc @@ -23,26 +23,13 @@ namespace flutter { -HardwareBufferExternalTextureGL::HardwareBufferExternalTextureGL( - const std::shared_ptr& context, - int64_t id, - const fml::jni::ScopedJavaGlobalRef& image_texture_entry, - const std::shared_ptr& jni_facade) - : Texture(id), - context_(context), - image_texture_entry_(image_texture_entry), - jni_facade_(jni_facade) {} - -HardwareBufferExternalTextureGL::~HardwareBufferExternalTextureGL() {} +void HardwareBufferExternalTextureGL::Detach() { + image_.reset(); + texture_.reset(); +} -// Implementing flutter::Texture. -void HardwareBufferExternalTextureGL::Paint(PaintContext& context, - const SkRect& bounds, - bool freeze, - const DlImageSampling sampling) { - if (state_ == AttachmentState::kDetached) { - return; - } +void HardwareBufferExternalTextureGL::ProcessFrame(PaintContext& context, + const SkRect& bounds) { if (state_ == AttachmentState::kUninitialized) { GLuint texture_name; glGenTextures(1, &texture_name); @@ -50,80 +37,7 @@ void HardwareBufferExternalTextureGL::Paint(PaintContext& context, state_ = AttachmentState::kAttached; } glBindTexture(GL_TEXTURE_EXTERNAL_OES, texture_.get().texture_name); - if (!freeze && new_frame_ready_) { - new_frame_ready_ = false; - Update(); - } - GrGLTextureInfo textureInfo = {GL_TEXTURE_EXTERNAL_OES, - texture_.get().texture_name, GL_RGBA8_OES}; - auto backendTexture = - GrBackendTextures::MakeGL(1, 1, skgpu::Mipmapped::kNo, textureInfo); - sk_sp image = SkImages::BorrowTextureFrom( - context.gr_context, backendTexture, kTopLeft_GrSurfaceOrigin, - kRGBA_8888_SkColorType, kPremul_SkAlphaType, nullptr); - if (image) { - DlAutoCanvasRestore autoRestore(context.canvas, true); - context.canvas->Translate(bounds.x(), bounds.y()); - context.canvas->Scale(bounds.width(), bounds.height()); - auto dl_image = DlImage::Make(image); - context.canvas->DrawImage(dl_image, {0, 0}, sampling, context.paint); - } else { - FML_LOG(ERROR) << "Skia could not borrow texture"; - } -} -// Implementing flutter::Texture. -void HardwareBufferExternalTextureGL::MarkNewFrameAvailable() { - new_frame_ready_ = true; -} - -// Implementing flutter::Texture. -void HardwareBufferExternalTextureGL::OnTextureUnregistered() {} - -// Implementing flutter::ContextListener. -void HardwareBufferExternalTextureGL::OnGrContextCreated() { - state_ = AttachmentState::kUninitialized; -} - -AHardwareBuffer* HardwareBufferExternalTextureGL::GetLatestHardwareBuffer() { - JNIEnv* env = fml::jni::AttachCurrentThread(); - FML_CHECK(env != nullptr); - - // ImageTextureEntry.acquireLatestImage. - JavaLocalRef image_java = jni_facade_->ImageTextureEntryAcquireLatestImage( - JavaLocalRef(image_texture_entry_)); - if (image_java.obj() == nullptr) { - return nullptr; - } - - // Image.getHardwareBuffer. - JavaLocalRef hardware_buffer_java = - jni_facade_->ImageGetHardwareBuffer(image_java); - if (hardware_buffer_java.obj() == nullptr) { - jni_facade_->ImageClose(image_java); - return nullptr; - } - - // Convert into NDK HardwareBuffer. - AHardwareBuffer* latest_hardware_buffer = - NDKHelpers::AHardwareBuffer_fromHardwareBuffer( - env, hardware_buffer_java.obj()); - if (latest_hardware_buffer == nullptr) { - return nullptr; - } - - // Keep hardware buffer alive. - NDKHelpers::AHardwareBuffer_acquire(latest_hardware_buffer); - - // Now that we have referenced the native hardware buffer, close the Java - // Image and HardwareBuffer objects. - jni_facade_->HardwareBufferClose(hardware_buffer_java); - jni_facade_->ImageClose(image_java); - - return latest_hardware_buffer; -} - -void HardwareBufferExternalTextureGL::Update() { EGLDisplay display = eglGetCurrentDisplay(); FML_CHECK(display != EGL_NO_DISPLAY); @@ -153,15 +67,23 @@ void HardwareBufferExternalTextureGL::Update() { // Drop our temporary reference to the hardware buffer as the call to // eglCreateImageKHR now has the reference. NDKHelpers::AHardwareBuffer_release(latest_hardware_buffer); -} -// Implementing flutter::ContextListener. -void HardwareBufferExternalTextureGL::OnGrContextDestroyed() { - if (state_ == AttachmentState::kAttached) { - image_.reset(); - texture_.reset(); - } - state_ = AttachmentState::kDetached; + GrGLTextureInfo textureInfo = {GL_TEXTURE_EXTERNAL_OES, + texture_.get().texture_name, GL_RGBA8_OES}; + auto backendTexture = + GrBackendTextures::MakeGL(1, 1, skgpu::Mipmapped::kNo, textureInfo); + dl_image_ = DlImage::Make(SkImages::BorrowTextureFrom( + context.gr_context, backendTexture, kTopLeft_GrSurfaceOrigin, + kRGBA_8888_SkColorType, kPremul_SkAlphaType, nullptr)); } +HardwareBufferExternalTextureGL::HardwareBufferExternalTextureGL( + const std::shared_ptr& context, + int64_t id, + const fml::jni::ScopedJavaGlobalRef& image_texture_entry, + const std::shared_ptr& jni_facade) + : HardwareBufferExternalTexture(id, image_texture_entry, jni_facade) {} + +HardwareBufferExternalTextureGL::~HardwareBufferExternalTextureGL() {} + } // namespace flutter diff --git a/shell/platform/android/hardware_buffer_external_texture_gl.h b/shell/platform/android/hardware_buffer_external_texture_gl.h index 6890f5bcb3007..b266c4d6ec25c 100644 --- a/shell/platform/android/hardware_buffer_external_texture_gl.h +++ b/shell/platform/android/hardware_buffer_external_texture_gl.h @@ -2,64 +2,34 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -#ifndef FLUTTER_SHELL_PLATFORM_HARDWARE_BUFFER_EXTERNAL_TEXTURE_GL_H_ -#define FLUTTER_SHELL_PLATFORM_HARDWARE_BUFFER_EXTERNAL_TEXTURE_GL_H_ +#ifndef FLUTTER_SHELL_PLATFORM_ANDROID_HARDWARE_BUFFER_EXTERNAL_TEXTURE_GL_H_ +#define FLUTTER_SHELL_PLATFORM_ANDROID_HARDWARE_BUFFER_EXTERNAL_TEXTURE_GL_H_ -#include +#include "flutter/shell/platform/android/hardware_buffer_external_texture.h" #include "flutter/impeller/toolkit/egl/egl.h" #include "flutter/impeller/toolkit/egl/image.h" #include "flutter/impeller/toolkit/gles/texture.h" -#include -#include - -#include "flutter/common/graphics/texture.h" -#include "flutter/fml/logging.h" #include "flutter/shell/platform/android/android_context_gl_skia.h" -#include "flutter/shell/platform/android/platform_view_android_jni_impl.h" namespace flutter { -class HardwareBufferExternalTextureGL : public flutter::Texture { +class HardwareBufferExternalTextureGL : public HardwareBufferExternalTexture { public: - explicit HardwareBufferExternalTextureGL( + HardwareBufferExternalTextureGL( const std::shared_ptr& context, int64_t id, const fml::jni::ScopedJavaGlobalRef& hardware_buffer_texture_entry, const std::shared_ptr& jni_facade); - ~HardwareBufferExternalTextureGL() override; - - // |flutter::Texture|. - void Paint(PaintContext& context, - const SkRect& bounds, - bool freeze, - const DlImageSampling sampling) override; - - // |flutter::Texture|. - void MarkNewFrameAvailable() override; - // |flutter::Texture| - void OnTextureUnregistered() override; - - // |flutter::ContextListener| - void OnGrContextCreated() override; - - // |flutter::ContextListener| - void OnGrContextDestroyed() override; + ~HardwareBufferExternalTextureGL() override; private: - AHardwareBuffer* GetLatestHardwareBuffer(); - void Update(); - - const std::shared_ptr context_; - fml::jni::ScopedJavaGlobalRef image_texture_entry_; - std::shared_ptr jni_facade_; + void ProcessFrame(PaintContext& context, const SkRect& bounds) override; + void Detach() override; - enum class AttachmentState { kUninitialized, kAttached, kDetached }; - AttachmentState state_ = AttachmentState::kUninitialized; - bool new_frame_ready_ = false; impeller::UniqueEGLImageKHR image_; impeller::UniqueGLTexture texture_; @@ -68,4 +38,4 @@ class HardwareBufferExternalTextureGL : public flutter::Texture { } // namespace flutter -#endif // FLUTTER_SHELL_PLATFORM_ANDROID_EXTERNAL_TEXTURE_GL_H_ +#endif // FLUTTER_SHELL_PLATFORM_ANDROID_HARDWARE_BUFFER_EXTERNAL_TEXTURE_GL_H_ diff --git a/shell/platform/android/hardware_buffer_external_texture_vk.cc b/shell/platform/android/hardware_buffer_external_texture_vk.cc new file mode 100644 index 0000000000000..c99441b37b580 --- /dev/null +++ b/shell/platform/android/hardware_buffer_external_texture_vk.cc @@ -0,0 +1,61 @@ + +#include "flutter/shell/platform/android/hardware_buffer_external_texture_vk.h" + +#include "flutter/impeller/renderer/backend/vulkan/android_hardware_buffer_texture_source_vk.h" +#include "flutter/impeller/renderer/backend/vulkan/texture_vk.h" +#include "flutter/shell/platform/android/ndk_helpers.h" +#include "impeller/core/formats.h" +#include "impeller/core/texture_descriptor.h" +#include "impeller/display_list/dl_image_impeller.h" + +namespace flutter { + +HardwareBufferExternalTextureVK::HardwareBufferExternalTextureVK( + const std::shared_ptr& impeller_context, + int64_t id, + const fml::jni::ScopedJavaGlobalRef& image_texture_entry, + const std::shared_ptr& jni_facade) + : HardwareBufferExternalTexture(id, image_texture_entry, jni_facade), + impeller_context_(impeller_context) {} + +HardwareBufferExternalTextureVK::~HardwareBufferExternalTextureVK() {} + +void HardwareBufferExternalTextureVK::ProcessFrame(PaintContext& context, + const SkRect& bounds) { + if (state_ == AttachmentState::kUninitialized) { + // First processed frame we are attached. + state_ = AttachmentState::kAttached; + } + + AHardwareBuffer* latest_hardware_buffer = GetLatestHardwareBuffer(); + if (latest_hardware_buffer == nullptr) { + FML_LOG(WARNING) << "GetLatestHardwareBuffer returned null."; + return; + } + + AHardwareBuffer_Desc hb_desc = {}; + flutter::NDKHelpers::AHardwareBuffer_describe(latest_hardware_buffer, + &hb_desc); + + impeller::TextureDescriptor desc; + desc.storage_mode = impeller::StorageMode::kDevicePrivate; + desc.size = {static_cast(bounds.width()), + static_cast(bounds.height())}; + // TODO(johnmccutchan): Use hb_desc to compute the correct format at runtime. + desc.format = impeller::PixelFormat::kR8G8B8A8UNormInt; + desc.mip_count = 1; + + auto texture_source = + std::make_shared( + desc, impeller_context_->GetDevice(), latest_hardware_buffer, + hb_desc); + + auto texture = + std::make_shared(impeller_context_, texture_source); + + dl_image_ = impeller::DlImageImpeller::Make(texture); +} + +void HardwareBufferExternalTextureVK::Detach() {} + +} // namespace flutter diff --git a/shell/platform/android/hardware_buffer_external_texture_vk.h b/shell/platform/android/hardware_buffer_external_texture_vk.h new file mode 100644 index 0000000000000..76c2779eff28d --- /dev/null +++ b/shell/platform/android/hardware_buffer_external_texture_vk.h @@ -0,0 +1,37 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef FLUTTER_SHELL_PLATFORM_ANDROID_HARDWARE_BUFFER_EXTERNAL_TEXTURE_VK_H_ +#define FLUTTER_SHELL_PLATFORM_ANDROID_HARDWARE_BUFFER_EXTERNAL_TEXTURE_VK_H_ + +#include "flutter/shell/platform/android/hardware_buffer_external_texture.h" + +#include "flutter/impeller/renderer/backend/vulkan/android_hardware_buffer_texture_source_vk.h" +#include "flutter/impeller/renderer/backend/vulkan/context_vk.h" +#include "flutter/impeller/renderer/backend/vulkan/vk.h" +#include "flutter/shell/platform/android/android_context_vulkan_impeller.h" + +namespace flutter { + +class HardwareBufferExternalTextureVK : public HardwareBufferExternalTexture { + public: + HardwareBufferExternalTextureVK( + const std::shared_ptr& impeller_context, + int64_t id, + const fml::jni::ScopedJavaGlobalRef& + hardware_buffer_texture_entry, + const std::shared_ptr& jni_facade); + + ~HardwareBufferExternalTextureVK() override; + + private: + void ProcessFrame(PaintContext& context, const SkRect& bounds) override; + void Detach() override; + + const std::shared_ptr impeller_context_; +}; + +} // namespace flutter + +#endif // FLUTTER_SHELL_PLATFORM_ANDROID_HARDWARE_BUFFER_EXTERNAL_TEXTURE_VK_H_ diff --git a/shell/platform/android/io/flutter/plugin/platform/ImageReaderPlatformViewRenderTarget.java b/shell/platform/android/io/flutter/plugin/platform/ImageReaderPlatformViewRenderTarget.java index 8a7c41b8b4c63..9dc93b0176d68 100644 --- a/shell/platform/android/io/flutter/plugin/platform/ImageReaderPlatformViewRenderTarget.java +++ b/shell/platform/android/io/flutter/plugin/platform/ImageReaderPlatformViewRenderTarget.java @@ -53,9 +53,12 @@ public void onImageAvailable(ImageReader reader) { protected ImageReader createImageReader33() { final ImageReader.Builder builder = new ImageReader.Builder(bufferWidth, bufferHeight); // Allow for double buffering. - builder.setMaxImages(2); - // Assume that we will be producing 32-bit RGBA values. - builder.setDefaultHardwareBufferFormat(HardwareBuffer.RGBA_8888); + builder.setMaxImages(3); + // Use PRIVATE image format so that we can support video decoding. + // TODO(johnmccutchan): Should we always use PRIVATE here? It may impact our ability + // to read back texture data. If we don't always want to use it, how do we decide when + // to use it or not? Perhaps PlatformViews can indicate if they may contain DRM'd content. + builder.setImageFormat(ImageFormat.PRIVATE); // Hint that consumed images will only be read by GPU. builder.setUsage(HardwareBuffer.USAGE_GPU_SAMPLED_IMAGE); final ImageReader reader = builder.build(); @@ -66,12 +69,12 @@ protected ImageReader createImageReader33() { @TargetApi(29) protected ImageReader createImageReader29() { return ImageReader.newInstance( - bufferWidth, bufferHeight, ImageFormat.UNKNOWN, 2, HardwareBuffer.USAGE_GPU_SAMPLED_IMAGE); + bufferWidth, bufferHeight, ImageFormat.PRIVATE, 2, HardwareBuffer.USAGE_GPU_SAMPLED_IMAGE); } @TargetApi(26) protected ImageReader createImageReader26() { - return ImageReader.newInstance(bufferWidth, bufferHeight, ImageFormat.UNKNOWN, 2); + return ImageReader.newInstance(bufferWidth, bufferHeight, ImageFormat.PRIVATE, 2); } protected ImageReader createImageReader() { diff --git a/shell/platform/android/io/flutter/plugin/platform/PlatformViewsController.java b/shell/platform/android/io/flutter/plugin/platform/PlatformViewsController.java index b0600b65bd22a..f9500cdf00c20 100644 --- a/shell/platform/android/io/flutter/plugin/platform/PlatformViewsController.java +++ b/shell/platform/android/io/flutter/plugin/platform/PlatformViewsController.java @@ -357,11 +357,10 @@ public void resize( // Resize the buffer only when the current buffer size is smaller than the new size. // This is required to prevent a situation when smooth keyboard animation // resizes the texture too often, such that the GPU and the platform thread don't agree on - // the - // timing of the new size. + // the timing of the new size. // Resizing the texture causes pixel stretching since the size of the GL texture used in - // the engine - // is set by the framework, but the texture buffer size is set by the platform down below. + // the engine is set by the framework, but the texture buffer size is set by the + // platform down below. if (physicalWidth > viewWrapper.getRenderTargetWidth() || physicalHeight > viewWrapper.getRenderTargetHeight()) { viewWrapper.resizeRenderTarget(physicalWidth, physicalHeight); diff --git a/shell/platform/android/ndk_helpers.cc b/shell/platform/android/ndk_helpers.cc index 568be5469ab74..89291f6b660e6 100644 --- a/shell/platform/android/ndk_helpers.cc +++ b/shell/platform/android/ndk_helpers.cc @@ -9,6 +9,7 @@ #include "flutter/fml/logging.h" +#include #include namespace flutter { @@ -20,6 +21,8 @@ typedef AHardwareBuffer* (*fp_AHardwareBuffer_fromHardwareBuffer)( jobject hardwareBufferObj); typedef void (*fp_AHardwareBuffer_acquire)(AHardwareBuffer* buffer); typedef void (*fp_AHardwareBuffer_release)(AHardwareBuffer* buffer); +typedef void (*fp_AHardwareBuffer_describe)(AHardwareBuffer* buffer, + AHardwareBuffer_Desc* desc); typedef EGLClientBuffer (*fp_eglGetNativeClientBufferANDROID)( AHardwareBuffer* buffer); @@ -28,6 +31,8 @@ AHardwareBuffer* (*_AHardwareBuffer_fromHardwareBuffer)( jobject hardwareBufferObj) = nullptr; void (*_AHardwareBuffer_acquire)(AHardwareBuffer* buffer) = nullptr; void (*_AHardwareBuffer_release)(AHardwareBuffer* buffer) = nullptr; +void (*_AHardwareBuffer_describe)(AHardwareBuffer* buffer, + AHardwareBuffer_Desc* desc) = nullptr; EGLClientBuffer (*_eglGetNativeClientBufferANDROID)(AHardwareBuffer* buffer) = nullptr; @@ -57,6 +62,11 @@ void InitOnceCallback() { ->ResolveFunction( "AHardwareBuffer_release") .value_or(nullptr); + _AHardwareBuffer_describe = + android + ->ResolveFunction( + "AHardwareBuffer_describe") + .value_or(nullptr); } } // namespace @@ -91,6 +101,13 @@ void NDKHelpers::AHardwareBuffer_release(AHardwareBuffer* buffer) { _AHardwareBuffer_release(buffer); } +void NDKHelpers::AHardwareBuffer_describe(AHardwareBuffer* buffer, + AHardwareBuffer_Desc* desc) { + NDKHelpers::Init(); + FML_CHECK(_AHardwareBuffer_describe != nullptr); + _AHardwareBuffer_describe(buffer, desc); +} + EGLClientBuffer NDKHelpers::eglGetNativeClientBufferANDROID( AHardwareBuffer* buffer) { NDKHelpers::Init(); diff --git a/shell/platform/android/ndk_helpers.h b/shell/platform/android/ndk_helpers.h index 3ec0500d99e9a..f1c2592467dea 100644 --- a/shell/platform/android/ndk_helpers.h +++ b/shell/platform/android/ndk_helpers.h @@ -25,6 +25,8 @@ class NDKHelpers { jobject hardwareBufferObj); static void AHardwareBuffer_acquire(AHardwareBuffer* buffer); static void AHardwareBuffer_release(AHardwareBuffer* buffer); + static void AHardwareBuffer_describe(AHardwareBuffer* buffer, + AHardwareBuffer_Desc* desc); static EGLClientBuffer eglGetNativeClientBufferANDROID( AHardwareBuffer* buffer); diff --git a/shell/platform/android/platform_view_android.cc b/shell/platform/android/platform_view_android.cc index ff941ef81dc3c..936761b3345c3 100644 --- a/shell/platform/android/platform_view_android.cc +++ b/shell/platform/android/platform_view_android.cc @@ -18,6 +18,8 @@ #include "flutter/shell/platform/android/android_surface_gl_skia.h" #include "flutter/shell/platform/android/android_surface_software.h" #include "flutter/shell/platform/android/hardware_buffer_external_texture_gl.h" +#include "flutter/shell/platform/android/hardware_buffer_external_texture_vk.h" +#include "shell/platform/android/android_context_vulkan_impeller.h" #if IMPELLER_ENABLE_VULKAN // b/258506856 for why this is behind an if #include "flutter/shell/platform/android/android_surface_vulkan_impeller.h" #endif @@ -313,8 +315,14 @@ void PlatformViewAndroid::RegisterImageTexture( RegisterTexture(std::make_shared( std::static_pointer_cast(android_context_), texture_id, image_texture_entry, jni_facade_)); + } else if (android_context_->RenderingApi() == AndroidRenderingAPI::kVulkan) { + RegisterTexture(std::make_shared( + std::static_pointer_cast( + android_context_->GetImpellerContext()), + texture_id, image_texture_entry, jni_facade_)); } else { - FML_LOG(INFO) << "Attempted to use a GL texture in a non GL context."; + FML_LOG(INFO) << "Attempted to use a HardwareBuffer texture with an " + "unsupported rendering API."; } }