From 88599d15cf9a62af223cda9f6de7d559ccb86ec2 Mon Sep 17 00:00:00 2001 From: Brandon DeRosier Date: Sat, 29 Oct 2022 21:55:31 -0700 Subject: [PATCH 01/11] APNG decoder --- fml/endianness.h | 14 +- lib/ui/BUILD.gn | 3 + lib/ui/painting/image_decoder_unittests.cc | 2 +- lib/ui/painting/image_generator.cc | 4 +- lib/ui/painting/image_generator.h | 12 +- lib/ui/painting/image_generator_apng.cc | 561 ++++++++++++++++++ lib/ui/painting/image_generator_apng.h | 223 +++++++ lib/ui/painting/image_generator_registry.cc | 8 + .../image_generator_registry_unittests.cc | 2 +- lib/ui/painting/multi_frame_codec.cc | 6 + shell/common/shell_unittests.cc | 2 +- .../android/android_image_generator.cc | 2 +- .../android/android_image_generator.h | 2 +- 13 files changed, 827 insertions(+), 14 deletions(-) create mode 100644 lib/ui/painting/image_generator_apng.cc create mode 100644 lib/ui/painting/image_generator_apng.h diff --git a/fml/endianness.h b/fml/endianness.h index 269b461492184..eff90f9cafb3c 100644 --- a/fml/endianness.h +++ b/fml/endianness.h @@ -26,9 +26,17 @@ namespace fml { +template +struct IsByteSwappable + : public std:: + integral_constant || std::is_enum_v> { +}; +template +constexpr bool IsByteSwappableV = IsByteSwappable::value; + /// @brief Flips the endianness of the given value. /// The given value must be an integral type of size 1, 2, 4, or 8. -template >> +template >> constexpr T ByteSwap(T n) { if constexpr (sizeof(T) == 1) { return n; @@ -47,7 +55,7 @@ constexpr T ByteSwap(T n) { /// current architecture. This is effectively a cross platform /// ntohl/ntohs (as network byte order is always Big Endian). /// The given value must be an integral type of size 1, 2, 4, or 8. -template >> +template >> constexpr T BigEndianToArch(T n) { #if FML_ARCH_CPU_LITTLE_ENDIAN return ByteSwap(n); @@ -59,7 +67,7 @@ constexpr T BigEndianToArch(T n) { /// @brief Convert a known little endian value to match the endianness of the /// current architecture. /// The given value must be an integral type of size 1, 2, 4, or 8. -template >> +template >> constexpr T LittleEndianToArch(T n) { #if !FML_ARCH_CPU_LITTLE_ENDIAN return ByteSwap(n); diff --git a/lib/ui/BUILD.gn b/lib/ui/BUILD.gn index adaec644f017b..1556126b2d5e4 100644 --- a/lib/ui/BUILD.gn +++ b/lib/ui/BUILD.gn @@ -59,6 +59,8 @@ source_set("ui") { "painting/image_filter.h", "painting/image_generator.cc", "painting/image_generator.h", + "painting/image_generator_apng.cc", + "painting/image_generator_apng.h", "painting/image_generator_registry.cc", "painting/image_generator_registry.h", "painting/image_shader.cc", @@ -161,6 +163,7 @@ source_set("ui") { "//third_party/dart/runtime/bin:dart_io_api", "//third_party/rapidjson", "//third_party/skia", + "//third_party/zlib:zlib", ] if (impeller_supports_rendering) { diff --git a/lib/ui/painting/image_decoder_unittests.cc b/lib/ui/painting/image_decoder_unittests.cc index 7a5e3cfd0aa00..4f91282281b28 100644 --- a/lib/ui/painting/image_decoder_unittests.cc +++ b/lib/ui/painting/image_decoder_unittests.cc @@ -162,7 +162,7 @@ class UnknownImageGenerator : public ImageGenerator { unsigned int GetPlayCount() const { return 1; } - const ImageGenerator::FrameInfo GetFrameInfo(unsigned int frame_index) const { + const ImageGenerator::FrameInfo GetFrameInfo(unsigned int frame_index) { return {std::nullopt, 0, SkCodecAnimation::DisposalMethod::kKeep}; } diff --git a/lib/ui/painting/image_generator.cc b/lib/ui/painting/image_generator.cc index e5648fc24a97b..3db7625c741df 100644 --- a/lib/ui/painting/image_generator.cc +++ b/lib/ui/painting/image_generator.cc @@ -51,7 +51,7 @@ unsigned int BuiltinSkiaImageGenerator::GetPlayCount() const { } const ImageGenerator::FrameInfo BuiltinSkiaImageGenerator::GetFrameInfo( - unsigned int frame_index) const { + unsigned int frame_index) { return {.required_frame = std::nullopt, .duration = 0, .disposal_method = SkCodecAnimation::DisposalMethod::kKeep}; @@ -105,7 +105,7 @@ unsigned int BuiltinSkiaCodecImageGenerator::GetPlayCount() const { } const ImageGenerator::FrameInfo BuiltinSkiaCodecImageGenerator::GetFrameInfo( - unsigned int frame_index) const { + unsigned int frame_index) { SkCodec::FrameInfo info = {}; codec_generator_->getFrameInfo(frame_index, &info); return { diff --git a/lib/ui/painting/image_generator.h b/lib/ui/painting/image_generator.h index 4792f3b374b92..a5515c8057319 100644 --- a/lib/ui/painting/image_generator.h +++ b/lib/ui/painting/image_generator.h @@ -37,11 +37,15 @@ class ImageGenerator { /// blended with. std::optional required_frame; - /// Number of milliseconds to show this frame. + /// Number of milliseconds to show this frame. 0 means only show it for one + /// frame. unsigned int duration; /// How this frame should be modified before decoding the next one. SkCodecAnimation::DisposalMethod disposal_method; + + /// How this frame should be blended with the previous frame. + SkCodecAnimation::Blend blend_mode; }; virtual ~ImageGenerator(); @@ -80,7 +84,7 @@ class ImageGenerator { /// @return Information about the given frame. If the image is /// single-frame, a default result is returned. /// @see `GetFrameCount` - virtual const FrameInfo GetFrameInfo(unsigned int frame_index) const = 0; + virtual const FrameInfo GetFrameInfo(unsigned int frame_index) = 0; /// @brief Given a scale value, find the closest image size that can be /// used for efficiently decoding the image. If subpixel image @@ -152,7 +156,7 @@ class BuiltinSkiaImageGenerator : public ImageGenerator { // |ImageGenerator| const ImageGenerator::FrameInfo GetFrameInfo( - unsigned int frame_index) const override; + unsigned int frame_index) override; // |ImageGenerator| SkISize GetScaledDimensions(float desired_scale) override; @@ -192,7 +196,7 @@ class BuiltinSkiaCodecImageGenerator : public ImageGenerator { // |ImageGenerator| const ImageGenerator::FrameInfo GetFrameInfo( - unsigned int frame_index) const override; + unsigned int frame_index) override; // |ImageGenerator| SkISize GetScaledDimensions(float desired_scale) override; diff --git a/lib/ui/painting/image_generator_apng.cc b/lib/ui/painting/image_generator_apng.cc new file mode 100644 index 0000000000000..91c5f103f5b33 --- /dev/null +++ b/lib/ui/painting/image_generator_apng.cc @@ -0,0 +1,561 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "image_generator_apng.h" +#include + +#include "third_party/libpng/png.h" +#include "third_party/skia/include/codec/SkCodecAnimation.h" +#include "third_party/skia/src/codec/SkPngCodec.h" +#include "third_party/skia/src/core/SkRasterPipeline.h" +#include "third_party/zlib/zlib.h" // For crc32 + +namespace flutter { + +APNGImageGenerator::~APNGImageGenerator() = default; + +APNGImageGenerator::APNGImageGenerator(sk_sp& data, + SkImageInfo& image_info, + APNGImage&& default_image, + unsigned int frame_count, + unsigned int play_count, + const void* next_chunk_p, + const std::vector header) + : data_(data), + image_info_(image_info), + frame_count_(frame_count), + play_count_(play_count), + first_frame_index_(default_image.frame_info.has_value() ? 0 : 1), + next_chunk_p_(next_chunk_p), + header_(header) { + images_.push_back(std::move(default_image)); +} + +const SkImageInfo& APNGImageGenerator::GetInfo() { + return image_info_; +} + +unsigned int APNGImageGenerator::GetFrameCount() const { + return frame_count_; +} + +unsigned int APNGImageGenerator::GetPlayCount() const { + return frame_count_ > 1 ? play_count_ : 1; +} + +const ImageGenerator::FrameInfo APNGImageGenerator::GetFrameInfo( + unsigned int frame_index) { + unsigned int image_index = first_frame_index_ + frame_index; + if (!DemuxToImageIndex(image_index)) { + return {}; + } + + return images_[image_index].frame_info.value(); +} + +SkISize APNGImageGenerator::GetScaledDimensions(float desired_scale) { + return image_info_.dimensions(); +} + +bool APNGImageGenerator::GetPixels(const SkImageInfo& info, + void* pixels, + size_t row_bytes, + unsigned int frame_index, + std::optional prior_frame) { + FML_DCHECK(images_.size() > 0); + unsigned int image_index = first_frame_index_ + frame_index; + + //---------------------------------------------------------------------------- + /// 1. Demux the frame from the APNG stream. + /// + + if (!DemuxToImageIndex(image_index)) { + FML_DLOG(ERROR) << "Couldn't demux image at index " << image_index + << " (frame index: " << frame_index + << ") from APNG stream."; + return RenderDefaultImage(info, pixels, row_bytes); + } + + //---------------------------------------------------------------------------- + /// 2. Decode the frame. + /// + + APNGImage& frame = images_[image_index]; + auto frame_info = frame.codec->getInfo(); + auto frame_row_bytes = frame_info.bytesPerPixel() * frame_info.width(); + + if (frame.pixels.empty()) { + frame.pixels.resize(frame_row_bytes * frame_info.height()); + SkCodec::Result result = frame.codec->getPixels( + frame.codec->getInfo(), frame.pixels.data(), frame_row_bytes); + if (result != SkCodec::kSuccess) { + FML_DLOG(ERROR) << "Failed to decode image at index " << image_index + << " (frame index: " << frame_index + << ") of APNG. SkCodec::Result: " << result; + return RenderDefaultImage(info, pixels, row_bytes); + } + } + + //---------------------------------------------------------------------------- + /// 3. Composite the frame onto the canvas. + /// + + for (int i = 0; i < frame_info.height(); i++) { + void* source = frame.pixels.data() + i * frame_row_bytes; + void* destination = static_cast(pixels) + + frame.x_offset * frame_info.bytesPerPixel() + + (i + frame.y_offset) * row_bytes; + + BlendLine(info.colorType(), destination, frame_info.colorType(), source, + info.alphaType(), frame.frame_info->blend_mode, + frame_info.width()); + } + + return true; +} + +std::unique_ptr APNGImageGenerator::MakeFromData( + sk_sp data) { + // Ensure the buffer is large enough to at least contain the PNG signature + // and a chunk header. + if (data->size() < 8 + sizeof(ChunkHeader)) { + return nullptr; + } + // Validate the full PNG signature. + const uint8_t* data_p = static_cast(data.get()->data()); + if (png_sig_cmp(static_cast(data_p), 0, 8)) { + return nullptr; + } + + // Validate the header chunk. + const ChunkHeader* chunk = reinterpret_cast(data_p + 8); + if (!IsValidChunkHeader(data_p, data->size(), chunk) || + chunk->get_data_length() != sizeof(ImageHeaderChunkData) || + chunk->get_type() != kImageHeaderChunkType) { + return nullptr; + } + + // Walk the chunks to find the "animation control" chunk. If an "image data" + // chunk is found first, this PNG is not animated. + do { + if (chunk->get_type() == kImageDataChunkType) { + return nullptr; + } else if (chunk->get_type() == kAnimationControlChunkType) { + break; + } + + chunk = GetNextChunk(data_p, data->size(), chunk); + } while (chunk != nullptr); + if (chunk == nullptr) { + return nullptr; + } + + const AnimationControlChunkData* animation_data = + CastChunkData(chunk); + + // Extract the header signature and chunks to prepend when demuxing images. + std::optional> header; + const void* first_chunk_p; + std::tie(header, first_chunk_p) = ExtractHeader(data_p, data->size()); + if (!header.has_value()) { + return nullptr; + } + + // Demux the first image in the APNG chunk stream in order to interpret + // extent and blending info immediately. + std::optional default_image; + const void* next_chunk_p; + std::tie(default_image, next_chunk_p) = + DemuxNextImage(data_p, data->size(), header.value(), first_chunk_p); + if (default_image == std::nullopt) { + return nullptr; + } + + unsigned int play_count = animation_data->get_num_plays(); + if (play_count == 0) { + play_count = kInfinitePlayCount; + } + + SkImageInfo image_info = default_image.value().codec->getInfo(); + return std::unique_ptr( + new APNGImageGenerator(data, image_info, std::move(default_image.value()), + animation_data->get_num_frames(), play_count, + next_chunk_p, header.value())); +} + +bool APNGImageGenerator::IsValidChunkHeader(const void* buffer, + size_t size, + const ChunkHeader* chunk) { + // Ensure the chunk doesn't start before the beginning of the buffer. + if (reinterpret_cast(chunk) < + static_cast(buffer)) { + return false; + } + + // Ensure the buffer is large enough to contain at least the chunk header. + if (reinterpret_cast(chunk) + sizeof(ChunkHeader) > + static_cast(buffer) + size) { + return false; + } + + // Ensure the buffer is large enough to contain the chunk's given data size + // and CRC. + const uint8_t* chunk_end = + reinterpret_cast(chunk) + GetChunkSize(chunk); + if (chunk_end > static_cast(buffer) + size) { + return false; + } + + // Ensure the 4-byte type only contains ISO 646 letters. + uint32_t type = chunk->get_type(); + for (int i = 0; i < 4; i++) { + uint8_t c = type >> i * 8 & 0xFF; + if (!((c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z'))) { + return false; + } + } + + return true; +} + +const APNGImageGenerator::ChunkHeader* APNGImageGenerator::GetNextChunk( + const void* buffer, + size_t size, + const ChunkHeader* current_chunk) { + FML_DCHECK((uint8_t*)current_chunk + sizeof(ChunkHeader) <= + (uint8_t*)buffer + size); + + const ChunkHeader* next_chunk = reinterpret_cast( + reinterpret_cast(current_chunk) + + GetChunkSize(current_chunk)); + if (!IsValidChunkHeader(buffer, size, next_chunk)) { + return nullptr; + } + + return next_chunk; +} + +std::pair>, const void*> +APNGImageGenerator::ExtractHeader(const void* buffer_p, size_t buffer_size) { + std::vector result = {137, 80, 78, 71, + 13, 10, 26, 10}; // PNG signature + + const ChunkHeader* chunk = reinterpret_cast( + static_cast(buffer_p) + 8); + // Validate the first chunk to ensure it's safe to read. + if (!IsValidChunkHeader(buffer_p, buffer_size, chunk)) { + return std::make_pair(std::nullopt, nullptr); + } + + // Walk the chunks and copy in the non-APNG chunks until we come across a + // frame or image chunk. + do { + if (chunk->get_type() != kAnimationControlChunkType) { + size_t chunk_size = GetChunkSize(chunk); + result.resize(result.size() + chunk_size); + memcpy(result.data() + result.size() - chunk_size, chunk, chunk_size); + } + + chunk = GetNextChunk(buffer_p, buffer_size, chunk); + } while (chunk != nullptr && chunk->get_type() != kFrameControlChunkType && + chunk->get_type() != kImageDataChunkType && + chunk->get_type() != kFrameDataChunkType); + + // nullptr means the end of the buffer was reached, which means there's no + // frame or image data, so just return nothing because the PNG isn't even + // valid. + if (chunk == nullptr) { + return std::make_pair(std::nullopt, nullptr); + } + + return std::make_pair(result, chunk); +} + +std::pair, const void*> +APNGImageGenerator::DemuxNextImage(const void* buffer_p, + size_t buffer_size, + const std::vector header, + const void* chunk_p) { + const ChunkHeader* chunk = reinterpret_cast(chunk_p); + // Validate the given chunk to ensure it's safe to read. + if (!IsValidChunkHeader(buffer_p, buffer_size, chunk)) { + return std::make_pair(std::nullopt, nullptr); + } + + // Expect frame data to begin at fdAT or IDAT + if (chunk->get_type() != kFrameControlChunkType && + chunk->get_type() != kImageDataChunkType) { + return std::make_pair(std::nullopt, nullptr); + } + + APNGImage result; + const FrameControlChunkData* control_data = nullptr; + + // The presence of an fcTL chunk is optional for the first (default) image + // of a PNG. Both cases are handled in APNGImage. + if (chunk->get_type() == kFrameControlChunkType) { + control_data = CastChunkData(chunk); + + ImageGenerator::FrameInfo frame_info; + switch (control_data->get_blend_op()) { + case 0: // APNG_BLEND_OP_SOURCE + frame_info.blend_mode = SkCodecAnimation::Blend::kSrc; + break; + case 1: // APNG_BLEND_OP_OVER + frame_info.blend_mode = SkCodecAnimation::Blend::kSrcOver; + break; + default: + return std::make_pair(std::nullopt, nullptr); + } + switch (control_data->get_dispose_op()) { + case 0: // APNG_DISPOSE_OP_NONE + frame_info.disposal_method = SkCodecAnimation::DisposalMethod::kKeep; + break; + case 1: // APNG_DISPOSE_OP_BACKGROUND + frame_info.disposal_method = + SkCodecAnimation::DisposalMethod::kRestoreBGColor; + break; + case 2: // APNG_DISPOSE_OP_PREVIOUS + frame_info.disposal_method = + SkCodecAnimation::DisposalMethod::kRestorePrevious; + break; + default: + return std::make_pair(std::nullopt, nullptr); + } + uint16_t denominator = control_data->get_delay_den() == 0 + ? 100 + : control_data->get_delay_den(); + frame_info.duration = + static_cast(control_data->get_delay_num() * 1000.f / denominator); + + // TODO(bdero): Populate frame_info.required_frame depending on the + // blend_mode and disposal_method. + + result.frame_info = frame_info; + result.x_offset = control_data->get_x_offset(); + result.y_offset = control_data->get_y_offset(); + } + + std::vector image_chunks; + size_t chunk_space = 0; + + // Walk the chunks until the next frame, end chunk, or an invalid chunk is + // reached, recording the chunks to copy along with their required space. + // TODO(bdero): Validate that IDAT/fdAT chunks are contiguous. + // TODO(bdero): Validate the acTL/fcTL/fdAT sequence number ordering. + do { + if (chunk->get_type() != kFrameControlChunkType) { + image_chunks.push_back(chunk); + chunk_space += GetChunkSize(chunk); + + // fdAT chunks are converted into IDAT chunks when demuxed. The only + // difference between these chunk types is that fdAT has a 4 byte + // sequence number prepended to its data, so subtract that space from + // the buffer. + if (chunk->get_type() == kFrameDataChunkType) { + chunk_space -= 4; + } + } + + chunk = GetNextChunk(buffer_p, buffer_size, chunk); + } while (chunk != nullptr && chunk->get_type() != kFrameControlChunkType && + chunk->get_type() != kImageTrailerChunkType); + + const uint8_t end_chunk[] = {0, 0, 0, 0, 'I', 'E', + 'N', 'D', 0xAE, 0x42, 0x60, 0x82}; + + // Form a buffer for the new encoded PNG and copy the chunks in. + sk_sp new_png_buffer = SkData::MakeUninitialized( + header.size() + chunk_space + sizeof(end_chunk)); + + { + uint8_t* write_cursor = + static_cast(new_png_buffer->writable_data()); + + // Copy the signature/header chunks + memcpy(write_cursor, header.data(), header.size()); + // If this is a frame, override the width/height in the IHDR chunk. + if (control_data) { + ChunkHeader* ihdr_header = + reinterpret_cast(write_cursor + 8); + ImageHeaderChunkData* ihdr_data = const_cast( + CastChunkData(ihdr_header)); + ihdr_data->set_width(control_data->get_width()); + ihdr_data->set_height(control_data->get_height()); + ihdr_header->UpdateChunkCrc32(); + } + write_cursor += header.size(); + + // Copy the image data/ancillary chunks. + for (const ChunkHeader* c : image_chunks) { + if (c->get_type() == kFrameDataChunkType) { + // Write a new IDAT chunk header. + ChunkHeader* write_header = + reinterpret_cast(write_cursor); + write_header->set_data_length(c->get_data_length() - 4); + write_header->set_type(kImageDataChunkType); + write_cursor += sizeof(ChunkHeader); + + // Copy all of the data except for the 4 byte sequence number at the + // beginning of the fdAT data. + memcpy(write_cursor, + reinterpret_cast(c) + sizeof(ChunkHeader) + 4, + write_header->get_data_length()); + write_cursor += write_header->get_data_length(); + + // Recompute the chunk CRC. + write_header->UpdateChunkCrc32(); + write_cursor += 4; + } else { + size_t chunk_size = GetChunkSize(c); + memcpy(write_cursor, c, chunk_size); + write_cursor += chunk_size; + } + } + + // Copy the trailer chunk. + memcpy(write_cursor, &end_chunk, sizeof(end_chunk)); + } + + SkCodec::Result header_parse_result; + result.codec = SkPngCodec::MakeFromStream( + SkMemoryStream::Make(new_png_buffer), &header_parse_result); + if (header_parse_result != SkCodec::Result::kSuccess) { + FML_DLOG(ERROR) + << "Failed to parse image header during APNG demux. SkCodec::Result: " + << header_parse_result; + return std::make_pair(std::nullopt, nullptr); + } + + if (chunk->get_type() == kImageTrailerChunkType) { + chunk = nullptr; + } + + return std::make_pair(std::optional{std::move(result)}, chunk); +} + +bool APNGImageGenerator::DemuxNextImageInternal() { + if (next_chunk_p_ == nullptr) { + return false; + } + + std::optional image; + const void* data_p = const_cast(data_.get()->data()); + std::tie(image, next_chunk_p_) = + DemuxNextImage(data_p, data_->size(), header_, next_chunk_p_); + if (!image.has_value()) { + return false; + } + + if (!images_.empty() && images_.back().frame_info->disposal_method == + SkCodecAnimation::DisposalMethod::kKeep) { + image->frame_info->required_frame = images_.size() - 1; + } + + // Calling SkCodec::getInfo at least once prior to decoding is mandatory. + SkImageInfo info = image.value().codec->getInfo(); + FML_DCHECK(info.colorInfo() == image_info_.colorInfo()); + + images_.push_back(std::move(image.value())); + + auto default_info = images_[0].codec->getInfo(); + if (info.colorType() != default_info.colorType()) { + return false; + } + return true; +} + +bool APNGImageGenerator::DemuxToImageIndex(unsigned int image_index) { + // If the requested image doesn't exist yet, demux more frames from the APNG + // stream. + if (image_index >= images_.size()) { + while (DemuxNextImageInternal() && image_index >= images_.size()) { + } + + if (image_index >= images_.size()) { + // The chunk stream was exhausted before the image was found. + return false; + } + } + + return true; +} + +void APNGImageGenerator::ChunkHeader::UpdateChunkCrc32() { + uint32_t* crc_p = + reinterpret_cast(reinterpret_cast(this) + + sizeof(ChunkHeader) + get_data_length()); + *crc_p = fml::BigEndianToArch(ComputeChunkCrc32()); +} + +uint32_t APNGImageGenerator::ChunkHeader::ComputeChunkCrc32() { + // Exclude the length field at the beginning of the chunk header. + size_t length = sizeof(ChunkHeader) - 4 + get_data_length(); + uint8_t* chunk_data_p = reinterpret_cast(this) + 4; + uint32_t crc = 0; + + // zlib's crc32 can only takes 16 bits at a time for the length, but PNG + // supports a 32 bit chunk length, so looping is necessary here. + // Note that crc32 is always called at least once, even if the chunk has an + // empty data section. + do { + uint16_t length16 = length; + if (length16 == 0 && length > 0) { + length16 = std::numeric_limits::max(); + } + + crc = crc32(crc, chunk_data_p, length16); + length -= length16; + chunk_data_p += length16; + } while (length > 0); + + return crc; +} + +void APNGImageGenerator::BlendLine(SkColorType dest_colortype, + void* dest, + SkColorType source_colortype, + const void* source, + SkAlphaType dest_alphatype, + SkCodecAnimation::Blend blend_mode, + int width) { + SkRasterPipeline_MemoryCtx dst_ctx = {dest, 0}, + src_ctx = {const_cast(source), 0}; + + SkRasterPipeline_<256> p; + + p.append_load_dst(dest_colortype, &dst_ctx); + if (kUnpremul_SkAlphaType == dest_alphatype) { + p.append(SkRasterPipeline::premul_dst); + } + + p.append_load(source_colortype, &src_ctx); + p.append(SkRasterPipeline::premul); + + if (blend_mode == SkCodecAnimation::Blend::kSrcOver) { + p.append(SkRasterPipeline::srcover); + } + + if (kUnpremul_SkAlphaType == dest_alphatype) { + p.append(SkRasterPipeline::unpremul); + } + p.append_store(dest_colortype, &dst_ctx); + + p.run(0, 0, width, 1); +} + +bool APNGImageGenerator::RenderDefaultImage(const SkImageInfo& info, + void* pixels, + size_t row_bytes) { + SkCodec::Result result = images_[0].codec->getPixels(info, pixels, row_bytes); + if (result != SkCodec::kSuccess) { + FML_DLOG(ERROR) << "Failed to decode the APNG's default/fallback image. " + "SkCodec::Result: " + << result; + return false; + } + return true; +} + +} // namespace flutter diff --git a/lib/ui/painting/image_generator_apng.h b/lib/ui/painting/image_generator_apng.h new file mode 100644 index 0000000000000..8407491f87bc1 --- /dev/null +++ b/lib/ui/painting/image_generator_apng.h @@ -0,0 +1,223 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "image_generator.h" + +#include "flutter/fml/endianness.h" +#include "flutter/fml/logging.h" + +#define PNG_FIELD(T, name) \ + private: \ + T name; \ + \ + public: \ + T get_##name() const { \ + return fml::BigEndianToArch(name); \ + } \ + void set_##name(T n) { \ + name = fml::BigEndianToArch(n); \ + } + +namespace flutter { + +class APNGImageGenerator : public ImageGenerator { + public: + ~APNGImageGenerator(); + + // |ImageGenerator| + const SkImageInfo& GetInfo() override; + + // |ImageGenerator| + unsigned int GetFrameCount() const override; + + // |ImageGenerator| + unsigned int GetPlayCount() const override; + + // |ImageGenerator| + const ImageGenerator::FrameInfo GetFrameInfo( + unsigned int frame_index) override; + + // |ImageGenerator| + SkISize GetScaledDimensions(float desired_scale) override; + + // |ImageGenerator| + bool GetPixels(const SkImageInfo& info, + void* pixels, + size_t row_bytes, + unsigned int frame_index, + std::optional prior_frame) override; + + static std::unique_ptr MakeFromData(sk_sp data); + + private: + static constexpr size_t kChunkCrcSize = 4; + + enum ChunkType { + kImageHeaderChunkType = 'IHDR', + kAnimationControlChunkType = 'acTL', + kImageDataChunkType = 'IDAT', + kFrameControlChunkType = 'fcTL', + kFrameDataChunkType = 'fdAT', + kImageTrailerChunkType = 'IEND', + }; + + class __attribute__((packed, aligned(1))) ChunkHeader { + PNG_FIELD(uint32_t, data_length) + PNG_FIELD(ChunkType, type) + + public: + void UpdateChunkCrc32(); + + private: + uint32_t ComputeChunkCrc32(); + }; + + class __attribute__((packed, aligned(1))) ImageHeaderChunkData { + PNG_FIELD(uint32_t, width) + PNG_FIELD(uint32_t, height) + PNG_FIELD(uint8_t, bit_depth) + PNG_FIELD(uint8_t, color_type) + PNG_FIELD(uint8_t, compression_method) + PNG_FIELD(uint8_t, filter_method) + PNG_FIELD(uint8_t, interlace_method) + }; + + class __attribute__((packed, aligned(1))) AnimationControlChunkData { + PNG_FIELD(uint32_t, num_frames) + PNG_FIELD(uint32_t, num_plays) + }; + + class __attribute__((packed, aligned(1))) FrameControlChunkData { + PNG_FIELD(uint32_t, sequence_number) + PNG_FIELD(uint32_t, width) + PNG_FIELD(uint32_t, height) + PNG_FIELD(uint32_t, x_offset) + PNG_FIELD(uint32_t, y_offset) + PNG_FIELD(uint16_t, delay_num) + PNG_FIELD(uint16_t, delay_den) + PNG_FIELD(uint8_t, dispose_op) + PNG_FIELD(uint8_t, blend_op) + }; + + /// @brief The first PNG frame is always the "default" PNG frame. Absense of + /// `frame_info` is only possible on the "default" PNG frame. + /// Each frame goes through two decoding stages: + /// 1. Demuxing stage: An individual PNG codec is created for a frame + /// while walking through the APNG chunk stream -- this is placed + /// in the `codec` field. + /// 2. Decoding stage: When a frame is requested for the first time, + /// the decoded image is requested from the `SkCodec` and then + /// (depending on the `frame_info`) composited with a previous + /// frame. The final "canvas" frame is placed in the + /// `composited_image` field. At this point, the `codec` is freed + /// and the `composited_image` is handed to the caller for drawing. + struct APNGImage { + std::unique_ptr codec; + + // The rendered frame pixels. + std::vector pixels; + + // Absense of frame info is possible on the "default" image. + std::optional frame_info; + + // X offset of this image when composited. Only applicable to frames. + unsigned int x_offset; + + // X offset of this image when composited. Only applicable to frames. + unsigned int y_offset; + }; + + APNGImageGenerator(sk_sp& data, + SkImageInfo& image_info, + APNGImage&& default_image, + unsigned int frame_count, + unsigned int play_count, + const void* next_chunk_p, + const std::vector header); + + static bool IsValidChunkHeader(const void* buffer, + size_t size, + const ChunkHeader* chunk); + + static const ChunkHeader* GetNextChunk(const void* buffer, + size_t size, + const ChunkHeader* current_chunk); + + /// @brief This is a utility template for casting a png buffer pointer to a + /// chunk header. It's primary purpose is to statically insert runtime + /// debug checks that detect invalid decoding behavior. + template + static constexpr const T* CastChunkData(const ChunkHeader* chunk) { + if constexpr (std::is_same_v) { + FML_DCHECK(chunk->get_type() == kImageHeaderChunkType); + } else if constexpr (std::is_same_v) { + FML_DCHECK(chunk->get_type() == kAnimationControlChunkType); + } else if constexpr (std::is_same_v) { + FML_DCHECK(chunk->get_type() == kFrameControlChunkType); + } else { + static_assert(!sizeof(T), "Invalid chunk struct"); + } + + return reinterpret_cast(reinterpret_cast(chunk) + + sizeof(ChunkHeader)); + } + + static constexpr size_t GetChunkSize(const ChunkHeader* chunk) { + return sizeof(ChunkHeader) + chunk->get_data_length() + kChunkCrcSize; + } + + static constexpr bool IsChunkCopySafe(const ChunkHeader* chunk) { + // The safe-to-copy bit is the 5th bit of the chunk name's 4th byte. This is + // the same as checking that the 4th byte is lowercase. + return (chunk->get_type() & 0x20) != 0; + } + + /// @brief Extract a header that's safe to use for both the "default" image + /// and individual PNG frames. Strip the animation control chunk. + static std::pair>, const void*> + ExtractHeader(const void* buffer_p, size_t buffer_size); + + /// @brief Takes a chunk pointer to a chunk and demuxes/interprets the next + /// image in the APNG sequence. It also provides the next `chunk_p` + /// to use. + /// @see `APNGImage` + static std::pair, const void*> DemuxNextImage( + const void* buffer_p, + size_t buffer_size, + const std::vector header, + const void* chunk_p); + + bool DemuxNextImageInternal(); + + bool DemuxToImageIndex(unsigned int image_index); + + static void BlendLine(SkColorType dest_colortype, + void* dest, + SkColorType source_colortype, + const void* source, + SkAlphaType dest_alphatype, + SkCodecAnimation::Blend blend_mode, + int width); + + bool RenderDefaultImage(const SkImageInfo& info, + void* pixels, + size_t row_bytes); + + FML_DISALLOW_COPY_ASSIGN_AND_MOVE(APNGImageGenerator); + sk_sp data_; + SkImageInfo image_info_; + unsigned int frame_count_; + unsigned int play_count_; + + // The first image is always the default image, which may or may not be a + // frame. All subsequent images are guaranteed to have frame data. + std::vector images_; + + unsigned int first_frame_index_; + + const void* next_chunk_p_; + std::vector header_; +}; + +} // namespace flutter diff --git a/lib/ui/painting/image_generator_registry.cc b/lib/ui/painting/image_generator_registry.cc index 4460182f0a673..ca5d83a36d359 100644 --- a/lib/ui/painting/image_generator_registry.cc +++ b/lib/ui/painting/image_generator_registry.cc @@ -14,9 +14,17 @@ #include "third_party/skia/include/ports/SkImageGeneratorWIC.h" #endif +#include "image_generator_apng.h" + namespace flutter { ImageGeneratorRegistry::ImageGeneratorRegistry() : weak_factory_(this) { + AddFactory( + [](sk_sp buffer) { + return APNGImageGenerator::MakeFromData(std::move(buffer)); + }, + 0); + AddFactory( [](sk_sp buffer) { return BuiltinSkiaCodecImageGenerator::MakeFromData(std::move(buffer)); diff --git a/lib/ui/painting/image_generator_registry_unittests.cc b/lib/ui/painting/image_generator_registry_unittests.cc index 3051e42410644..54eb45309e3a0 100644 --- a/lib/ui/painting/image_generator_registry_unittests.cc +++ b/lib/ui/painting/image_generator_registry_unittests.cc @@ -62,7 +62,7 @@ class FakeImageGenerator : public ImageGenerator { unsigned int GetPlayCount() const { return 1; } - const ImageGenerator::FrameInfo GetFrameInfo(unsigned int frame_index) const { + const ImageGenerator::FrameInfo GetFrameInfo(unsigned int frame_index) { return {std::nullopt, 0, SkCodecAnimation::DisposalMethod::kKeep}; } diff --git a/lib/ui/painting/multi_frame_codec.cc b/lib/ui/painting/multi_frame_codec.cc index 76af13ba161fa..acaa1405fe3d8 100644 --- a/lib/ui/painting/multi_frame_codec.cc +++ b/lib/ui/painting/multi_frame_codec.cc @@ -107,6 +107,8 @@ sk_sp MultiFrameCodec::State::GetNextFrameImage( std::optional prior_frame_index = std::nullopt; if (requiredFrameIndex != SkCodec::kNoFrame) { + // We currently assume that frames can only ever depend on the immediately + // previous frame, if any. if (lastRequiredFrame_ == nullptr) { FML_LOG(ERROR) << "Frame " << nextFrameIndex_ << " depends on frame " << requiredFrameIndex @@ -118,6 +120,8 @@ sk_sp MultiFrameCodec::State::GetNextFrameImage( << " instead"; } + // Copy the previous frame's output buffer into the current frame as the + // starting point. if (lastRequiredFrame_->getPixels() && CopyToBitmap(&bitmap, lastRequiredFrame_->colorType(), *lastRequiredFrame_)) { @@ -125,6 +129,8 @@ sk_sp MultiFrameCodec::State::GetNextFrameImage( } } + // Write the new frame to the output buffer. The bitmap pixels as supplied + // are already set in accordance with the previous frame's disposal policy. if (!generator_->GetPixels(info, bitmap.getPixels(), bitmap.rowBytes(), nextFrameIndex_, requiredFrameIndex)) { FML_LOG(ERROR) << "Could not getPixels for frame " << nextFrameIndex_; diff --git a/shell/common/shell_unittests.cc b/shell/common/shell_unittests.cc index d850eab688012..9f775f87d1047 100644 --- a/shell/common/shell_unittests.cc +++ b/shell/common/shell_unittests.cc @@ -2238,7 +2238,7 @@ class SinglePixelImageGenerator : public ImageGenerator { unsigned int GetPlayCount() const { return 1; } - const ImageGenerator::FrameInfo GetFrameInfo(unsigned int frame_index) const { + const ImageGenerator::FrameInfo GetFrameInfo(unsigned int frame_index) { return {std::nullopt, 0, SkCodecAnimation::DisposalMethod::kKeep}; } diff --git a/shell/platform/android/android_image_generator.cc b/shell/platform/android/android_image_generator.cc index 714eef933b145..9ba4c050c9a9a 100644 --- a/shell/platform/android/android_image_generator.cc +++ b/shell/platform/android/android_image_generator.cc @@ -38,7 +38,7 @@ unsigned int AndroidImageGenerator::GetPlayCount() const { } const ImageGenerator::FrameInfo AndroidImageGenerator::GetFrameInfo( - unsigned int frame_index) const { + unsigned int frame_index) { return {.required_frame = std::nullopt, .duration = 0, .disposal_method = SkCodecAnimation::DisposalMethod::kKeep}; diff --git a/shell/platform/android/android_image_generator.h b/shell/platform/android/android_image_generator.h index 6d13c94116492..53cbadde10afc 100644 --- a/shell/platform/android/android_image_generator.h +++ b/shell/platform/android/android_image_generator.h @@ -32,7 +32,7 @@ class AndroidImageGenerator : public ImageGenerator { // |ImageGenerator| const ImageGenerator::FrameInfo GetFrameInfo( - unsigned int frame_index) const override; + unsigned int frame_index) override; // |ImageGenerator| SkISize GetScaledDimensions(float desired_scale) override; From 93b1aae311a4df993fae58cdc0f4441939b56193 Mon Sep 17 00:00:00 2001 From: Brandon DeRosier Date: Sat, 29 Oct 2022 22:15:10 -0700 Subject: [PATCH 02/11] Fix warnings --- lib/ui/painting/image_generator_apng.cc | 5 +++-- lib/ui/painting/image_generator_apng.h | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/lib/ui/painting/image_generator_apng.cc b/lib/ui/painting/image_generator_apng.cc index 91c5f103f5b33..2524ed0982628 100644 --- a/lib/ui/painting/image_generator_apng.cc +++ b/lib/ui/painting/image_generator_apng.cc @@ -7,6 +7,7 @@ #include "third_party/libpng/png.h" #include "third_party/skia/include/codec/SkCodecAnimation.h" +#include "third_party/skia/include/core/SkStream.h" #include "third_party/skia/src/codec/SkPngCodec.h" #include "third_party/skia/src/core/SkRasterPipeline.h" #include "third_party/zlib/zlib.h" // For crc32 @@ -21,7 +22,7 @@ APNGImageGenerator::APNGImageGenerator(sk_sp& data, unsigned int frame_count, unsigned int play_count, const void* next_chunk_p, - const std::vector header) + const std::vector& header) : data_(data), image_info_(image_info), frame_count_(frame_count), @@ -275,7 +276,7 @@ APNGImageGenerator::ExtractHeader(const void* buffer_p, size_t buffer_size) { std::pair, const void*> APNGImageGenerator::DemuxNextImage(const void* buffer_p, size_t buffer_size, - const std::vector header, + const std::vector& header, const void* chunk_p) { const ChunkHeader* chunk = reinterpret_cast(chunk_p); // Validate the given chunk to ensure it's safe to read. diff --git a/lib/ui/painting/image_generator_apng.h b/lib/ui/painting/image_generator_apng.h index 8407491f87bc1..4e1b262642726 100644 --- a/lib/ui/painting/image_generator_apng.h +++ b/lib/ui/painting/image_generator_apng.h @@ -134,7 +134,7 @@ class APNGImageGenerator : public ImageGenerator { unsigned int frame_count, unsigned int play_count, const void* next_chunk_p, - const std::vector header); + const std::vector& header); static bool IsValidChunkHeader(const void* buffer, size_t size, @@ -185,7 +185,7 @@ class APNGImageGenerator : public ImageGenerator { static std::pair, const void*> DemuxNextImage( const void* buffer_p, size_t buffer_size, - const std::vector header, + const std::vector& header, const void* chunk_p); bool DemuxNextImageInternal(); From 35d747607569722839cf45cd06ba12bc620e5bbc Mon Sep 17 00:00:00 2001 From: Brandon DeRosier Date: Sat, 29 Oct 2022 22:21:39 -0700 Subject: [PATCH 03/11] Remove private skia blending utils --- lib/ui/painting/image_generator_apng.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/ui/painting/image_generator_apng.cc b/lib/ui/painting/image_generator_apng.cc index 2524ed0982628..b957807972bc5 100644 --- a/lib/ui/painting/image_generator_apng.cc +++ b/lib/ui/painting/image_generator_apng.cc @@ -9,7 +9,6 @@ #include "third_party/skia/include/codec/SkCodecAnimation.h" #include "third_party/skia/include/core/SkStream.h" #include "third_party/skia/src/codec/SkPngCodec.h" -#include "third_party/skia/src/core/SkRasterPipeline.h" #include "third_party/zlib/zlib.h" // For crc32 namespace flutter { @@ -521,6 +520,7 @@ void APNGImageGenerator::BlendLine(SkColorType dest_colortype, SkAlphaType dest_alphatype, SkCodecAnimation::Blend blend_mode, int width) { + /* SkRasterPipeline_MemoryCtx dst_ctx = {dest, 0}, src_ctx = {const_cast(source), 0}; @@ -544,6 +544,7 @@ void APNGImageGenerator::BlendLine(SkColorType dest_colortype, p.append_store(dest_colortype, &dst_ctx); p.run(0, 0, width, 1); + */ } bool APNGImageGenerator::RenderDefaultImage(const SkImageInfo& info, From f74b9c495436e70ebfe3291449fdb6188b88b492 Mon Sep 17 00:00:00 2001 From: Brandon DeRosier Date: Sun, 30 Oct 2022 00:25:47 -0700 Subject: [PATCH 04/11] Licenses --- ci/licenses_golden/licenses_flutter | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ci/licenses_golden/licenses_flutter b/ci/licenses_golden/licenses_flutter index 4d518c29f3bae..fb538a5b4aaba 100644 --- a/ci/licenses_golden/licenses_flutter +++ b/ci/licenses_golden/licenses_flutter @@ -4273,6 +4273,8 @@ FILE: ../../../flutter/lib/ui/painting/image_filter.cc FILE: ../../../flutter/lib/ui/painting/image_filter.h FILE: ../../../flutter/lib/ui/painting/image_generator.cc FILE: ../../../flutter/lib/ui/painting/image_generator.h +FILE: ../../../flutter/lib/ui/painting/image_generator_apng.cc +FILE: ../../../flutter/lib/ui/painting/image_generator_apng.h FILE: ../../../flutter/lib/ui/painting/image_generator_registry.cc FILE: ../../../flutter/lib/ui/painting/image_generator_registry.h FILE: ../../../flutter/lib/ui/painting/image_shader.cc From 07c6f292170659a8dbc500b9412755797644200e Mon Sep 17 00:00:00 2001 From: Brandon DeRosier Date: Sun, 30 Oct 2022 05:27:21 -0700 Subject: [PATCH 05/11] Compositing --- lib/ui/painting/image_generator_apng.cc | 83 ++++++++++++++++++++++--- 1 file changed, 75 insertions(+), 8 deletions(-) diff --git a/lib/ui/painting/image_generator_apng.cc b/lib/ui/painting/image_generator_apng.cc index b957807972bc5..5ecbdf2eb258e 100644 --- a/lib/ui/painting/image_generator_apng.cc +++ b/lib/ui/painting/image_generator_apng.cc @@ -4,9 +4,13 @@ #include "image_generator_apng.h" #include +#include +#include "flutter/fml/logging.h" #include "third_party/libpng/png.h" #include "third_party/skia/include/codec/SkCodecAnimation.h" +#include "third_party/skia/include/core/SkAlphaType.h" +#include "third_party/skia/include/core/SkColorType.h" #include "third_party/skia/include/core/SkStream.h" #include "third_party/skia/src/codec/SkPngCodec.h" #include "third_party/zlib/zlib.h" // For crc32 @@ -101,15 +105,78 @@ bool APNGImageGenerator::GetPixels(const SkImageInfo& info, /// 3. Composite the frame onto the canvas. /// - for (int i = 0; i < frame_info.height(); i++) { - void* source = frame.pixels.data() + i * frame_row_bytes; - void* destination = static_cast(pixels) + - frame.x_offset * frame_info.bytesPerPixel() + - (i + frame.y_offset) * row_bytes; + FML_DCHECK(info.colorType() == kN32_SkColorType); + FML_DCHECK(frame_info.colorType() == kN32_SkColorType); - BlendLine(info.colorType(), destination, frame_info.colorType(), source, - info.alphaType(), frame.frame_info->blend_mode, - frame_info.width()); + // Regardless of the byte order (RGBA vs BGRA), the blending operations are + // the same. + struct Pixel { + uint8_t channel[4]; + + uint8_t GetAlpha() { return channel[3]; } + + void SetAlpha(uint8_t value) { channel[3] = value; } + + void Premultiply() { + for (int i = 0; i < 3; i++) { + channel[i] = channel[i] * GetAlpha() / 0xFF; + } + } + + void Unpremultiply() { + if (GetAlpha() == 0) { + channel[0] = channel[1] = channel[2] = 0; + return; + } + for (int i = 0; i < 3; i++) { + channel[i] = channel[i] * 0xFF / GetAlpha(); + } + } + }; + + FML_DCHECK(frame_info.bytesPerPixel() == sizeof(Pixel)); + + for (int y = 0; y < frame_info.height(); y++) { + auto src_row = frame.pixels.data() + y * frame_row_bytes; + auto dst_row = static_cast(pixels) + + (y + frame.y_offset) * row_bytes + + frame.x_offset * frame_info.bytesPerPixel(); + + switch (frame.frame_info->blend_mode) { + case SkCodecAnimation::Blend::kSrcOver: { + for (int x = 0; x < frame_info.width(); x++) { + auto x_offset_bytes = x * frame_info.bytesPerPixel(); + + Pixel src = *reinterpret_cast(src_row + x_offset_bytes); + Pixel* dst_p = reinterpret_cast(dst_row + x_offset_bytes); + Pixel dst = *dst_p; + + if (info.alphaType() == kUnpremul_SkAlphaType) { + dst.Premultiply(); + } + if (frame_info.alphaType() == kUnpremul_SkAlphaType) { + src.Premultiply(); + } + + for (int i = 0; i < 3; i++) { + dst.channel[i] = src.channel[i] + + dst.channel[i] * (0xFF - src.GetAlpha()) / 0xFF; + } + dst.SetAlpha(src.GetAlpha() + + dst.GetAlpha() * (0xFF - src.GetAlpha()) / 0xFF); + + if (info.alphaType() == kUnpremul_SkAlphaType) { + dst.Unpremultiply(); + } + + *dst_p = dst; + } + break; + } + case SkCodecAnimation::Blend::kSrc: + memcpy(dst_row, src_row, frame_row_bytes); + break; + } } return true; From 9f650695f448405381c5d8780a069380aee5caff Mon Sep 17 00:00:00 2001 From: Brandon DeRosier Date: Sun, 30 Oct 2022 06:24:46 -0700 Subject: [PATCH 06/11] Fix disposal --- lib/ui/painting/image_generator_apng.cc | 57 +++++++++---------------- lib/ui/painting/image_generator_apng.h | 8 ---- 2 files changed, 19 insertions(+), 46 deletions(-) diff --git a/lib/ui/painting/image_generator_apng.cc b/lib/ui/painting/image_generator_apng.cc index 5ecbdf2eb258e..391389c361eb4 100644 --- a/lib/ui/painting/image_generator_apng.cc +++ b/lib/ui/painting/image_generator_apng.cc @@ -105,8 +105,20 @@ bool APNGImageGenerator::GetPixels(const SkImageInfo& info, /// 3. Composite the frame onto the canvas. /// - FML_DCHECK(info.colorType() == kN32_SkColorType); - FML_DCHECK(frame_info.colorType() == kN32_SkColorType); + if (info.colorType() != kN32_SkColorType) { + FML_DLOG(ERROR) << "Failed to composite image at index " << image_index + << " (frame index: " << frame_index + << ") of APNG due to the destination surface having an " + "unsupported color type."; + return false; + } + if (frame_info.colorType() != kN32_SkColorType) { + FML_DLOG(ERROR) + << "Failed to composite image at index " << image_index + << " (frame index: " << frame_index + << ") of APNG due to the frame having an unsupported color type."; + return false; + } // Regardless of the byte order (RGBA vs BGRA), the blending operations are // the same. @@ -515,8 +527,11 @@ bool APNGImageGenerator::DemuxNextImageInternal() { return false; } - if (!images_.empty() && images_.back().frame_info->disposal_method == - SkCodecAnimation::DisposalMethod::kKeep) { + if (images_.size() > first_frame_index_ && + images_.back().frame_info->disposal_method == + SkCodecAnimation::DisposalMethod::kKeep) { + // Offset by 2 because the first image is the default image, which may not + // be a frame. image->frame_info->required_frame = images_.size() - 1; } @@ -580,40 +595,6 @@ uint32_t APNGImageGenerator::ChunkHeader::ComputeChunkCrc32() { return crc; } -void APNGImageGenerator::BlendLine(SkColorType dest_colortype, - void* dest, - SkColorType source_colortype, - const void* source, - SkAlphaType dest_alphatype, - SkCodecAnimation::Blend blend_mode, - int width) { - /* - SkRasterPipeline_MemoryCtx dst_ctx = {dest, 0}, - src_ctx = {const_cast(source), 0}; - - SkRasterPipeline_<256> p; - - p.append_load_dst(dest_colortype, &dst_ctx); - if (kUnpremul_SkAlphaType == dest_alphatype) { - p.append(SkRasterPipeline::premul_dst); - } - - p.append_load(source_colortype, &src_ctx); - p.append(SkRasterPipeline::premul); - - if (blend_mode == SkCodecAnimation::Blend::kSrcOver) { - p.append(SkRasterPipeline::srcover); - } - - if (kUnpremul_SkAlphaType == dest_alphatype) { - p.append(SkRasterPipeline::unpremul); - } - p.append_store(dest_colortype, &dst_ctx); - - p.run(0, 0, width, 1); - */ -} - bool APNGImageGenerator::RenderDefaultImage(const SkImageInfo& info, void* pixels, size_t row_bytes) { diff --git a/lib/ui/painting/image_generator_apng.h b/lib/ui/painting/image_generator_apng.h index 4e1b262642726..dd337069598c0 100644 --- a/lib/ui/painting/image_generator_apng.h +++ b/lib/ui/painting/image_generator_apng.h @@ -192,14 +192,6 @@ class APNGImageGenerator : public ImageGenerator { bool DemuxToImageIndex(unsigned int image_index); - static void BlendLine(SkColorType dest_colortype, - void* dest, - SkColorType source_colortype, - const void* source, - SkAlphaType dest_alphatype, - SkCodecAnimation::Blend blend_mode, - int width); - bool RenderDefaultImage(const SkImageInfo& info, void* pixels, size_t row_bytes); From eb8f089577c8d3a1038ff1dae118a0b530f8573a Mon Sep 17 00:00:00 2001 From: Brandon DeRosier Date: Mon, 31 Oct 2022 01:17:02 -0700 Subject: [PATCH 07/11] Improve behavior when a required frame isn't cached. Also fixes https://github.com/flutter/flutter/issues/61150 --- lib/ui/painting/image_generator_apng.cc | 21 ++++++++++------- lib/ui/painting/multi_frame_codec.cc | 30 ++++++++++++------------- 2 files changed, 28 insertions(+), 23 deletions(-) diff --git a/lib/ui/painting/image_generator_apng.cc b/lib/ui/painting/image_generator_apng.cc index 391389c361eb4..132d8cdc26a6d 100644 --- a/lib/ui/painting/image_generator_apng.cc +++ b/lib/ui/painting/image_generator_apng.cc @@ -163,6 +163,7 @@ bool APNGImageGenerator::GetPixels(const SkImageInfo& info, Pixel* dst_p = reinterpret_cast(dst_row + x_offset_bytes); Pixel dst = *dst_p; + // Ensure both colors are premultiplied for the blending operation. if (info.alphaType() == kUnpremul_SkAlphaType) { dst.Premultiply(); } @@ -170,13 +171,13 @@ bool APNGImageGenerator::GetPixels(const SkImageInfo& info, src.Premultiply(); } - for (int i = 0; i < 3; i++) { + for (int i = 0; i < 4; i++) { dst.channel[i] = src.channel[i] + dst.channel[i] * (0xFF - src.GetAlpha()) / 0xFF; } - dst.SetAlpha(src.GetAlpha() + - dst.GetAlpha() * (0xFF - src.GetAlpha()) / 0xFF); + // The final color is premultiplied. Unpremultiply to match the + // backdrop surface if necessary. if (info.alphaType() == kUnpremul_SkAlphaType) { dst.Unpremultiply(); } @@ -408,9 +409,6 @@ APNGImageGenerator::DemuxNextImage(const void* buffer_p, frame_info.duration = static_cast(control_data->get_delay_num() * 1000.f / denominator); - // TODO(bdero): Populate frame_info.required_frame depending on the - // blend_mode and disposal_method. - result.frame_info = frame_info; result.x_offset = control_data->get_x_offset(); result.y_offset = control_data->get_y_offset(); @@ -527,11 +525,18 @@ bool APNGImageGenerator::DemuxNextImageInternal() { return false; } + if (images_.back().frame_info->disposal_method == + SkCodecAnimation::DisposalMethod::kRestorePrevious) { + FML_DLOG(INFO) + << "DisposalMethod::kRestorePrevious is not supported by the " + "MultiFrameCodec. Falling back to DisposalMethod::kRestoreBGColor " + " behavior instead."; + } + if (images_.size() > first_frame_index_ && images_.back().frame_info->disposal_method == SkCodecAnimation::DisposalMethod::kKeep) { - // Offset by 2 because the first image is the default image, which may not - // be a frame. + // Mark the required frame as the previous frame in all cases. image->frame_info->required_frame = images_.size() - 1; } diff --git a/lib/ui/painting/multi_frame_codec.cc b/lib/ui/painting/multi_frame_codec.cc index acaa1405fe3d8..a380656ac1c04 100644 --- a/lib/ui/painting/multi_frame_codec.cc +++ b/lib/ui/painting/multi_frame_codec.cc @@ -108,24 +108,24 @@ sk_sp MultiFrameCodec::State::GetNextFrameImage( if (requiredFrameIndex != SkCodec::kNoFrame) { // We currently assume that frames can only ever depend on the immediately - // previous frame, if any. + // previous frame, if any. This means that + // `DisposalMethod::kRestorePrevious` is not supported. if (lastRequiredFrame_ == nullptr) { - FML_LOG(ERROR) << "Frame " << nextFrameIndex_ << " depends on frame " - << requiredFrameIndex - << " and no required frames are cached."; - return nullptr; + FML_DLOG(INFO) + << "Frame " << nextFrameIndex_ << " depends on frame " + << requiredFrameIndex + << " and no required frames are cached. Using blank slate instead."; } else if (lastRequiredFrameIndex_ != requiredFrameIndex) { FML_DLOG(INFO) << "Required frame " << requiredFrameIndex - << " is not cached. Using " << lastRequiredFrameIndex_ - << " instead"; - } - - // Copy the previous frame's output buffer into the current frame as the - // starting point. - if (lastRequiredFrame_->getPixels() && - CopyToBitmap(&bitmap, lastRequiredFrame_->colorType(), - *lastRequiredFrame_)) { - prior_frame_index = requiredFrameIndex; + << " is not cached. Using blank slate instead."; + } else { + // Copy the previous frame's output buffer into the current frame as the + // starting point. + if (lastRequiredFrame_->getPixels() && + CopyToBitmap(&bitmap, lastRequiredFrame_->colorType(), + *lastRequiredFrame_)) { + prior_frame_index = requiredFrameIndex; + } } } From 219b5ad468e907bc6c7fbff1a816a68aad475160 Mon Sep 17 00:00:00 2001 From: Brandon DeRosier Date: Tue, 31 Jan 2023 10:43:21 -0800 Subject: [PATCH 08/11] Address comments --- lib/ui/painting/image_generator_apng.cc | 2 -- lib/ui/painting/image_generator_apng.h | 4 ++-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/lib/ui/painting/image_generator_apng.cc b/lib/ui/painting/image_generator_apng.cc index 132d8cdc26a6d..85f978c1dbd2b 100644 --- a/lib/ui/painting/image_generator_apng.cc +++ b/lib/ui/painting/image_generator_apng.cc @@ -127,8 +127,6 @@ bool APNGImageGenerator::GetPixels(const SkImageInfo& info, uint8_t GetAlpha() { return channel[3]; } - void SetAlpha(uint8_t value) { channel[3] = value; } - void Premultiply() { for (int i = 0; i < 3; i++) { channel[i] = channel[i] * GetAlpha() / 0xFF; diff --git a/lib/ui/painting/image_generator_apng.h b/lib/ui/painting/image_generator_apng.h index dd337069598c0..1a62efe94267f 100644 --- a/lib/ui/painting/image_generator_apng.h +++ b/lib/ui/painting/image_generator_apng.h @@ -100,7 +100,7 @@ class APNGImageGenerator : public ImageGenerator { PNG_FIELD(uint8_t, blend_op) }; - /// @brief The first PNG frame is always the "default" PNG frame. Absense of + /// @brief The first PNG frame is always the "default" PNG frame. Absence of /// `frame_info` is only possible on the "default" PNG frame. /// Each frame goes through two decoding stages: /// 1. Demuxing stage: An individual PNG codec is created for a frame @@ -145,7 +145,7 @@ class APNGImageGenerator : public ImageGenerator { const ChunkHeader* current_chunk); /// @brief This is a utility template for casting a png buffer pointer to a - /// chunk header. It's primary purpose is to statically insert runtime + /// chunk header. Its primary purpose is to statically insert runtime /// debug checks that detect invalid decoding behavior. template static constexpr const T* CastChunkData(const ChunkHeader* chunk) { From ac36a326684518202e1a821857184966c2aefcb4 Mon Sep 17 00:00:00 2001 From: Brandon DeRosier Date: Thu, 2 Feb 2023 14:13:34 -0800 Subject: [PATCH 09/11] Update licenses --- ci/licenses_golden/licenses_flutter | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ci/licenses_golden/licenses_flutter b/ci/licenses_golden/licenses_flutter index fb538a5b4aaba..b2250fc45b381 100644 --- a/ci/licenses_golden/licenses_flutter +++ b/ci/licenses_golden/licenses_flutter @@ -1793,6 +1793,8 @@ ORIGIN: ../../../flutter/lib/ui/painting/image_filter.cc + ../../../flutter/LICE ORIGIN: ../../../flutter/lib/ui/painting/image_filter.h + ../../../flutter/LICENSE ORIGIN: ../../../flutter/lib/ui/painting/image_generator.cc + ../../../flutter/LICENSE ORIGIN: ../../../flutter/lib/ui/painting/image_generator.h + ../../../flutter/LICENSE +ORIGIN: ../../../flutter/lib/ui/painting/image_generator_apng.cc + ../../../flutter/LICENSE +ORIGIN: ../../../flutter/lib/ui/painting/image_generator_apng.h + ../../../flutter/LICENSE ORIGIN: ../../../flutter/lib/ui/painting/image_generator_registry.cc + ../../../flutter/LICENSE ORIGIN: ../../../flutter/lib/ui/painting/image_generator_registry.h + ../../../flutter/LICENSE ORIGIN: ../../../flutter/lib/ui/painting/image_shader.cc + ../../../flutter/LICENSE From 152e1b48d4429bef66bcefb3f46d894f436ed0a0 Mon Sep 17 00:00:00 2001 From: Brandon DeRosier Date: Thu, 2 Feb 2023 14:30:29 -0800 Subject: [PATCH 10/11] Update signature --- lib/ui/painting/image_generator_apng.cc | 12 +++++++----- lib/ui/painting/image_generator_apng.h | 1 + 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/lib/ui/painting/image_generator_apng.cc b/lib/ui/painting/image_generator_apng.cc index 85f978c1dbd2b..d07072941b7a2 100644 --- a/lib/ui/painting/image_generator_apng.cc +++ b/lib/ui/painting/image_generator_apng.cc @@ -197,12 +197,13 @@ std::unique_ptr APNGImageGenerator::MakeFromData( sk_sp data) { // Ensure the buffer is large enough to at least contain the PNG signature // and a chunk header. - if (data->size() < 8 + sizeof(ChunkHeader)) { + if (data->size() < sizeof(kPngSignature) + sizeof(ChunkHeader)) { return nullptr; } // Validate the full PNG signature. const uint8_t* data_p = static_cast(data.get()->data()); - if (png_sig_cmp(static_cast(data_p), 0, 8)) { + if (png_sig_cmp(static_cast(data_p), 0, + sizeof(kPngSignature))) { return nullptr; } @@ -316,11 +317,12 @@ const APNGImageGenerator::ChunkHeader* APNGImageGenerator::GetNextChunk( std::pair>, const void*> APNGImageGenerator::ExtractHeader(const void* buffer_p, size_t buffer_size) { - std::vector result = {137, 80, 78, 71, - 13, 10, 26, 10}; // PNG signature + std::vector result; + result.emplace(result.end(), kPngSignature, + kPngSignature + sizeof(kPngSignature)); const ChunkHeader* chunk = reinterpret_cast( - static_cast(buffer_p) + 8); + static_cast(buffer_p) + sizeof(kPngSignature)); // Validate the first chunk to ensure it's safe to read. if (!IsValidChunkHeader(buffer_p, buffer_size, chunk)) { return std::make_pair(std::nullopt, nullptr); diff --git a/lib/ui/painting/image_generator_apng.h b/lib/ui/painting/image_generator_apng.h index 1a62efe94267f..77aa5f2aa3912 100644 --- a/lib/ui/painting/image_generator_apng.h +++ b/lib/ui/painting/image_generator_apng.h @@ -51,6 +51,7 @@ class APNGImageGenerator : public ImageGenerator { static std::unique_ptr MakeFromData(sk_sp data); private: + static constexpr uint8_t kPngSignature[8] = {137, 80, 78, 71, 13, 10, 26, 10}; static constexpr size_t kChunkCrcSize = 4; enum ChunkType { From 614a020058555343243f72a9cf8519bcff65fa41 Mon Sep 17 00:00:00 2001 From: Brandon DeRosier Date: Fri, 3 Feb 2023 00:51:36 -0800 Subject: [PATCH 11/11] Address remaining comments --- lib/ui/painting/image_generator_apng.cc | 26 ++++++++++++------------- lib/ui/painting/image_generator_apng.h | 2 +- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/lib/ui/painting/image_generator_apng.cc b/lib/ui/painting/image_generator_apng.cc index d07072941b7a2..943d7752015db 100644 --- a/lib/ui/painting/image_generator_apng.cc +++ b/lib/ui/painting/image_generator_apng.cc @@ -217,17 +217,18 @@ std::unique_ptr APNGImageGenerator::MakeFromData( // Walk the chunks to find the "animation control" chunk. If an "image data" // chunk is found first, this PNG is not animated. - do { + while (true) { + chunk = GetNextChunk(data_p, data->size(), chunk); + + if (chunk == nullptr) { + return nullptr; + } if (chunk->get_type() == kImageDataChunkType) { return nullptr; - } else if (chunk->get_type() == kAnimationControlChunkType) { + } + if (chunk->get_type() == kAnimationControlChunkType) { break; } - - chunk = GetNextChunk(data_p, data->size(), chunk); - } while (chunk != nullptr); - if (chunk == nullptr) { - return nullptr; } const AnimationControlChunkData* animation_data = @@ -247,7 +248,7 @@ std::unique_ptr APNGImageGenerator::MakeFromData( const void* next_chunk_p; std::tie(default_image, next_chunk_p) = DemuxNextImage(data_p, data->size(), header.value(), first_chunk_p); - if (default_image == std::nullopt) { + if (!default_image.has_value()) { return nullptr; } @@ -317,9 +318,8 @@ const APNGImageGenerator::ChunkHeader* APNGImageGenerator::GetNextChunk( std::pair>, const void*> APNGImageGenerator::ExtractHeader(const void* buffer_p, size_t buffer_size) { - std::vector result; - result.emplace(result.end(), kPngSignature, - kPngSignature + sizeof(kPngSignature)); + std::vector result(sizeof(kPngSignature)); + memcpy(result.data(), kPngSignature, sizeof(kPngSignature)); const ChunkHeader* chunk = reinterpret_cast( static_cast(buffer_p) + sizeof(kPngSignature)); @@ -455,7 +455,7 @@ APNGImageGenerator::DemuxNextImage(const void* buffer_p, // If this is a frame, override the width/height in the IHDR chunk. if (control_data) { ChunkHeader* ihdr_header = - reinterpret_cast(write_cursor + 8); + reinterpret_cast(write_cursor + sizeof(kPngSignature)); ImageHeaderChunkData* ihdr_data = const_cast( CastChunkData(ihdr_header)); ihdr_data->set_width(control_data->get_width()); @@ -582,7 +582,7 @@ uint32_t APNGImageGenerator::ChunkHeader::ComputeChunkCrc32() { uint8_t* chunk_data_p = reinterpret_cast(this) + 4; uint32_t crc = 0; - // zlib's crc32 can only takes 16 bits at a time for the length, but PNG + // zlib's crc32 can only take 16 bits at a time for the length, but PNG // supports a 32 bit chunk length, so looping is necessary here. // Note that crc32 is always called at least once, even if the chunk has an // empty data section. diff --git a/lib/ui/painting/image_generator_apng.h b/lib/ui/painting/image_generator_apng.h index 77aa5f2aa3912..a18ef58cf008c 100644 --- a/lib/ui/painting/image_generator_apng.h +++ b/lib/ui/painting/image_generator_apng.h @@ -119,7 +119,7 @@ class APNGImageGenerator : public ImageGenerator { // The rendered frame pixels. std::vector pixels; - // Absense of frame info is possible on the "default" image. + // Absence of frame info is possible on the "default" image. std::optional frame_info; // X offset of this image when composited. Only applicable to frames.