diff --git a/docs/source/api_ref_decoders.rst b/docs/source/api_ref_decoders.rst index 1417d7aea..b3a1f3250 100644 --- a/docs/source/api_ref_decoders.rst +++ b/docs/source/api_ref_decoders.rst @@ -33,3 +33,4 @@ For an audio decoder tutorial, see: :ref:`sphx_glr_generated_examples_decoding_a VideoStreamMetadata AudioStreamMetadata + CpuFallbackStatus diff --git a/src/torchcodec/_core/CudaDeviceInterface.cpp b/src/torchcodec/_core/CudaDeviceInterface.cpp index 80e16506f..dfb660e85 100644 --- a/src/torchcodec/_core/CudaDeviceInterface.cpp +++ b/src/torchcodec/_core/CudaDeviceInterface.cpp @@ -242,6 +242,8 @@ void CudaDeviceInterface::convertAVFrameToFrameOutput( std::optional preAllocatedOutputTensor) { validatePreAllocatedTensorShape(preAllocatedOutputTensor, avFrame); + hasDecodedFrame_ = true; + // All of our CUDA decoding assumes NV12 format. We handle non-NV12 formats by // converting them to NV12. avFrame = maybeConvertAVFrameToNV12OrRGB24(avFrame); @@ -359,6 +361,10 @@ std::string CudaDeviceInterface::getDetails() { // Note: for this interface specifically the fallback is only known after a // frame has been decoded, not before: that's when FFmpeg decides to fallback, // so we can't know earlier. + if (!hasDecodedFrame_) { + return std::string( + "FFmpeg CUDA Device Interface. Fallback status unknown (no frames decoded)."); + } return std::string("FFmpeg CUDA Device Interface. Using ") + (usingCPUFallback_ ? "CPU fallback." : "NVDEC."); } diff --git a/src/torchcodec/_core/CudaDeviceInterface.h b/src/torchcodec/_core/CudaDeviceInterface.h index 238bcf000..d4460b169 100644 --- a/src/torchcodec/_core/CudaDeviceInterface.h +++ b/src/torchcodec/_core/CudaDeviceInterface.h @@ -71,6 +71,7 @@ class CudaDeviceInterface : public DeviceInterface { std::unique_ptr nv12Conversion_; bool usingCPUFallback_ = false; + bool hasDecodedFrame_ = false; }; } // namespace facebook::torchcodec diff --git a/src/torchcodec/decoders/__init__.py b/src/torchcodec/decoders/__init__.py index 980ba98a9..ef08cce83 100644 --- a/src/torchcodec/decoders/__init__.py +++ b/src/torchcodec/decoders/__init__.py @@ -7,6 +7,6 @@ from .._core import AudioStreamMetadata, VideoStreamMetadata from ._audio_decoder import AudioDecoder # noqa from ._decoder_utils import set_cuda_backend # noqa -from ._video_decoder import VideoDecoder # noqa +from ._video_decoder import CpuFallbackStatus, VideoDecoder # noqa SimpleVideoDecoder = VideoDecoder diff --git a/src/torchcodec/decoders/_video_decoder.py b/src/torchcodec/decoders/_video_decoder.py index 03f0d539a..b8518f766 100644 --- a/src/torchcodec/decoders/_video_decoder.py +++ b/src/torchcodec/decoders/_video_decoder.py @@ -9,6 +9,7 @@ import json import numbers from collections.abc import Sequence +from dataclasses import dataclass, field from pathlib import Path from typing import Literal @@ -25,6 +26,56 @@ from torchcodec.transforms._decoder_transforms import _make_transform_specs +@dataclass +class CpuFallbackStatus: + """Information about CPU fallback status. + + This class tracks whether the decoder fell back to CPU decoding. + Users should not instantiate this class directly; instead, access it + via the :attr:`VideoDecoder.cpu_fallback` attribute. + + Usage: + + - Use ``str(cpu_fallback_status)`` or ``print(cpu_fallback_status)`` to see the cpu fallback status + - Use ``if cpu_fallback_status:`` to check if any fallback occurred + """ + + status_known: bool = False + """Whether the fallback status has been determined. + For the Beta CUDA backend (see :func:`~torchcodec.decoders.set_cuda_backend`), + this is always ``True`` immediately after decoder creation. + For the FFmpeg CUDA backend, this becomes ``True`` after decoding + the first frame.""" + _nvcuvid_unavailable: bool = field(default=False, init=False) + _video_not_supported: bool = field(default=False, init=False) + _is_fallback: bool = field(default=False, init=False) + _backend: str = field(default="", init=False) + + def __bool__(self): + """Returns True if fallback occurred.""" + return self.status_known and self._is_fallback + + def __str__(self): + """Returns a human-readable string representation of the cpu fallback status.""" + if not self.status_known: + return f"[{self._backend}] Fallback status: Unknown" + + reasons = [] + if self._nvcuvid_unavailable: + reasons.append("NVcuvid unavailable") + elif self._video_not_supported: + reasons.append("Video not supported") + elif self._is_fallback: + reasons.append("Unknown reason - try the Beta interface to know more!") + + if reasons: + return ( + f"[{self._backend}] Fallback status: Falling back due to: " + + ", ".join(reasons) + ) + return f"[{self._backend}] Fallback status: No fallback required" + + class VideoDecoder: """A single-stream video decoder. @@ -103,6 +154,10 @@ class VideoDecoder: stream_index (int): The stream index that this decoder is retrieving frames from. If a stream index was provided at initialization, this is the same value. If it was left unspecified, this is the :term:`best stream`. + cpu_fallback (CpuFallbackStatus): Information about whether the decoder fell back to CPU + decoding. Use ``bool(cpu_fallback)`` to check if fallback occurred, or + ``str(cpu_fallback)`` to get a human-readable status message. The status is only + determined after at least one frame has been decoded. """ def __init__( @@ -186,9 +241,42 @@ def __init__( custom_frame_mappings=custom_frame_mappings_data, ) + self._cpu_fallback = CpuFallbackStatus() + if device.startswith("cuda"): + if device_variant == "beta": + self._cpu_fallback._backend = "Beta CUDA" + else: + self._cpu_fallback._backend = "FFmpeg CUDA" + else: + self._cpu_fallback._backend = "CPU" + def __len__(self) -> int: return self._num_frames + @property + def cpu_fallback(self) -> CpuFallbackStatus: + # We only query the CPU fallback info if status is unknown. That happens + # either when: + # - this @property has never been called before + # - no frame has been decoded yet on the FFmpeg interface. + # Note that for the beta interface, we're able to know the fallback status + # right when the VideoDecoder is instantiated, but the status_known + # attribute is initialized to False. + if not self._cpu_fallback.status_known: + backend_details = core._get_backend_details(self._decoder) + + if "status unknown" not in backend_details: + self._cpu_fallback.status_known = True + + if "CPU fallback" in backend_details: + self._cpu_fallback._is_fallback = True + if "NVCUVID not available" in backend_details: + self._cpu_fallback._nvcuvid_unavailable = True + elif self._cpu_fallback._backend == "Beta CUDA": + self._cpu_fallback._video_not_supported = True + + return self._cpu_fallback + def _getitem_int(self, key: int) -> Tensor: assert isinstance(key, int) diff --git a/test/test_decoders.py b/test/test_decoders.py index d688ffbf0..8fcd78665 100644 --- a/test/test_decoders.py +++ b/test/test_decoders.py @@ -27,6 +27,7 @@ assert_frames_equal, AV1_VIDEO, BT709_FULL_RANGE, + cuda_devices, cuda_version_used_for_building_torch, get_ffmpeg_major_version, get_python_version, @@ -1672,22 +1673,27 @@ def test_beta_cuda_interface_cpu_fallback(self): # to the CPU path, too. ref_dec = VideoDecoder(H265_VIDEO.path, device="cuda") - ref_frames = ref_dec.get_frame_at(0) - assert ( - _core._get_backend_details(ref_dec._decoder) - == "FFmpeg CUDA Device Interface. Using CPU fallback." - ) + + # Before accessing any frames, status should be unknown + assert not ref_dec.cpu_fallback.status_known + + ref_frame = ref_dec.get_frame_at(0) + + assert "FFmpeg CUDA" in str(ref_dec.cpu_fallback) + assert ref_dec.cpu_fallback.status_known + assert ref_dec.cpu_fallback with set_cuda_backend("beta"): beta_dec = VideoDecoder(H265_VIDEO.path, device="cuda") - assert ( - _core._get_backend_details(beta_dec._decoder) - == "Beta CUDA Device Interface. Using CPU fallback." - ) + assert "Beta CUDA" in str(beta_dec.cpu_fallback) + # For beta interface, status is known immediately + assert beta_dec.cpu_fallback.status_known + assert beta_dec.cpu_fallback + beta_frame = beta_dec.get_frame_at(0) - assert psnr(ref_frames.data, beta_frame.data) > 25 + assert psnr(ref_frame.data, beta_frame.data) > 25 @needs_cuda def test_beta_cuda_interface_error(self): @@ -1715,7 +1721,7 @@ def test_set_cuda_backend(self): # Check that the default is the ffmpeg backend assert _get_cuda_backend() == "ffmpeg" dec = VideoDecoder(H265_VIDEO.path, device="cuda") - assert _core._get_backend_details(dec._decoder).startswith("FFmpeg CUDA") + assert "FFmpeg CUDA" in str(dec.cpu_fallback) # Check the setting "beta" effectively uses the BETA backend. # We also show that the affects decoder creation only. When the decoder @@ -1724,9 +1730,9 @@ def test_set_cuda_backend(self): with set_cuda_backend("beta"): dec = VideoDecoder(H265_VIDEO.path, device="cuda") assert _get_cuda_backend() == "ffmpeg" - assert _core._get_backend_details(dec._decoder).startswith("Beta CUDA") + assert "Beta CUDA" in str(dec.cpu_fallback) with set_cuda_backend("ffmpeg"): - assert _core._get_backend_details(dec._decoder).startswith("Beta CUDA") + assert "Beta CUDA" in str(dec.cpu_fallback) # Hacky way to ensure passing "cuda:1" is supported by both backends. We # just check that there's an error when passing cuda:N where N is too @@ -1737,6 +1743,53 @@ def test_set_cuda_backend(self): with set_cuda_backend(backend): VideoDecoder(H265_VIDEO.path, device=f"cuda:{bad_device_number}") + def test_cpu_fallback_no_fallback_on_cpu_device(self): + """Test that CPU device doesn't trigger fallback (it's not a fallback scenario).""" + decoder = VideoDecoder(NASA_VIDEO.path, device="cpu") + + assert decoder.cpu_fallback.status_known + _ = decoder[0] + + assert not decoder.cpu_fallback + assert "No fallback required" in str(decoder.cpu_fallback) + + @needs_cuda + @pytest.mark.parametrize("device", cuda_devices()) + def test_cpu_fallback_h265_video(self, device): + """Test that H265 video triggers CPU fallback on CUDA interfaces.""" + # H265_VIDEO is known to trigger CPU fallback on CUDA + # because its dimensions are too small + decoder, _ = make_video_decoder(H265_VIDEO.path, device=device) + + if "beta" in device: + # For beta interface, status is known immediately + assert decoder.cpu_fallback.status_known + else: + # For FFmpeg interface, status is unknown until first frame is decoded + assert not decoder.cpu_fallback.status_known + + decoder.get_frame_at(0) + + assert decoder.cpu_fallback.status_known + assert decoder.cpu_fallback + if "beta" in device: + # Beta interface provides the specific reason for fallback + assert "Video not supported" in str(decoder.cpu_fallback) + else: + # FFmpeg interface doesn't know the specific reason + assert "Unknown reason" in str(decoder.cpu_fallback) + + @needs_cuda + @pytest.mark.parametrize("device", cuda_devices()) + def test_cpu_fallback_no_fallback_on_supported_video(self, device): + """Test that supported videos don't trigger fallback on CUDA.""" + decoder, _ = make_video_decoder(NASA_VIDEO.path, device=device) + + decoder[0] + + assert not decoder.cpu_fallback + assert "No fallback required" in str(decoder.cpu_fallback) + class TestAudioDecoder: @pytest.mark.parametrize("asset", (NASA_AUDIO, NASA_AUDIO_MP3, SINE_MONO_S32)) diff --git a/test/utils.py b/test/utils.py index 6f4c77b76..6ccb52bd7 100644 --- a/test/utils.py +++ b/test/utils.py @@ -51,6 +51,13 @@ def all_supported_devices(): ) +def cuda_devices(): + return ( + pytest.param("cuda", marks=pytest.mark.needs_cuda), + pytest.param(_CUDA_BETA_DEVICE_STR, marks=pytest.mark.needs_cuda), + ) + + def unsplit_device_str(device_str: str) -> str: # helper meant to be used as # device, device_variant = unsplit_device_str(device)