From 8179fb6c184086928859e627ec06e138a2c9b5ca Mon Sep 17 00:00:00 2001 From: crueter Date: Sat, 6 Dec 2025 13:46:16 -0500 Subject: [PATCH] [ffmpeg] update to 8.0.1, enable VTBox/mediacodec/AMF/d3d12 hwaccels MediaCodec needs some jvm shenanigans to work, but the others should Just Work(TM). Need tests on Windows (AMD/Intel), macOS Signed-off-by: crueter --- externals/ffmpeg/cpmfile.json | 6 +- src/video_core/host1x/ffmpeg/ffmpeg.cpp | 394 ++++++++++++------------ 2 files changed, 205 insertions(+), 195 deletions(-) diff --git a/externals/ffmpeg/cpmfile.json b/externals/ffmpeg/cpmfile.json index 0d0a612eda..405d060411 100644 --- a/externals/ffmpeg/cpmfile.json +++ b/externals/ffmpeg/cpmfile.json @@ -1,8 +1,8 @@ { "ffmpeg": { "repo": "FFmpeg/FFmpeg", - "sha": "ddf443f1e9", - "hash": "ded1c313843f23805102565bd3ca92602fb9c2951e059ca5e1a486ab3ef7d589acccf3cde05c5ff0cfc5199c3a261dccb4d2a93254e585824850696fb41a292e", + "sha": "5e56937b74", + "hash": "9ab0457dcd6ce6359b5053c1662f57910d332f68ca0cca9d4134d858464840917027374de3d97e0863c3a7daaea2fe4f4cd17d1c6d8e7f740f4ad91e71c2932b", "bundled": true }, "ffmpeg-ci": { @@ -10,7 +10,7 @@ "package": "FFmpeg", "name": "ffmpeg", "repo": "crueter-ci/FFmpeg", - "version": "8.0-ddf443f1e9", + "version": "8.0.1-5e56937b74", "min_version": "4.1" } } diff --git a/src/video_core/host1x/ffmpeg/ffmpeg.cpp b/src/video_core/host1x/ffmpeg/ffmpeg.cpp index bbbbe615ce..f4858bc869 100644 --- a/src/video_core/host1x/ffmpeg/ffmpeg.cpp +++ b/src/video_core/host1x/ffmpeg/ffmpeg.cpp @@ -17,6 +17,8 @@ extern "C" { // for querying VAAPI driver information #include #endif + +#include } namespace FFmpeg { @@ -27,282 +29,290 @@ constexpr AVPixelFormat PreferredGpuFormat = AV_PIX_FMT_NV12; constexpr AVPixelFormat PreferredCpuFormat = AV_PIX_FMT_YUV420P; constexpr std::array PreferredGpuDecoders = { #if defined (_WIN32) - AV_HWDEVICE_TYPE_CUDA, + AV_HWDEVICE_TYPE_CUDA, + AV_HWDEVICE_TYPE_D3D12VA, AV_HWDEVICE_TYPE_D3D11VA, - AV_HWDEVICE_TYPE_DXVA2, + AV_HWDEVICE_TYPE_DXVA2, #elif defined(__FreeBSD__) - AV_HWDEVICE_TYPE_VDPAU, + AV_HWDEVICE_TYPE_VDPAU, +#elif defined(__APPLE__) + AV_HWDEVICE_TYPE_VIDEOTOOLBOX, +#elif defined(ANDROID) + AV_HWDEVICE_TYPE_MEDIACODEC, #elif defined(__unix__) - AV_HWDEVICE_TYPE_CUDA, - AV_HWDEVICE_TYPE_VAAPI, - AV_HWDEVICE_TYPE_VDPAU, + AV_HWDEVICE_TYPE_CUDA, + AV_HWDEVICE_TYPE_VAAPI, + AV_HWDEVICE_TYPE_VDPAU, #endif AV_HWDEVICE_TYPE_VULKAN, +#if defined(__linux__) || defined(__MINGW64__) + AV_HWDEVICE_TYPE_AMF, +#endif }; AVPixelFormat GetGpuFormat(AVCodecContext* codec_context, const AVPixelFormat* pix_fmts) { - const auto desc = av_pix_fmt_desc_get(codec_context->pix_fmt); - if (desc && !(desc->flags & AV_PIX_FMT_FLAG_HWACCEL)) { - for (int i = 0;; i++) { - const AVCodecHWConfig* config = avcodec_get_hw_config(codec_context->codec, i); - if (!config) { - break; - } - - for (const auto type : PreferredGpuDecoders) { - if (config->methods & AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX && config->device_type == type) { - codec_context->pix_fmt = config->pix_fmt; - } - } - } - } - - for (const AVPixelFormat* p = pix_fmts; *p != AV_PIX_FMT_NONE; ++p) { - if (*p == codec_context->pix_fmt) { - return codec_context->pix_fmt; - } - } - - LOG_INFO(HW_GPU, "Could not find supported GPU pixel format, falling back to CPU decoder"); - av_buffer_unref(&codec_context->hw_device_ctx); - codec_context->pix_fmt = PreferredCpuFormat; - return codec_context->pix_fmt; + const auto desc = av_pix_fmt_desc_get(codec_context->pix_fmt); + if (desc && !(desc->flags & AV_PIX_FMT_FLAG_HWACCEL)) { + for (int i = 0;; i++) { + const AVCodecHWConfig* config = avcodec_get_hw_config(codec_context->codec, i); + if (!config) { + break; + } + + for (const auto type : PreferredGpuDecoders) { + if (config->methods & AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX && config->device_type == type) { + codec_context->pix_fmt = config->pix_fmt; + } + } + } + } + + for (const AVPixelFormat* p = pix_fmts; *p != AV_PIX_FMT_NONE; ++p) { + if (*p == codec_context->pix_fmt) { + return codec_context->pix_fmt; + } + } + + LOG_INFO(HW_GPU, "Could not find supported GPU pixel format, falling back to CPU decoder"); + av_buffer_unref(&codec_context->hw_device_ctx); + codec_context->pix_fmt = PreferredCpuFormat; + return codec_context->pix_fmt; } std::string AVError(int errnum) { - char errbuf[AV_ERROR_MAX_STRING_SIZE] = {}; - av_make_error_string(errbuf, sizeof(errbuf) - 1, errnum); - return errbuf; + char errbuf[AV_ERROR_MAX_STRING_SIZE] = {}; + av_make_error_string(errbuf, sizeof(errbuf) - 1, errnum); + return errbuf; } } Packet::Packet(std::span data) { - m_packet = av_packet_alloc(); - m_packet->data = const_cast(data.data()); - m_packet->size = static_cast(data.size()); + m_packet = av_packet_alloc(); + m_packet->data = const_cast(data.data()); + m_packet->size = static_cast(data.size()); } Packet::~Packet() { - av_packet_free(&m_packet); + av_packet_free(&m_packet); } Frame::Frame() { - m_frame = av_frame_alloc(); + m_frame = av_frame_alloc(); } Frame::~Frame() { - av_frame_free(&m_frame); + av_frame_free(&m_frame); } Decoder::Decoder(Tegra::Host1x::NvdecCommon::VideoCodec codec) { - const AVCodecID av_codec = [&] { - switch (codec) { - case Tegra::Host1x::NvdecCommon::VideoCodec::H264: - return AV_CODEC_ID_H264; - case Tegra::Host1x::NvdecCommon::VideoCodec::VP8: - return AV_CODEC_ID_VP8; - case Tegra::Host1x::NvdecCommon::VideoCodec::VP9: - return AV_CODEC_ID_VP9; - default: - UNIMPLEMENTED_MSG("Unknown codec {}", codec); - return AV_CODEC_ID_NONE; - } - }(); - - m_codec = avcodec_find_decoder(av_codec); + const AVCodecID av_codec = [&] { + switch (codec) { + case Tegra::Host1x::NvdecCommon::VideoCodec::H264: + return AV_CODEC_ID_H264; + case Tegra::Host1x::NvdecCommon::VideoCodec::VP8: + return AV_CODEC_ID_VP8; + case Tegra::Host1x::NvdecCommon::VideoCodec::VP9: + return AV_CODEC_ID_VP9; + default: + UNIMPLEMENTED_MSG("Unknown codec {}", codec); + return AV_CODEC_ID_NONE; + } + }(); + + m_codec = avcodec_find_decoder(av_codec); } bool Decoder::SupportsDecodingOnDevice(AVPixelFormat* out_pix_fmt, AVHWDeviceType type) const { - for (int i = 0;; i++) { - const AVCodecHWConfig* config = avcodec_get_hw_config(m_codec, i); - if (!config) { - LOG_DEBUG(HW_GPU, "{} decoder does not support device type {}", m_codec->name, av_hwdevice_get_type_name(type)); - break; - } - - if (config->methods & AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX && config->device_type == type) { - LOG_INFO(HW_GPU, "Using {} GPU decoder", av_hwdevice_get_type_name(type)); - *out_pix_fmt = config->pix_fmt; - return true; - } - } - - return false; + for (int i = 0;; i++) { + const AVCodecHWConfig* config = avcodec_get_hw_config(m_codec, i); + if (!config) { + LOG_DEBUG(HW_GPU, "{} decoder does not support device type {}", m_codec->name, av_hwdevice_get_type_name(type)); + break; + } + + if (config->methods & AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX && config->device_type == type) { + LOG_INFO(HW_GPU, "Using {} GPU decoder", av_hwdevice_get_type_name(type)); + *out_pix_fmt = config->pix_fmt; + return true; + } + } + + return false; } std::vector HardwareContext::GetSupportedDeviceTypes() { - std::vector types; - AVHWDeviceType current_device_type = AV_HWDEVICE_TYPE_NONE; + std::vector types; + AVHWDeviceType current_device_type = AV_HWDEVICE_TYPE_NONE; - while (true) { - current_device_type = av_hwdevice_iterate_types(current_device_type); - if (current_device_type == AV_HWDEVICE_TYPE_NONE) { - return types; - } + while (true) { + current_device_type = av_hwdevice_iterate_types(current_device_type); + if (current_device_type == AV_HWDEVICE_TYPE_NONE) { + return types; + } - types.push_back(current_device_type); - } + types.push_back(current_device_type); + } } HardwareContext::~HardwareContext() { - av_buffer_unref(&m_gpu_decoder); + av_buffer_unref(&m_gpu_decoder); } bool HardwareContext::InitializeForDecoder(DecoderContext& decoder_context, const Decoder& decoder) { - const auto supported_types = GetSupportedDeviceTypes(); - for (const auto type : PreferredGpuDecoders) { - AVPixelFormat hw_pix_fmt; - - if (std::ranges::find(supported_types, type) == supported_types.end()) { - LOG_DEBUG(HW_GPU, "{} explicitly unsupported", av_hwdevice_get_type_name(type)); - continue; - } - - if (!this->InitializeWithType(type)) { - continue; - } - - if (decoder.SupportsDecodingOnDevice(&hw_pix_fmt, type)) { - decoder_context.InitializeHardwareDecoder(*this, hw_pix_fmt); - return true; - } - } - - return false; + const auto supported_types = GetSupportedDeviceTypes(); + for (const auto type : PreferredGpuDecoders) { + AVPixelFormat hw_pix_fmt; + + if (std::ranges::find(supported_types, type) == supported_types.end()) { + LOG_DEBUG(HW_GPU, "{} explicitly unsupported", av_hwdevice_get_type_name(type)); + continue; + } + + if (!this->InitializeWithType(type)) { + continue; + } + + if (decoder.SupportsDecodingOnDevice(&hw_pix_fmt, type)) { + decoder_context.InitializeHardwareDecoder(*this, hw_pix_fmt); + return true; + } + } + + return false; } bool HardwareContext::InitializeWithType(AVHWDeviceType type) { - av_buffer_unref(&m_gpu_decoder); + av_buffer_unref(&m_gpu_decoder); - if (const int ret = av_hwdevice_ctx_create(&m_gpu_decoder, type, nullptr, nullptr, 0); ret < 0) { - LOG_DEBUG(HW_GPU, "av_hwdevice_ctx_create({}) failed: {}", av_hwdevice_get_type_name(type), AVError(ret)); - return false; - } + if (const int ret = av_hwdevice_ctx_create(&m_gpu_decoder, type, nullptr, nullptr, 0); ret < 0) { + LOG_DEBUG(HW_GPU, "av_hwdevice_ctx_create({}) failed: {}", av_hwdevice_get_type_name(type), AVError(ret)); + return false; + } #ifdef LIBVA_FOUND - if (type == AV_HWDEVICE_TYPE_VAAPI) { - // We need to determine if this is an impersonated VAAPI driver. - auto* hwctx = reinterpret_cast(m_gpu_decoder->data); - auto* vactx = static_cast(hwctx->hwctx); - const char* vendor_name = vaQueryVendorString(vactx->display); - if (strstr(vendor_name, "VDPAU backend")) { - // VDPAU impersonated VAAPI impls are super buggy, we need to skip them. - LOG_DEBUG(HW_GPU, "Skipping VDPAU impersonated VAAPI driver"); - return false; - } else { - // According to some user testing, certain VAAPI drivers (Intel?) could be buggy. - // Log the driver name just in case. - LOG_DEBUG(HW_GPU, "Using VAAPI driver: {}", vendor_name); - } - } + if (type == AV_HWDEVICE_TYPE_VAAPI) { + // We need to determine if this is an impersonated VAAPI driver. + auto* hwctx = reinterpret_cast(m_gpu_decoder->data); + auto* vactx = static_cast(hwctx->hwctx); + const char* vendor_name = vaQueryVendorString(vactx->display); + if (strstr(vendor_name, "VDPAU backend")) { + // VDPAU impersonated VAAPI impls are super buggy, we need to skip them. + LOG_DEBUG(HW_GPU, "Skipping VDPAU impersonated VAAPI driver"); + return false; + } else { + // According to some user testing, certain VAAPI drivers (Intel?) could be buggy. + // Log the driver name just in case. + LOG_DEBUG(HW_GPU, "Using VAAPI driver: {}", vendor_name); + } + } #endif - return true; + return true; } DecoderContext::DecoderContext(const Decoder& decoder) : m_decoder{decoder} { - m_codec_context = avcodec_alloc_context3(m_decoder.GetCodec()); - av_opt_set(m_codec_context->priv_data, "tune", "zerolatency", 0); - m_codec_context->thread_count = 0; - m_codec_context->thread_type &= ~FF_THREAD_FRAME; + m_codec_context = avcodec_alloc_context3(m_decoder.GetCodec()); + av_opt_set(m_codec_context->priv_data, "tune", "zerolatency", 0); + m_codec_context->thread_count = 0; + m_codec_context->thread_type &= ~FF_THREAD_FRAME; } DecoderContext::~DecoderContext() { - av_buffer_unref(&m_codec_context->hw_device_ctx); - avcodec_free_context(&m_codec_context); + av_buffer_unref(&m_codec_context->hw_device_ctx); + avcodec_free_context(&m_codec_context); } void DecoderContext::InitializeHardwareDecoder(const HardwareContext& context, AVPixelFormat hw_pix_fmt) { - m_codec_context->hw_device_ctx = av_buffer_ref(context.GetBufferRef()); - m_codec_context->get_format = GetGpuFormat; - m_codec_context->pix_fmt = hw_pix_fmt; + m_codec_context->hw_device_ctx = av_buffer_ref(context.GetBufferRef()); + m_codec_context->get_format = GetGpuFormat; + m_codec_context->pix_fmt = hw_pix_fmt; } bool DecoderContext::OpenContext(const Decoder& decoder) { - if (const int ret = avcodec_open2(m_codec_context, decoder.GetCodec(), nullptr); ret < 0) { - LOG_ERROR(HW_GPU, "avcodec_open2 error: {}", AVError(ret)); - return false; - } + if (const int ret = avcodec_open2(m_codec_context, decoder.GetCodec(), nullptr); ret < 0) { + LOG_ERROR(HW_GPU, "avcodec_open2 error: {}", AVError(ret)); + return false; + } - if (!m_codec_context->hw_device_ctx) { - LOG_INFO(HW_GPU, "Using FFmpeg CPU decoder"); - } + if (!m_codec_context->hw_device_ctx) { + LOG_INFO(HW_GPU, "Using FFmpeg CPU decoder"); + } - return true; + return true; } bool DecoderContext::SendPacket(const Packet& packet) { - if (const int ret = avcodec_send_packet(m_codec_context, packet.GetPacket()); ret < 0 && ret != AVERROR_EOF && ret != AVERROR(EAGAIN)) { - LOG_ERROR(HW_GPU, "avcodec_send_packet error: {}", AVError(ret)); - return false; - } + if (const int ret = avcodec_send_packet(m_codec_context, packet.GetPacket()); ret < 0 && ret != AVERROR_EOF && ret != AVERROR(EAGAIN)) { + LOG_ERROR(HW_GPU, "avcodec_send_packet error: {}", AVError(ret)); + return false; + } - return true; + return true; } std::shared_ptr DecoderContext::ReceiveFrame() { - auto ReceiveImpl = [&](AVFrame* frame) -> int { + auto ReceiveImpl = [&](AVFrame* frame) -> int { const int ret = avcodec_receive_frame(m_codec_context, frame); - if (ret < 0 && ret != AVERROR_EOF && ret != AVERROR(EAGAIN)) { - LOG_ERROR(HW_GPU, "avcodec_receive_frame error: {}", AVError(ret)); - } - return ret; - }; - - std::shared_ptr intermediate_frame = std::make_shared(); - if (ReceiveImpl(intermediate_frame->GetFrame()) < 0) { - return {}; - } - - m_final_frame = std::make_shared(); - if (m_codec_context->hw_device_ctx) { - m_final_frame->SetFormat(PreferredGpuFormat); - if (const int ret = av_hwframe_transfer_data(m_final_frame->GetFrame(), intermediate_frame->GetFrame(), 0); ret < 0) { - LOG_ERROR(HW_GPU, "av_hwframe_transfer_data error: {}", AVError(ret)); - return {}; - } - } else { - m_final_frame = std::move(intermediate_frame); - } - - return std::move(m_final_frame); + if (ret < 0 && ret != AVERROR_EOF && ret != AVERROR(EAGAIN)) { + LOG_ERROR(HW_GPU, "avcodec_receive_frame error: {}", AVError(ret)); + } + return ret; + }; + + std::shared_ptr intermediate_frame = std::make_shared(); + if (ReceiveImpl(intermediate_frame->GetFrame()) < 0) { + return {}; + } + + m_final_frame = std::make_shared(); + if (m_codec_context->hw_device_ctx) { + m_final_frame->SetFormat(PreferredGpuFormat); + if (const int ret = av_hwframe_transfer_data(m_final_frame->GetFrame(), intermediate_frame->GetFrame(), 0); ret < 0) { + LOG_ERROR(HW_GPU, "av_hwframe_transfer_data error: {}", AVError(ret)); + return {}; + } + } else { + m_final_frame = std::move(intermediate_frame); + } + + return std::move(m_final_frame); } void DecodeApi::Reset() { - m_hardware_context.reset(); - m_decoder_context.reset(); - m_decoder.reset(); + m_hardware_context.reset(); + m_decoder_context.reset(); + m_decoder.reset(); } bool DecodeApi::Initialize(Tegra::Host1x::NvdecCommon::VideoCodec codec) { - this->Reset(); - m_decoder.emplace(codec); - m_decoder_context.emplace(*m_decoder); - - // Enable GPU decoding if requested. - if (Settings::values.nvdec_emulation.GetValue() == Settings::NvdecEmulation::Gpu) { - m_hardware_context.emplace(); - m_hardware_context->InitializeForDecoder(*m_decoder_context, *m_decoder); - } - - // Open the decoder context. - if (!m_decoder_context->OpenContext(*m_decoder)) { - this->Reset(); - return false; - } - - return true; + this->Reset(); + m_decoder.emplace(codec); + m_decoder_context.emplace(*m_decoder); + + // Enable GPU decoding if requested. + if (Settings::values.nvdec_emulation.GetValue() == Settings::NvdecEmulation::Gpu) { + m_hardware_context.emplace(); + m_hardware_context->InitializeForDecoder(*m_decoder_context, *m_decoder); + } + + // Open the decoder context. + if (!m_decoder_context->OpenContext(*m_decoder)) { + this->Reset(); + return false; + } + + return true; } bool DecodeApi::SendPacket(std::span packet_data) { - FFmpeg::Packet packet(packet_data); - return m_decoder_context->SendPacket(packet); + FFmpeg::Packet packet(packet_data); + return m_decoder_context->SendPacket(packet); } std::shared_ptr DecodeApi::ReceiveFrame() { - // Receive raw frame from decoder. - return m_decoder_context->ReceiveFrame(); + // Receive raw frame from decoder. + return m_decoder_context->ReceiveFrame(); } }