From 251314c142c4800b0a16514d04b358cdd10cc316 Mon Sep 17 00:00:00 2001 From: CamilleLaVey Date: Fri, 7 Nov 2025 14:01:57 -0400 Subject: [PATCH] Initial Implementation for Android Native Hardware Decode --- .../yuzu/yuzu_emu/media/NativeMediaCodec.java | 162 ++++++ src/video_core/CMakeLists.txt | 6 + src/video_core/host1x/codecs/decoder.cpp | 5 + src/video_core/host1x/codecs/decoder.h | 6 + src/video_core/host1x/codecs/h264.cpp | 16 + src/video_core/host1x/codecs/h264.h | 3 + src/video_core/host1x/codecs/vp8.cpp | 9 + src/video_core/host1x/codecs/vp8.h | 3 + src/video_core/host1x/codecs/vp9.cpp | 9 + src/video_core/host1x/codecs/vp9.h | 3 + src/video_core/host1x/ffmpeg/ffmpeg.cpp | 516 +++++++++++------- src/video_core/host1x/ffmpeg/ffmpeg.h | 10 + .../host1x/ffmpeg/mediacodec_bridge.h | 22 + .../ffmpeg/mediacodec_bridge_android.cpp | 114 ++++ .../vulkan_common/vulkan_device.cpp | 20 - src/video_core/vulkan_common/vulkan_device.h | 6 + 16 files changed, 698 insertions(+), 212 deletions(-) create mode 100644 src/android/app/src/main/java/org/yuzu/yuzu_emu/media/NativeMediaCodec.java create mode 100644 src/video_core/host1x/ffmpeg/mediacodec_bridge.h create mode 100644 src/video_core/host1x/ffmpeg/mediacodec_bridge_android.cpp diff --git a/src/android/app/src/main/java/org/yuzu/yuzu_emu/media/NativeMediaCodec.java b/src/android/app/src/main/java/org/yuzu/yuzu_emu/media/NativeMediaCodec.java new file mode 100644 index 0000000000..70bd5f775d --- /dev/null +++ b/src/android/app/src/main/java/org/yuzu/yuzu_emu/media/NativeMediaCodec.java @@ -0,0 +1,162 @@ +// SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +package org.yuzu.yuzu_emu.media; + +import android.media.Image; +import android.media.MediaCodec; +import android.media.MediaCodecInfo; +import android.media.MediaFormat; +import android.os.Build; +import android.util.Log; + +import java.nio.ByteBuffer; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; + +public class NativeMediaCodec { + private static final String TAG = "NativeMediaCodec"; + private static final ConcurrentHashMap decoders = new ConcurrentHashMap<>(); + private static final AtomicInteger nextId = new AtomicInteger(1); + + // Called from native code to create a decoder for the given mime (e.g. "video/avc"). + // Returns a decoder id (>0) on success, or 0 on failure. + public static int createDecoder(String mime, int width, int height) { + try { + MediaCodec codec = MediaCodec.createDecoderByType(mime); + MediaFormat format = MediaFormat.createVideoFormat(mime, width, height); + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) { + format.setInteger(MediaFormat.KEY_COLOR_FORMAT, + MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Flexible); + } + final int id = nextId.getAndIncrement(); + decoders.put(id, codec); + // Request YUV_420_888 output (Image) if available + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { + codec.setCallback(new MediaCodec.Callback() { + private final int decoderId = id; + + @Override + public void onInputBufferAvailable(MediaCodec mc, int index) { + // input will be fed by native code via dequeue + } + + @Override + public void onOutputBufferAvailable(MediaCodec mc, int index, MediaCodec.BufferInfo info) { + try { + Image image = mc.getOutputImage(index); + if (image != null) { + byte[] data = ImageToNV12(image); + onFrameDecoded(decoderId, data, image.getWidth(), image.getHeight(), info.presentationTimeUs); + image.close(); + } + } catch (Throwable t) { + Log.w(TAG, "onOutputBufferAvailable failed: " + t); + } finally { + try { mc.releaseOutputBuffer(index, false); } catch (Throwable ignored) {} + } + } + + @Override + public void onError(MediaCodec mc, MediaCodec.CodecException e) { + Log.w(TAG, "MediaCodec error: " + e); + } + + @Override + public void onOutputFormatChanged(MediaCodec mc, MediaFormat format) { + Log.i(TAG, "Output format changed: " + format); + } + }); + } + + codec.configure(format, null, null, 0); + codec.start(); + return id; + } catch (Exception e) { + Log.w(TAG, "createDecoder failed: " + e); + return 0; + } + } + + private static byte[] ImageToNV12(Image image) { + // Convert YUV_420_888 to NV12 (Y plane, interleaved UV) + final Image.Plane[] planes = image.getPlanes(); + int w = image.getWidth(); + int h = image.getHeight(); + int ySize = w * h; + int chromaWidth = (w + 1) / 2; + int chromaHeight = (h + 1) / 2; + int uvRowStrideOut = chromaWidth * 2; + int uvSize = uvRowStrideOut * chromaHeight; + byte[] out = new byte[ySize + uvSize]; + + Image.Plane yPlane = planes[0]; + ByteBuffer yBuffer = yPlane.getBuffer().duplicate(); + int yRowStride = yPlane.getRowStride(); + int yPixelStride = yPlane.getPixelStride(); + for (int row = 0; row < h; row++) { + int srcRow = row * yRowStride; + int dstRow = row * w; + for (int col = 0; col < w; col++) { + out[dstRow + col] = yBuffer.get(srcRow + col * yPixelStride); + } + } + + Image.Plane uPlane = planes[1]; + Image.Plane vPlane = planes[2]; + ByteBuffer uBuffer = uPlane.getBuffer().duplicate(); + ByteBuffer vBuffer = vPlane.getBuffer().duplicate(); + int uRowStride = uPlane.getRowStride(); + int vRowStride = vPlane.getRowStride(); + int uPixelStride = uPlane.getPixelStride(); + int vPixelStride = vPlane.getPixelStride(); + + int uvOffset = ySize; + for (int row = 0; row < chromaHeight; row++) { + int uRow = row * uRowStride; + int vRow = row * vRowStride; + int dstRow = uvOffset + row * uvRowStrideOut; + for (int col = 0; col < chromaWidth; col++) { + int dst = dstRow + col * 2; + out[dst] = uBuffer.get(uRow + col * uPixelStride); + out[dst + 1] = vBuffer.get(vRow + col * vPixelStride); + } + } + return out; + } + + // Native callback to deliver decoded frames to native code + private static native void onFrameDecoded(int decoderId, byte[] data, int width, int height, long pts); + + // Called from native code to feed packet data to decoder + public static boolean decode(int decoderId, byte[] packet, long pts) { + MediaCodec codec = decoders.get(decoderId); + if (codec == null) return false; + try { + int inputIndex = codec.dequeueInputBuffer(10000); + if (inputIndex >= 0) { + ByteBuffer inputBuf = codec.getInputBuffer(inputIndex); + if (inputBuf == null) { + Log.w(TAG, "decode input buffer null"); + codec.queueInputBuffer(inputIndex, 0, 0, pts, 0); + return false; + } + inputBuf.clear(); + inputBuf.put(packet); + codec.queueInputBuffer(inputIndex, 0, packet.length, pts, 0); + } + return true; + } catch (Exception e) { + Log.w(TAG, "decode error: " + e); + return false; + } + } + + public static void releaseDecoder(int decoderId) { + MediaCodec codec = decoders.remove(decoderId); + if (codec != null) { + try { codec.stop(); } catch (Throwable ignored) {} + try { codec.release(); } catch (Throwable ignored) {} + } + } +} diff --git a/src/video_core/CMakeLists.txt b/src/video_core/CMakeLists.txt index f437663963..d6e9864282 100644 --- a/src/video_core/CMakeLists.txt +++ b/src/video_core/CMakeLists.txt @@ -329,6 +329,12 @@ target_include_directories(video_core PRIVATE ${FFmpeg_INCLUDE_DIR}) target_link_libraries(video_core PRIVATE ${FFmpeg_LIBRARIES}) target_link_options(video_core PRIVATE ${FFmpeg_LDFLAGS}) + # Android-specific Java/ JNI bridge for MediaCodec + +if(ANDROID) + target_sources(video_core PRIVATE host1x/ffmpeg/mediacodec_bridge_android.cpp) +endif() + add_dependencies(video_core host_shaders) target_include_directories(video_core PRIVATE ${HOST_SHADERS_INCLUDE}) diff --git a/src/video_core/host1x/codecs/decoder.cpp b/src/video_core/host1x/codecs/decoder.cpp index 887eb28c8c..71a96f45f3 100755 --- a/src/video_core/host1x/codecs/decoder.cpp +++ b/src/video_core/host1x/codecs/decoder.cpp @@ -25,6 +25,11 @@ void Decoder::Decode() { } const auto packet_data = ComposeFrame(); +#ifdef __ANDROID__ + if (const auto frame_dims = CurrentFrameDimensions()) { + decode_api.EnsureMediaCodecDecoder(frame_dims->first, frame_dims->second); + } +#endif // Send assembled bitstream to decoder. if (!decode_api.SendPacket(packet_data)) { return; diff --git a/src/video_core/host1x/codecs/decoder.h b/src/video_core/host1x/codecs/decoder.h index 22e6db8151..9477a035f4 100755 --- a/src/video_core/host1x/codecs/decoder.h +++ b/src/video_core/host1x/codecs/decoder.h @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -40,6 +41,11 @@ public: /// Return name of the current codec [[nodiscard]] virtual std::string_view GetCurrentCodecName() const = 0; + /// Returns the current frame dimensions if available + [[nodiscard]] virtual std::optional> CurrentFrameDimensions() const { + return std::nullopt; + } + protected: explicit Decoder(Host1x::Host1x& host1x, s32 id, const Host1x::NvdecCommon::NvdecRegisters& regs, diff --git a/src/video_core/host1x/codecs/h264.cpp b/src/video_core/host1x/codecs/h264.cpp index 0896fa6001..74bc5cb47e 100644 --- a/src/video_core/host1x/codecs/h264.cpp +++ b/src/video_core/host1x/codecs/h264.cpp @@ -318,4 +318,20 @@ void H264BitWriter::Flush() { buffer = 0; buffer_pos = 0; } + +std::optional> H264::CurrentFrameDimensions() const { + const u32 width_mbs = current_context.h264_parameter_set.pic_width_in_mbs; + const u32 height_mbs = current_context.h264_parameter_set.frame_height_in_mbs; + if (width_mbs == 0 || height_mbs == 0) { + return std::nullopt; + } + const bool frame_mbs_only = current_context.h264_parameter_set.frame_mbs_only_flag != 0; + const u32 pic_height_mbs = height_mbs / (frame_mbs_only ? 1u : 2u); + const int width = static_cast(width_mbs) * 16; + const int height = static_cast(pic_height_mbs) * 16; + if (width <= 0 || height <= 0) { + return std::nullopt; + } + return std::pair{width, height}; +} } // namespace Tegra::Decoders diff --git a/src/video_core/host1x/codecs/h264.h b/src/video_core/host1x/codecs/h264.h index d946c6937d..10a0be8d01 100644 --- a/src/video_core/host1x/codecs/h264.h +++ b/src/video_core/host1x/codecs/h264.h @@ -3,6 +3,7 @@ #pragma once +#include #include #include @@ -262,6 +263,8 @@ public: return "H264"; } + [[nodiscard]] std::optional> CurrentFrameDimensions() const override; + private: bool is_first_frame{true}; Common::ScratchBuffer frame_scratch; diff --git a/src/video_core/host1x/codecs/vp8.cpp b/src/video_core/host1x/codecs/vp8.cpp index 6094f16e0e..8b9f2aede4 100644 --- a/src/video_core/host1x/codecs/vp8.cpp +++ b/src/video_core/host1x/codecs/vp8.cpp @@ -73,4 +73,13 @@ std::span VP8::ComposeFrame() { return frame_scratch; } +std::optional> VP8::CurrentFrameDimensions() const { + const int width = static_cast(current_context.frame_width); + const int height = static_cast(current_context.frame_height); + if (width <= 0 || height <= 0) { + return std::nullopt; + } + return std::pair{width, height}; +} + } // namespace Tegra::Decoders diff --git a/src/video_core/host1x/codecs/vp8.h b/src/video_core/host1x/codecs/vp8.h index 74800281d8..94924affcb 100644 --- a/src/video_core/host1x/codecs/vp8.h +++ b/src/video_core/host1x/codecs/vp8.h @@ -4,6 +4,7 @@ #pragma once #include +#include #include #include "common/common_funcs.h" @@ -51,6 +52,8 @@ public: return "VP8"; } + [[nodiscard]] std::optional> CurrentFrameDimensions() const override; + private: Common::ScratchBuffer frame_scratch; diff --git a/src/video_core/host1x/codecs/vp9.cpp b/src/video_core/host1x/codecs/vp9.cpp index f80709d785..08f5d00e27 100644 --- a/src/video_core/host1x/codecs/vp9.cpp +++ b/src/video_core/host1x/codecs/vp9.cpp @@ -489,6 +489,15 @@ Vp9FrameContainer VP9::GetCurrentFrame() { return current_frame; } +std::optional> VP9::CurrentFrameDimensions() const { + const int width = static_cast(current_frame_info.frame_size.width); + const int height = static_cast(current_frame_info.frame_size.height); + if (width <= 0 || height <= 0) { + return std::nullopt; + } + return std::pair{width, height}; +} + std::vector VP9::ComposeCompressedHeader() { VpxRangeEncoder writer{}; const bool update_probs = !current_frame_info.is_key_frame && current_frame_info.show_frame; diff --git a/src/video_core/host1x/codecs/vp9.h b/src/video_core/host1x/codecs/vp9.h index 9d42033cb3..8c0cf69cd1 100644 --- a/src/video_core/host1x/codecs/vp9.h +++ b/src/video_core/host1x/codecs/vp9.h @@ -4,6 +4,7 @@ #pragma once #include +#include #include #include @@ -136,6 +137,8 @@ public: return "VP9"; } + [[nodiscard]] std::optional> CurrentFrameDimensions() const override; + private: /// Returns true if the most recent frame was a hidden frame. [[nodiscard]] bool WasFrameHidden() const { diff --git a/src/video_core/host1x/ffmpeg/ffmpeg.cpp b/src/video_core/host1x/ffmpeg/ffmpeg.cpp index bbbbe615ce..6315a92a27 100644 --- a/src/video_core/host1x/ffmpeg/ffmpeg.cpp +++ b/src/video_core/host1x/ffmpeg/ffmpeg.cpp @@ -4,12 +4,18 @@ // SPDX-FileCopyrightText: Copyright 2023 yuzu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later +#include +#include + #include "common/assert.h" #include "common/logging/log.h" #include "common/scope_exit.h" #include "common/settings.h" #include "core/memory.h" #include "video_core/host1x/ffmpeg/ffmpeg.h" +#ifdef __ANDROID__ +#include "video_core/host1x/ffmpeg/mediacodec_bridge.h" +#endif #include "video_core/memory_manager.h" extern "C" { @@ -27,282 +33,408 @@ constexpr AVPixelFormat PreferredGpuFormat = AV_PIX_FMT_NV12; constexpr AVPixelFormat PreferredCpuFormat = AV_PIX_FMT_YUV420P; constexpr std::array PreferredGpuDecoders = { #if defined (_WIN32) - AV_HWDEVICE_TYPE_CUDA, + AV_HWDEVICE_TYPE_CUDA, AV_HWDEVICE_TYPE_D3D11VA, - AV_HWDEVICE_TYPE_DXVA2, + AV_HWDEVICE_TYPE_DXVA2, #elif defined(__FreeBSD__) - AV_HWDEVICE_TYPE_VDPAU, + AV_HWDEVICE_TYPE_VDPAU, #elif defined(__unix__) - AV_HWDEVICE_TYPE_CUDA, - AV_HWDEVICE_TYPE_VAAPI, - AV_HWDEVICE_TYPE_VDPAU, + AV_HWDEVICE_TYPE_CUDA, + AV_HWDEVICE_TYPE_VAAPI, + AV_HWDEVICE_TYPE_VDPAU, #endif AV_HWDEVICE_TYPE_VULKAN, }; AVPixelFormat GetGpuFormat(AVCodecContext* codec_context, const AVPixelFormat* pix_fmts) { - const auto desc = av_pix_fmt_desc_get(codec_context->pix_fmt); - if (desc && !(desc->flags & AV_PIX_FMT_FLAG_HWACCEL)) { - for (int i = 0;; i++) { - const AVCodecHWConfig* config = avcodec_get_hw_config(codec_context->codec, i); - if (!config) { - break; - } - - for (const auto type : PreferredGpuDecoders) { - if (config->methods & AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX && config->device_type == type) { - codec_context->pix_fmt = config->pix_fmt; - } - } - } - } - - for (const AVPixelFormat* p = pix_fmts; *p != AV_PIX_FMT_NONE; ++p) { - if (*p == codec_context->pix_fmt) { - return codec_context->pix_fmt; - } - } - - LOG_INFO(HW_GPU, "Could not find supported GPU pixel format, falling back to CPU decoder"); - av_buffer_unref(&codec_context->hw_device_ctx); - codec_context->pix_fmt = PreferredCpuFormat; - return codec_context->pix_fmt; + const auto desc = av_pix_fmt_desc_get(codec_context->pix_fmt); + if (desc && !(desc->flags & AV_PIX_FMT_FLAG_HWACCEL)) { + for (int i = 0;; i++) { + const AVCodecHWConfig* config = avcodec_get_hw_config(codec_context->codec, i); + if (!config) { + break; + } + + for (const auto type : PreferredGpuDecoders) { + if (config->methods & AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX && config->device_type == type) { + codec_context->pix_fmt = config->pix_fmt; + } + } + } + } + + for (const AVPixelFormat* p = pix_fmts; *p != AV_PIX_FMT_NONE; ++p) { + if (*p == codec_context->pix_fmt) { + return codec_context->pix_fmt; + } + } + + LOG_INFO(HW_GPU, "Could not find supported GPU pixel format, falling back to CPU decoder"); + av_buffer_unref(&codec_context->hw_device_ctx); + codec_context->pix_fmt = PreferredCpuFormat; + return codec_context->pix_fmt; } std::string AVError(int errnum) { - char errbuf[AV_ERROR_MAX_STRING_SIZE] = {}; - av_make_error_string(errbuf, sizeof(errbuf) - 1, errnum); - return errbuf; + char errbuf[AV_ERROR_MAX_STRING_SIZE] = {}; + av_make_error_string(errbuf, sizeof(errbuf) - 1, errnum); + return errbuf; } } Packet::Packet(std::span data) { - m_packet = av_packet_alloc(); - m_packet->data = const_cast(data.data()); - m_packet->size = static_cast(data.size()); + m_packet = av_packet_alloc(); + m_packet->data = const_cast(data.data()); + m_packet->size = static_cast(data.size()); } Packet::~Packet() { - av_packet_free(&m_packet); + av_packet_free(&m_packet); } Frame::Frame() { - m_frame = av_frame_alloc(); + m_frame = av_frame_alloc(); } Frame::~Frame() { - av_frame_free(&m_frame); + av_frame_free(&m_frame); } Decoder::Decoder(Tegra::Host1x::NvdecCommon::VideoCodec codec) { - const AVCodecID av_codec = [&] { - switch (codec) { - case Tegra::Host1x::NvdecCommon::VideoCodec::H264: - return AV_CODEC_ID_H264; - case Tegra::Host1x::NvdecCommon::VideoCodec::VP8: - return AV_CODEC_ID_VP8; - case Tegra::Host1x::NvdecCommon::VideoCodec::VP9: - return AV_CODEC_ID_VP9; - default: - UNIMPLEMENTED_MSG("Unknown codec {}", codec); - return AV_CODEC_ID_NONE; - } - }(); - - m_codec = avcodec_find_decoder(av_codec); + const AVCodecID av_codec = [&] { + switch (codec) { + case Tegra::Host1x::NvdecCommon::VideoCodec::H264: + return AV_CODEC_ID_H264; + case Tegra::Host1x::NvdecCommon::VideoCodec::VP8: + return AV_CODEC_ID_VP8; + case Tegra::Host1x::NvdecCommon::VideoCodec::VP9: + return AV_CODEC_ID_VP9; + default: + UNIMPLEMENTED_MSG("Unknown codec {}", codec); + return AV_CODEC_ID_NONE; + } + }(); + + m_codec = avcodec_find_decoder(av_codec); } bool Decoder::SupportsDecodingOnDevice(AVPixelFormat* out_pix_fmt, AVHWDeviceType type) const { - for (int i = 0;; i++) { - const AVCodecHWConfig* config = avcodec_get_hw_config(m_codec, i); - if (!config) { - LOG_DEBUG(HW_GPU, "{} decoder does not support device type {}", m_codec->name, av_hwdevice_get_type_name(type)); - break; - } - - if (config->methods & AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX && config->device_type == type) { - LOG_INFO(HW_GPU, "Using {} GPU decoder", av_hwdevice_get_type_name(type)); - *out_pix_fmt = config->pix_fmt; - return true; - } - } - - return false; + for (int i = 0;; i++) { + const AVCodecHWConfig* config = avcodec_get_hw_config(m_codec, i); + if (!config) { + LOG_DEBUG(HW_GPU, "{} decoder does not support device type {}", m_codec->name, av_hwdevice_get_type_name(type)); + break; + } + + if (config->methods & AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX && config->device_type == type) { + LOG_INFO(HW_GPU, "Using {} GPU decoder", av_hwdevice_get_type_name(type)); + *out_pix_fmt = config->pix_fmt; + return true; + } + } + + return false; } std::vector HardwareContext::GetSupportedDeviceTypes() { - std::vector types; - AVHWDeviceType current_device_type = AV_HWDEVICE_TYPE_NONE; + std::vector types; + AVHWDeviceType current_device_type = AV_HWDEVICE_TYPE_NONE; - while (true) { - current_device_type = av_hwdevice_iterate_types(current_device_type); - if (current_device_type == AV_HWDEVICE_TYPE_NONE) { - return types; - } + while (true) { + current_device_type = av_hwdevice_iterate_types(current_device_type); + if (current_device_type == AV_HWDEVICE_TYPE_NONE) { + return types; + } - types.push_back(current_device_type); - } + types.push_back(current_device_type); + } } HardwareContext::~HardwareContext() { - av_buffer_unref(&m_gpu_decoder); + av_buffer_unref(&m_gpu_decoder); } bool HardwareContext::InitializeForDecoder(DecoderContext& decoder_context, const Decoder& decoder) { - const auto supported_types = GetSupportedDeviceTypes(); - for (const auto type : PreferredGpuDecoders) { - AVPixelFormat hw_pix_fmt; - - if (std::ranges::find(supported_types, type) == supported_types.end()) { - LOG_DEBUG(HW_GPU, "{} explicitly unsupported", av_hwdevice_get_type_name(type)); - continue; - } - - if (!this->InitializeWithType(type)) { - continue; - } - - if (decoder.SupportsDecodingOnDevice(&hw_pix_fmt, type)) { - decoder_context.InitializeHardwareDecoder(*this, hw_pix_fmt); - return true; - } - } - - return false; + const auto supported_types = GetSupportedDeviceTypes(); + for (const auto type : PreferredGpuDecoders) { + AVPixelFormat hw_pix_fmt; + + if (std::ranges::find(supported_types, type) == supported_types.end()) { + LOG_DEBUG(HW_GPU, "{} explicitly unsupported", av_hwdevice_get_type_name(type)); + continue; + } + + if (!this->InitializeWithType(type)) { + continue; + } + + if (decoder.SupportsDecodingOnDevice(&hw_pix_fmt, type)) { + decoder_context.InitializeHardwareDecoder(*this, hw_pix_fmt); + return true; + } + } + + return false; } bool HardwareContext::InitializeWithType(AVHWDeviceType type) { - av_buffer_unref(&m_gpu_decoder); + av_buffer_unref(&m_gpu_decoder); - if (const int ret = av_hwdevice_ctx_create(&m_gpu_decoder, type, nullptr, nullptr, 0); ret < 0) { - LOG_DEBUG(HW_GPU, "av_hwdevice_ctx_create({}) failed: {}", av_hwdevice_get_type_name(type), AVError(ret)); - return false; - } + if (const int ret = av_hwdevice_ctx_create(&m_gpu_decoder, type, nullptr, nullptr, 0); ret < 0) { + LOG_DEBUG(HW_GPU, "av_hwdevice_ctx_create({}) failed: {}", av_hwdevice_get_type_name(type), AVError(ret)); + return false; + } #ifdef LIBVA_FOUND - if (type == AV_HWDEVICE_TYPE_VAAPI) { - // We need to determine if this is an impersonated VAAPI driver. - auto* hwctx = reinterpret_cast(m_gpu_decoder->data); - auto* vactx = static_cast(hwctx->hwctx); - const char* vendor_name = vaQueryVendorString(vactx->display); - if (strstr(vendor_name, "VDPAU backend")) { - // VDPAU impersonated VAAPI impls are super buggy, we need to skip them. - LOG_DEBUG(HW_GPU, "Skipping VDPAU impersonated VAAPI driver"); - return false; - } else { - // According to some user testing, certain VAAPI drivers (Intel?) could be buggy. - // Log the driver name just in case. - LOG_DEBUG(HW_GPU, "Using VAAPI driver: {}", vendor_name); - } - } + if (type == AV_HWDEVICE_TYPE_VAAPI) { + // We need to determine if this is an impersonated VAAPI driver. + auto* hwctx = reinterpret_cast(m_gpu_decoder->data); + auto* vactx = static_cast(hwctx->hwctx); + const char* vendor_name = vaQueryVendorString(vactx->display); + if (strstr(vendor_name, "VDPAU backend")) { + // VDPAU impersonated VAAPI impls are super buggy, we need to skip them. + LOG_DEBUG(HW_GPU, "Skipping VDPAU impersonated VAAPI driver"); + return false; + } else { + // According to some user testing, certain VAAPI drivers (Intel?) could be buggy. + // Log the driver name just in case. + LOG_DEBUG(HW_GPU, "Using VAAPI driver: {}", vendor_name); + } + } #endif - return true; + return true; } DecoderContext::DecoderContext(const Decoder& decoder) : m_decoder{decoder} { - m_codec_context = avcodec_alloc_context3(m_decoder.GetCodec()); - av_opt_set(m_codec_context->priv_data, "tune", "zerolatency", 0); - m_codec_context->thread_count = 0; - m_codec_context->thread_type &= ~FF_THREAD_FRAME; + m_codec_context = avcodec_alloc_context3(m_decoder.GetCodec()); + av_opt_set(m_codec_context->priv_data, "tune", "zerolatency", 0); + m_codec_context->thread_count = 0; + m_codec_context->thread_type &= ~FF_THREAD_FRAME; } DecoderContext::~DecoderContext() { - av_buffer_unref(&m_codec_context->hw_device_ctx); - avcodec_free_context(&m_codec_context); + av_buffer_unref(&m_codec_context->hw_device_ctx); + avcodec_free_context(&m_codec_context); } void DecoderContext::InitializeHardwareDecoder(const HardwareContext& context, AVPixelFormat hw_pix_fmt) { - m_codec_context->hw_device_ctx = av_buffer_ref(context.GetBufferRef()); - m_codec_context->get_format = GetGpuFormat; - m_codec_context->pix_fmt = hw_pix_fmt; + m_codec_context->hw_device_ctx = av_buffer_ref(context.GetBufferRef()); + m_codec_context->get_format = GetGpuFormat; + m_codec_context->pix_fmt = hw_pix_fmt; } bool DecoderContext::OpenContext(const Decoder& decoder) { - if (const int ret = avcodec_open2(m_codec_context, decoder.GetCodec(), nullptr); ret < 0) { - LOG_ERROR(HW_GPU, "avcodec_open2 error: {}", AVError(ret)); - return false; - } + if (const int ret = avcodec_open2(m_codec_context, decoder.GetCodec(), nullptr); ret < 0) { + LOG_ERROR(HW_GPU, "avcodec_open2 error: {}", AVError(ret)); + return false; + } - if (!m_codec_context->hw_device_ctx) { - LOG_INFO(HW_GPU, "Using FFmpeg CPU decoder"); - } + if (!m_codec_context->hw_device_ctx) { + LOG_INFO(HW_GPU, "Using FFmpeg CPU decoder"); + } - return true; + return true; } bool DecoderContext::SendPacket(const Packet& packet) { - if (const int ret = avcodec_send_packet(m_codec_context, packet.GetPacket()); ret < 0 && ret != AVERROR_EOF && ret != AVERROR(EAGAIN)) { - LOG_ERROR(HW_GPU, "avcodec_send_packet error: {}", AVError(ret)); - return false; - } + if (const int ret = avcodec_send_packet(m_codec_context, packet.GetPacket()); ret < 0 && ret != AVERROR_EOF && ret != AVERROR(EAGAIN)) { + LOG_ERROR(HW_GPU, "avcodec_send_packet error: {}", AVError(ret)); + return false; + } - return true; + return true; } std::shared_ptr DecoderContext::ReceiveFrame() { - auto ReceiveImpl = [&](AVFrame* frame) -> int { + auto ReceiveImpl = [&](AVFrame* frame) -> int { const int ret = avcodec_receive_frame(m_codec_context, frame); - if (ret < 0 && ret != AVERROR_EOF && ret != AVERROR(EAGAIN)) { - LOG_ERROR(HW_GPU, "avcodec_receive_frame error: {}", AVError(ret)); - } - return ret; - }; - - std::shared_ptr intermediate_frame = std::make_shared(); - if (ReceiveImpl(intermediate_frame->GetFrame()) < 0) { - return {}; - } - - m_final_frame = std::make_shared(); - if (m_codec_context->hw_device_ctx) { - m_final_frame->SetFormat(PreferredGpuFormat); - if (const int ret = av_hwframe_transfer_data(m_final_frame->GetFrame(), intermediate_frame->GetFrame(), 0); ret < 0) { - LOG_ERROR(HW_GPU, "av_hwframe_transfer_data error: {}", AVError(ret)); - return {}; - } - } else { - m_final_frame = std::move(intermediate_frame); - } - - return std::move(m_final_frame); + if (ret < 0 && ret != AVERROR_EOF && ret != AVERROR(EAGAIN)) { + LOG_ERROR(HW_GPU, "avcodec_receive_frame error: {}", AVError(ret)); + } + return ret; + }; + + std::shared_ptr intermediate_frame = std::make_shared(); + if (ReceiveImpl(intermediate_frame->GetFrame()) < 0) { + return {}; + } + + m_final_frame = std::make_shared(); + if (m_codec_context->hw_device_ctx) { + m_final_frame->SetFormat(PreferredGpuFormat); + if (const int ret = av_hwframe_transfer_data(m_final_frame->GetFrame(), intermediate_frame->GetFrame(), 0); ret < 0) { + LOG_ERROR(HW_GPU, "av_hwframe_transfer_data error: {}", AVError(ret)); + return {}; + } + } else { + m_final_frame = std::move(intermediate_frame); + } + + return std::move(m_final_frame); } void DecodeApi::Reset() { - m_hardware_context.reset(); - m_decoder_context.reset(); - m_decoder.reset(); +#ifdef __ANDROID__ + if (m_mediacodec_decoder_id != 0) { + FFmpeg::MediaCodecBridge::DestroyDecoder(m_mediacodec_decoder_id); + m_mediacodec_decoder_id = 0; + } + m_mediacodec_mime = nullptr; + m_mediacodec_width = 0; + m_mediacodec_height = 0; +#endif + m_hardware_context.reset(); + m_decoder_context.reset(); + m_decoder.reset(); } bool DecodeApi::Initialize(Tegra::Host1x::NvdecCommon::VideoCodec codec) { - this->Reset(); - m_decoder.emplace(codec); - m_decoder_context.emplace(*m_decoder); - - // Enable GPU decoding if requested. - if (Settings::values.nvdec_emulation.GetValue() == Settings::NvdecEmulation::Gpu) { - m_hardware_context.emplace(); - m_hardware_context->InitializeForDecoder(*m_decoder_context, *m_decoder); - } - - // Open the decoder context. - if (!m_decoder_context->OpenContext(*m_decoder)) { - this->Reset(); - return false; - } - - return true; + this->Reset(); + m_decoder.emplace(codec); + m_decoder_context.emplace(*m_decoder); + + // Enable GPU decoding if requested. + if (Settings::values.nvdec_emulation.GetValue() == Settings::NvdecEmulation::Gpu) { +#ifdef __ANDROID__ + if (FFmpeg::MediaCodecBridge::IsAvailable()) { + // Register mime type for deferred MediaCodec creation. + switch (codec) { + case Tegra::Host1x::NvdecCommon::VideoCodec::H264: + m_mediacodec_mime = "video/avc"; + break; + case Tegra::Host1x::NvdecCommon::VideoCodec::VP8: + m_mediacodec_mime = "video/x-vnd.on2.vp8"; + break; + case Tegra::Host1x::NvdecCommon::VideoCodec::VP9: + m_mediacodec_mime = "video/x-vnd.on2.vp9"; + break; + default: + m_mediacodec_mime = nullptr; + break; + } + } +#endif + #ifdef __ANDROID__ + if (m_mediacodec_mime == nullptr) { + m_hardware_context.emplace(); + m_hardware_context->InitializeForDecoder(*m_decoder_context, *m_decoder); + } + #else + m_hardware_context.emplace(); + m_hardware_context->InitializeForDecoder(*m_decoder_context, *m_decoder); +#endif + } + + // Open the decoder context. + if (!m_decoder_context->OpenContext(*m_decoder)) { + this->Reset(); + return false; + } + + return true; +} + +#ifdef __ANDROID__ +void DecodeApi::EnsureMediaCodecDecoder(int width, int height) { + if (!m_mediacodec_mime || width <= 0 || height <= 0) { + return; + } + if (!FFmpeg::MediaCodecBridge::IsAvailable()) { + return; + } + if (m_mediacodec_decoder_id > 0 && width == m_mediacodec_width && height == m_mediacodec_height) { + return; + } + if (m_mediacodec_decoder_id != 0) { + FFmpeg::MediaCodecBridge::DestroyDecoder(m_mediacodec_decoder_id); + m_mediacodec_decoder_id = 0; + m_mediacodec_width = 0; + m_mediacodec_height = 0; + } + const int id = FFmpeg::MediaCodecBridge::CreateDecoder(m_mediacodec_mime, width, height); + if (id > 0) { + m_mediacodec_decoder_id = id; + m_mediacodec_width = width; + m_mediacodec_height = height; + LOG_INFO(HW_GPU, "MediaCodec bridge created decoder id={} ({}x{})", id, width, height); + } else { + LOG_DEBUG(HW_GPU, "MediaCodec bridge failed to create decoder for {} ({}x{})", m_mediacodec_mime, + width, height); + m_mediacodec_mime = nullptr; + m_mediacodec_width = 0; + m_mediacodec_height = 0; + } } +#endif bool DecodeApi::SendPacket(std::span packet_data) { - FFmpeg::Packet packet(packet_data); - return m_decoder_context->SendPacket(packet); +#ifdef __ANDROID__ + if (m_mediacodec_decoder_id > 0) { + if (FFmpeg::MediaCodecBridge::SendPacket(m_mediacodec_decoder_id, packet_data.data(), packet_data.size(), 0)) { + return true; + } + LOG_DEBUG(HW_GPU, "MediaCodec bridge failed to queue packet, falling back to FFmpeg"); + } +#endif + FFmpeg::Packet packet(packet_data); + return m_decoder_context->SendPacket(packet); } std::shared_ptr DecodeApi::ReceiveFrame() { - // Receive raw frame from decoder. - return m_decoder_context->ReceiveFrame(); +#ifdef __ANDROID__ + if (m_mediacodec_decoder_id > 0) { + int width = 0; + int height = 0; + int64_t pts = 0; + if (auto frame_data = FFmpeg::MediaCodecBridge::PopDecodedFrame(m_mediacodec_decoder_id, width, height, pts)) { + if (width > 0 && height > 0 && !frame_data->empty()) { + auto frame = std::make_shared(); + AVFrame* av_frame = frame->GetFrame(); + av_frame->format = AV_PIX_FMT_NV12; + av_frame->width = width; + av_frame->height = height; + av_frame->pts = pts; + if (const int ret = av_frame_get_buffer(av_frame, 32); ret < 0) { + LOG_ERROR(HW_GPU, "av_frame_get_buffer failed: {}", AVError(ret)); + } else { + const size_t y_stride = static_cast(width); + const size_t y_plane_size = y_stride * static_cast(height); + if (frame_data->size() < y_plane_size) { + LOG_WARNING(HW_GPU, "MediaCodec frame too small: {} < {}", frame_data->size(), y_plane_size); + } else { + const u8* src_y = frame_data->data(); + u8* dst_y = av_frame->data[0]; + for (int row = 0; row < height; ++row) { + std::memcpy(dst_y + static_cast(row) * av_frame->linesize[0], + src_y + static_cast(row) * y_stride, y_stride); + } + const int chroma_height = (height + 1) / 2; + const size_t chroma_plane_size = frame_data->size() - y_plane_size; + const size_t chroma_stride = chroma_height > 0 + ? chroma_plane_size / static_cast(chroma_height) + : 0; + if (chroma_height > 0 && chroma_stride * static_cast(chroma_height) != chroma_plane_size) { + LOG_WARNING(HW_GPU, "MediaCodec chroma plane misaligned: stride {} * height {} != {}", + chroma_stride, chroma_height, chroma_plane_size); + } + const u8* src_uv = frame_data->data() + y_plane_size; + u8* dst_uv = av_frame->data[1]; + const size_t copy_stride = std::min(chroma_stride, static_cast(av_frame->linesize[1])); + for (int row = 0; row < chroma_height; ++row) { + std::memcpy(dst_uv + static_cast(row) * av_frame->linesize[1], + src_uv + static_cast(row) * chroma_stride, copy_stride); + } + return frame; + } + } + } + } + } +#endif + // Receive raw frame from decoder. + return m_decoder_context->ReceiveFrame(); } } diff --git a/src/video_core/host1x/ffmpeg/ffmpeg.h b/src/video_core/host1x/ffmpeg/ffmpeg.h index d60a8ac4a7..2123eaf40a 100644 --- a/src/video_core/host1x/ffmpeg/ffmpeg.h +++ b/src/video_core/host1x/ffmpeg/ffmpeg.h @@ -216,10 +216,20 @@ public: bool SendPacket(std::span packet_data); std::shared_ptr ReceiveFrame(); +#ifdef __ANDROID__ + void EnsureMediaCodecDecoder(int width, int height); +#endif + private: std::optional m_decoder; std::optional m_decoder_context; std::optional m_hardware_context; +#ifdef __ANDROID__ + int m_mediacodec_decoder_id = 0; + const char* m_mediacodec_mime = nullptr; + int m_mediacodec_width = 0; + int m_mediacodec_height = 0; +#endif }; } // namespace FFmpeg diff --git a/src/video_core/host1x/ffmpeg/mediacodec_bridge.h b/src/video_core/host1x/ffmpeg/mediacodec_bridge.h new file mode 100644 index 0000000000..ff44628b5e --- /dev/null +++ b/src/video_core/host1x/ffmpeg/mediacodec_bridge.h @@ -0,0 +1,22 @@ +#pragma once + +#include +#include +#include + +namespace FFmpeg::MediaCodecBridge { + +bool IsAvailable(); + +// Create a platform decoder for the given mime type ("video/avc", "video/x-vnd.on2.vp9", ...) +// Returns decoder id (>0) on success, or 0 on failure. +int CreateDecoder(const char* mime, int width, int height); +void DestroyDecoder(int id); + +// Feed an encoded packet to the decoder. Returns true if accepted. +bool SendPacket(int id, const uint8_t* data, size_t size, int64_t pts); + +// Pop a decoded NV12 frame. Returns std::nullopt if none available. On success, fills width,height,pts +std::optional> PopDecodedFrame(int id, int& width, int& height, int64_t& pts); + +} // namespace FFmpeg::MediaCodecBridge diff --git a/src/video_core/host1x/ffmpeg/mediacodec_bridge_android.cpp b/src/video_core/host1x/ffmpeg/mediacodec_bridge_android.cpp new file mode 100644 index 0000000000..2451ad0b05 --- /dev/null +++ b/src/video_core/host1x/ffmpeg/mediacodec_bridge_android.cpp @@ -0,0 +1,114 @@ +// Android-specific JNI bridge implementation +#ifdef __ANDROID__ + +#include "mediacodec_bridge.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include "common/android/id_cache.h" +#include "common/logging/log.h" + +namespace FFmpeg::MediaCodecBridge { + +static jclass g_native_media_codec_class = nullptr; +static jmethodID g_create_decoder = nullptr; +static jmethodID g_release_decoder = nullptr; +static jmethodID g_decode_method = nullptr; + +struct DecoderState { + int id; + std::mutex mtx; + std::vector frame; + int width = 0; + int height = 0; + int64_t pts = 0; + bool has_frame = false; +}; + +static std::mutex s_global_mtx; +static std::map> s_decoders; + +extern "C" JNIEXPORT void JNICALL Java_org_yuzu_yuzu_1emu_media_NativeMediaCodec_onFrameDecoded( + JNIEnv* env, jclass, jint decoderId, jbyteArray data, jint width, jint height, jlong pts) { + std::lock_guard lock(s_global_mtx); + auto it = s_decoders.find(decoderId); + if (it == s_decoders.end()) return; + auto& st = it->second; + const jsize len = env->GetArrayLength(data); + st->frame.resize(len); + env->GetByteArrayRegion(data, 0, len, reinterpret_cast(st->frame.data())); + st->width = width; + st->height = height; + st->pts = pts; + st->has_frame = true; +} + +bool IsAvailable() { + // We assume the bridge is available if the Java class can be found. + auto env = Common::Android::GetEnvForThread(); + if (!env) return false; + if (!g_native_media_codec_class) { + jclass cls = env->FindClass("org/yuzu/yuzu_emu/media/NativeMediaCodec"); + if (!cls) return false; + g_native_media_codec_class = reinterpret_cast(env->NewGlobalRef(cls)); + g_create_decoder = env->GetStaticMethodID(g_native_media_codec_class, "createDecoder", "(Ljava/lang/String;II)I"); + g_release_decoder = env->GetStaticMethodID(g_native_media_codec_class, "releaseDecoder", "(I)V"); + g_decode_method = env->GetStaticMethodID(g_native_media_codec_class, "decode", "(I[BJ)Z"); + } + return g_native_media_codec_class != nullptr; +} + +int CreateDecoder(const char* mime, int width, int height) { + auto env = Common::Android::GetEnvForThread(); + if (!env) return 0; + jstring jmime = env->NewStringUTF(mime); + const int id = env->CallStaticIntMethod(g_native_media_codec_class, g_create_decoder, jmime, width, height); + env->DeleteLocalRef(jmime); + if (id <= 0) return 0; + std::lock_guard lock(s_global_mtx); + auto st = std::make_shared(); + st->id = id; + s_decoders[id] = st; + return id; +} + +void DestroyDecoder(int id) { + auto env = Common::Android::GetEnvForThread(); + if (!env) return; + env->CallStaticVoidMethod(g_native_media_codec_class, g_release_decoder, id); + std::lock_guard lock(s_global_mtx); + s_decoders.erase(id); +} + +bool SendPacket(int id, const uint8_t* data, size_t size, int64_t pts) { + auto env = Common::Android::GetEnvForThread(); + if (!env) return false; + std::lock_guard lock(s_global_mtx); + auto it = s_decoders.find(id); + if (it == s_decoders.end()) return false; + jbyteArray arr = env->NewByteArray(static_cast(size)); + env->SetByteArrayRegion(arr, 0, static_cast(size), reinterpret_cast(data)); + jboolean ok = env->CallStaticBooleanMethod(g_native_media_codec_class, g_decode_method, id, arr, static_cast(pts)); + env->DeleteLocalRef(arr); + return ok; +} + +std::optional> PopDecodedFrame(int id, int& width, int& height, int64_t& pts) { + std::lock_guard lock(s_global_mtx); + auto it = s_decoders.find(id); + if (it == s_decoders.end()) return std::nullopt; + auto& st = it->second; + if (!st->has_frame) return std::nullopt; + st->has_frame = false; + width = st->width; + height = st->height; + pts = st->pts; + return st->frame; +} + +#endif // __ANDROID__ diff --git a/src/video_core/vulkan_common/vulkan_device.cpp b/src/video_core/vulkan_common/vulkan_device.cpp index dcf7dd3749..85910f88dc 100644 --- a/src/video_core/vulkan_common/vulkan_device.cpp +++ b/src/video_core/vulkan_common/vulkan_device.cpp @@ -1090,26 +1090,6 @@ bool Device::GetSuitability(bool requires_swapchain) { loaded_extensions.insert("VK_KHR_maintenance4"); extensions.maintenance4 = true; } - if (supported_extensions.contains("VK_KHR_maintenance5")) { - loaded_extensions.insert("VK_KHR_maintenance5"); - extensions.maintenance5 = true; - } - if (supported_extensions.contains("VK_KHR_maintenance6")) { - loaded_extensions.insert("VK_KHR_maintenance6"); - extensions.maintenance6 = true; - } - if (supported_extensions.contains("VK_KHR_maintenance7")) { - loaded_extensions.insert("VK_KHR_maintenance7"); - extensions.maintenance7 = true; - } - if (supported_extensions.contains("VK_KHR_maintenance8")) { - loaded_extensions.insert("VK_KHR_maintenance8"); - extensions.maintenance8 = true; - } - if (supported_extensions.contains("VK_KHR_maintenance9")) { - loaded_extensions.insert("VK_KHR_maintenance9"); - extensions.maintenance9 = true; - } #undef FEATURE_EXTENSION #undef EXTENSION diff --git a/src/video_core/vulkan_common/vulkan_device.h b/src/video_core/vulkan_common/vulkan_device.h index 1f10d1799a..3ad83bf4ce 100644 --- a/src/video_core/vulkan_common/vulkan_device.h +++ b/src/video_core/vulkan_common/vulkan_device.h @@ -854,6 +854,12 @@ private: bool maintenance8{}; bool maintenance9{}; + // Maintenance extensions (may not be present in older Vulkan headers). + bool maintenance1{}; + bool maintenance2{}; + bool maintenance3{}; + bool maintenance4{}; + #undef EXTENSION #undef FEATURE };