[host1x] Fix hardware detection and improve compatibility (#85)

This fixes green screen on unsupported devices trying to use GPU decoding and improve compatibility with default builds of FFmpeg on all platforms.

Co-authored-by: MaranBr <maranbr@outlook.com>
Reviewed-on: https://git.eden-emu.dev/eden-emu/eden/pulls/85
Co-authored-by: MaranBr <maranbr@eden-emu.dev>
Co-committed-by: MaranBr <maranbr@eden-emu.dev>

src/video_core/host1x/ffmpeg/ffmpeg.cpp

@@ -23,6 +23,8 @@ namespace FFmpeg {
 
 namespace {
 
+constexpr AVPixelFormat PreferredGpuFormat = AV_PIX_FMT_NV12;
+constexpr AVPixelFormat PreferredCpuFormat = AV_PIX_FMT_YUV420P;
 constexpr std::array PreferredGpuDecoders = {
 #if defined (_WIN32)
 	AV_HWDEVICE_TYPE_CUDA,
@@ -39,32 +41,31 @@ constexpr std::array PreferredGpuDecoders = {
 };
 
 AVPixelFormat GetGpuFormat(AVCodecContext* codec_context, const AVPixelFormat* pix_fmts) {
-	// Check if there is a pixel format supported by the GPU decoder.
 	const auto desc = av_pix_fmt_desc_get(codec_context->pix_fmt);
-	if (desc && desc->flags & AV_PIX_FMT_FLAG_HWACCEL) {
-		for (const AVPixelFormat* p = pix_fmts; *p != AV_PIX_FMT_NONE; ++p) {
-			if (*p == codec_context->pix_fmt) {
-				return codec_context->pix_fmt;
-			}
-		}
-	}
-
-	// Another check to confirm if there is a pixel format supported by specific GPU decoders.
+	if (desc && !(desc->flags & AV_PIX_FMT_FLAG_HWACCEL)) {
 		for (int i = 0;; i++) {
 			const AVCodecHWConfig* config = avcodec_get_hw_config(codec_context->codec, i);
 			if (!config) {
 				break;
 			}
 
-		if ((config->methods & AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX) && (config->device_type == AV_HWDEVICE_TYPE_CUDA || config->device_type == AV_HWDEVICE_TYPE_VAAPI)) {
-			return config->pix_fmt;
+			for (const auto type : PreferredGpuDecoders) {
+				if (config->methods & AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX && config->device_type == type) {
+					codec_context->pix_fmt = config->pix_fmt;
+				}
+			}
+		}
+	}
+
+	for (const AVPixelFormat* p = pix_fmts; *p != AV_PIX_FMT_NONE; ++p) {
+		if (*p == codec_context->pix_fmt) {
+			return codec_context->pix_fmt;
 		}
 	}
 
-	// Fallback to CPU decoder.
-    LOG_INFO(HW_GPU, "Could not find compatible GPU pixel format, falling back to CPU");
+	LOG_INFO(HW_GPU, "Could not find supported GPU pixel format, falling back to CPU decoder");
 	av_buffer_unref(&codec_context->hw_device_ctx);
-
+	codec_context->pix_fmt = PreferredCpuFormat;
 	return codec_context->pix_fmt;
 }
 
@@ -225,7 +226,7 @@ bool DecoderContext::OpenContext(const Decoder& decoder) {
 	}
 
 	if (!m_codec_context->hw_device_ctx) {
-        LOG_INFO(HW_GPU, "Using FFmpeg software decoding");
+		LOG_INFO(HW_GPU, "Using FFmpeg CPU decoder");
 	}
 
 	return true;
@@ -256,13 +257,12 @@ std::shared_ptr<Frame> DecoderContext::ReceiveFrame() {
 
 	m_temp_frame = std::make_shared<Frame>();
 	if (m_codec_context->hw_device_ctx) {
-		m_temp_frame->SetFormat(AV_PIX_FMT_NV12);
+		m_temp_frame->SetFormat(PreferredGpuFormat);
 		if (int ret = av_hwframe_transfer_data(m_temp_frame->GetFrame(), intermediate_frame->GetFrame(), 0); ret < 0) {
 			LOG_ERROR(HW_GPU, "av_hwframe_transfer_data error: {}", AVError(ret));
 			return {};
 		}
 	} else {
-		m_temp_frame->SetFormat(AV_PIX_FMT_YUV420P);
 		m_temp_frame = std::move(intermediate_frame);
 	}
 

src/video_core/host1x/ffmpeg/ffmpeg.h

@@ -23,15 +23,10 @@ extern "C" {
 #endif
 
 #include <libavcodec/avcodec.h>
+#include <libavcodec/codec.h>
 #include <libavutil/opt.h>
 #include <libavutil/pixdesc.h>
 
-#if defined(__FreeBSD__)
-#include <libavcodec/codec.h>
-#else
-#include <libavcodec/codec_internal.h>
-#endif
-
 #if defined(__GNUC__) || defined(__clang__)
 #pragma GCC diagnostic pop
 #endif