gl_rasterizer: Workaround invalid zeta clears

Some games (like Xenoblade Chronicles 2) clear both depth and stencil
buffers while there's a depth-only texture attached (e.g. D16 Unorm).
This commit reads the zeta format of the bound surface on
ConfigureFramebuffers and returns if depth and/or stencil attachments
were set. This is ignored on DrawArrays but on Clear it's used to just
clear those attachments, bypassing an OpenGL error.

src/video_core/renderer_opengl/gl_rasterizer.cpp

@@ -486,9 +486,9 @@ void RasterizerOpenGL::UpdatePagesCachedCount(VAddr addr, u64 size, int delta) {
         cached_pages.add({pages_interval, delta});
 }
 
-void RasterizerOpenGL::ConfigureFramebuffers(OpenGLState& current_state, bool using_color_fb,
-                                             bool using_depth_fb, bool preserve_contents,
-                                             std::optional<std::size_t> single_color_target) {
+std::pair<bool, bool> RasterizerOpenGL::ConfigureFramebuffers(
+    OpenGLState& current_state, bool using_color_fb, bool using_depth_fb, bool preserve_contents,
+    std::optional<std::size_t> single_color_target) {
     MICROPROFILE_SCOPE(OpenGL_Framebuffer);
     const auto& gpu = Core::System::GetInstance().GPU().Maxwell3D();
     const auto& regs = gpu.regs;
@@ -500,7 +500,7 @@ void RasterizerOpenGL::ConfigureFramebuffers(OpenGLState& current_state, bool us
         // Only skip if the previous ConfigureFramebuffers call was from the same kind (multiple or
         // single color targets). This is done because the guest registers may not change but the
         // host framebuffer may contain different attachments
-        return;
+        return current_depth_stencil_usage;
     }
     current_framebuffer_config_state = fb_config_state;
 
@@ -570,12 +570,14 @@ void RasterizerOpenGL::ConfigureFramebuffers(OpenGLState& current_state, bool us
         depth_surface->MarkAsModified(true, res_cache);
 
         fbkey.zeta = depth_surface->Texture().handle;
-        fbkey.stencil_enable = regs.stencil_enable;
+        fbkey.stencil_enable = regs.stencil_enable &&
+                               depth_surface->GetSurfaceParams().type == SurfaceType::DepthStencil;
     }
 
     SetupCachedFramebuffer(fbkey, current_state);
-
     SyncViewport(current_state);
+
+    return current_depth_stencil_usage = {static_cast<bool>(depth_surface), fbkey.stencil_enable};
 }
 
 void RasterizerOpenGL::Clear() {
@@ -643,8 +645,8 @@ void RasterizerOpenGL::Clear() {
         return;
     }
 
-    ConfigureFramebuffers(clear_state, use_color, use_depth || use_stencil, false,
-                          regs.clear_buffers.RT.Value());
+    const auto [clear_depth, clear_stencil] = ConfigureFramebuffers(
+        clear_state, use_color, use_depth || use_stencil, false, regs.clear_buffers.RT.Value());
     if (regs.clear_flags.scissor) {
         SyncScissorTest(clear_state);
     }
@@ -659,11 +661,11 @@ void RasterizerOpenGL::Clear() {
         glClearBufferfv(GL_COLOR, regs.clear_buffers.RT, regs.clear_color);
     }
 
-    if (use_depth && use_stencil) {
+    if (clear_depth && clear_stencil) {
         glClearBufferfi(GL_DEPTH_STENCIL, 0, regs.clear_depth, regs.clear_stencil);
-    } else if (use_depth) {
+    } else if (clear_depth) {
         glClearBufferfv(GL_DEPTH, 0, &regs.clear_depth);
-    } else if (use_stencil) {
+    } else if (clear_stencil) {
         glClearBufferiv(GL_STENCIL, 0, &regs.clear_stencil);
     }
 }

src/video_core/renderer_opengl/gl_rasterizer.h

@@ -122,10 +122,12 @@ private:
      * @param using_depth_fb If true, configure the depth/stencil framebuffer.
      * @param preserve_contents If true, tries to preserve data from a previously used framebuffer.
      * @param single_color_target Specifies if a single color buffer target should be used.
+     * @returns If depth (first) or stencil (second) are being stored in the bound zeta texture
+     * (requires using_depth_fb to be true)
      */
-    void ConfigureFramebuffers(OpenGLState& current_state, bool use_color_fb = true,
-                               bool using_depth_fb = true, bool preserve_contents = true,
-                               std::optional<std::size_t> single_color_target = {});
+    std::pair<bool, bool> ConfigureFramebuffers(
+        OpenGLState& current_state, bool use_color_fb = true, bool using_depth_fb = true,
+        bool preserve_contents = true, std::optional<std::size_t> single_color_target = {});
 
     /**
      * Configures the current constbuffers to use for the draw command.
@@ -221,6 +223,7 @@ private:
 
     std::map<FramebufferCacheKey, OGLFramebuffer> framebuffer_cache;
     FramebufferConfigState current_framebuffer_config_state;
+    std::pair<bool, bool> current_depth_stencil_usage{};
 
     std::array<SamplerInfo, Tegra::Engines::Maxwell3D::Regs::NumTextureSamplers> texture_samplers;