[shader_recompiler] Add workaround for AMD shader bug in LM3

Fixes red, green and blue lines artifact on AMD GPUs in Luigi's Mansion 3.

This removes the old global legacy rescaling toggle.
Instead it automatically detects the specific fragment shaders causing the issue (due to F32/U32 bitcasts on position attributes).
The legacy rescaling workaround is now applied exclusively to these broken shaders, keeping the default scaling math intact for everything else.

src/common/settings.h

@@ -557,9 +557,6 @@ struct Values {
     SwitchableSetting<bool> fix_bloom_effects{linkage, false, "fix_bloom_effects",
                                                      Category::RendererHacks};
 
-    SwitchableSetting<bool> rescale_hack{linkage, false, "rescale_hack",
-                                                     Category::RendererHacks};
-
     SwitchableSetting<bool> use_asynchronous_shaders{linkage, false, "use_asynchronous_shaders",
                                                      Category::RendererHacks};
 

src/qt_common/config/shared_translation.cpp

@@ -356,12 +356,6 @@ std::unique_ptr<TranslationMap> InitializeTranslations(QObject* parent)
            tr("Fix bloom effects"),
            tr("Removes bloom in Burnout."));
 
-    INSERT(Settings,
-           rescale_hack,
-           tr("Enable Legacy Rescale Pass"),
-           tr("May fix rescale issues in some games by relying on behavior from the previous implementation.\n"
-              "Legacy behavior workaround that fixes line artifacts on AMD and Intel GPUs, and grey texture flicker on Nvidia GPUs in Luigis Mansion 3."));
-
     // Renderer (Extensions)
     INSERT(Settings, dyna_state, tr("Extended Dynamic State"),
            tr("Controls the number of features that can be used in Extended Dynamic State.\n"

src/shader_recompiler/frontend/maxwell/translate_program.cpp

@@ -304,7 +304,7 @@ IR::Program TranslateProgram(ObjectPool<IR::Inst>& inst_pool, ObjectPool<IR::Blo
     Optimization::GlobalMemoryToStorageBufferPass(program, host_info);
     Optimization::TexturePass(env, program, host_info);
 
-    if (Settings::values.resolution_info.active || Settings::values.rescale_hack.GetValue()) {
+    if (Settings::values.resolution_info.active || Optimization::FragmentShaderNeedsRescalingPass(program)) {
         Optimization::RescalingPass(program);
     }
     Optimization::DeadCodeEliminationPass(program);

src/shader_recompiler/ir_opt/passes.h

@@ -22,6 +22,7 @@ void LowerFp64ToFp32(IR::Program& program);
 void LowerFp16ToFp32(IR::Program& program);
 void LowerInt64ToInt32(IR::Program& program);
 void RescalingPass(IR::Program& program);
+bool FragmentShaderNeedsRescalingPass(const IR::Program& program);
 void SsaRewritePass(IR::Program& program);
 void PositionPass(Environment& env, IR::Program& program);
 void TexturePass(Environment& env, IR::Program& program, const HostTranslateInfo& host_info);

src/shader_recompiler/ir_opt/rescaling_pass.cpp

@@ -32,7 +32,7 @@ namespace {
     return false;
 }
 
-void VisitMark(IR::Block& block, IR::Inst& inst) {
+void VisitMark(IR::Block& block, IR::Inst& inst, bool needs_hack) {
     switch (inst.GetOpcode()) {
     case IR::Opcode::ShuffleIndex:
     case IR::Opcode::ShuffleUp:
@@ -54,20 +54,16 @@ void VisitMark(IR::Block& block, IR::Inst& inst) {
         bool must_patch_outside = false;
         if (bitcast_inst->GetOpcode() == IR::Opcode::GetAttribute) {
             const IR::Attribute attr{bitcast_inst->Arg(0).Attribute()};
-            switch (attr) {
-            case IR::Attribute::PositionX:
-            case IR::Attribute::PositionY:
+            if (attr >= IR::Attribute::PositionX && attr <= IR::Attribute::PositionW) {
                 bitcast_inst->SetFlags<u32>(0xDEADBEEF);
                 must_patch_outside = true;
-                break;
-            default:
-                break;
             }
         }
         if (must_patch_outside) {
             const auto it{IR::Block::InstructionList::s_iterator_to(inst)};
             IR::IREmitter ir{block, it};
-            if (Settings::values.rescale_hack.GetValue()) {
+
+            if (needs_hack) {
                 const IR::F32 new_inst{&*block.PrependNewInst(it, inst)};
                 const IR::F32 up_factor{ir.FPRecip(ir.ResolutionDownFactor())};
                 const IR::Value converted{ir.FPMul(new_inst, up_factor)};
@@ -347,12 +343,44 @@ void Visit(const IR::Program& program, IR::Block& block, IR::Inst& inst) {
 }
 } // Anonymous namespace
 
+bool FragmentShaderNeedsRescalingPass(const IR::Program& program) {
+    if (program.stage != Stage::Fragment) return false;
+
+    for (const IR::Block* block : program.post_order_blocks) {
+        for (const IR::Inst& inst : block->Instructions()) {
+            const auto op = inst.GetOpcode();
+            if (op != IR::Opcode::ShuffleIndex && op != IR::Opcode::ShuffleUp &&
+                op != IR::Opcode::ShuffleDown && op != IR::Opcode::ShuffleButterfly) {
+                continue;
+            }
+
+            if (inst.Arg(0).IsImmediate()) continue;
+            const IR::Inst* arg_inst = inst.Arg(0).InstRecursive();
+
+            if (arg_inst->GetOpcode() != IR::Opcode::BitCastU32F32 || arg_inst->Arg(0).IsImmediate()) continue;
+            const IR::Inst* bitcast_inst = arg_inst->Arg(0).InstRecursive();
+
+            if (bitcast_inst->GetOpcode() == IR::Opcode::GetAttribute) {
+                const auto attr = bitcast_inst->Arg(0).Attribute();
+                if (attr >= IR::Attribute::PositionX && attr <= IR::Attribute::PositionW) {
+                    return true;
+                }
+            }
+        }
+    }
+    return false;
+}
+
 void RescalingPass(IR::Program& program) {
     const bool is_fragment_shader{program.stage == Stage::Fragment};
+    const bool needs_hack{FragmentShaderNeedsRescalingPass(program)};
+    if (needs_hack) {
+        LOG_WARNING(Shader, "F32/U32 bitcast detected. Applying rescaling workaround.");
+    }
     if (is_fragment_shader) {
         for (IR::Block* const block : program.post_order_blocks) {
             for (IR::Inst& inst : block->Instructions()) {
-                VisitMark(*block, inst);
+                VisitMark(*block, inst, needs_hack);
             }
         }
     }