|
|
@ -16,15 +16,10 @@ using Vector = std::array<u64, 2>; |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
std::optional<AxxEmitX64::DoNotFastmemMarker> AxxEmitX64::ShouldFastmem(AxxEmitContext& ctx, IR::Inst* inst) const { |
|
|
std::optional<AxxEmitX64::DoNotFastmemMarker> AxxEmitX64::ShouldFastmem(AxxEmitContext& ctx, IR::Inst* inst) const { |
|
|
if (!conf.fastmem_pointer || !exception_handler.SupportsFastmem()) { |
|
|
|
|
|
|
|
|
if (!conf.fastmem_pointer || !exception_handler.SupportsFastmem()) |
|
|
return std::nullopt; |
|
|
return std::nullopt; |
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
const auto marker = std::make_tuple(ctx.Location(), inst->GetName()); |
|
|
const auto marker = std::make_tuple(ctx.Location(), inst->GetName()); |
|
|
if (do_not_fastmem.count(marker) > 0) { |
|
|
|
|
|
return std::nullopt; |
|
|
|
|
|
} |
|
|
|
|
|
return marker; |
|
|
|
|
|
|
|
|
return do_not_fastmem.count(marker) <= 0 ? marker : std::nullopt; |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
FakeCall AxxEmitX64::FastmemCallback(u64 rip_) { |
|
|
FakeCall AxxEmitX64::FastmemCallback(u64 rip_) { |
|
|
@ -59,16 +54,12 @@ void AxxEmitX64::EmitMemoryRead(AxxEmitContext& ctx, IR::Inst* inst) { |
|
|
// Neither fastmem nor page table: Use callbacks |
|
|
// Neither fastmem nor page table: Use callbacks |
|
|
if constexpr (bitsize == 128) { |
|
|
if constexpr (bitsize == 128) { |
|
|
ctx.reg_alloc.HostCall(nullptr, {}, args[1]); |
|
|
ctx.reg_alloc.HostCall(nullptr, {}, args[1]); |
|
|
if (ordered) { |
|
|
|
|
|
code.mfence(); |
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
if (ordered) code.mfence(); |
|
|
code.CallFunction(memory_read_128); |
|
|
code.CallFunction(memory_read_128); |
|
|
ctx.reg_alloc.DefineValue(inst, xmm1); |
|
|
ctx.reg_alloc.DefineValue(inst, xmm1); |
|
|
} else { |
|
|
} else { |
|
|
ctx.reg_alloc.HostCall(inst, {}, args[1]); |
|
|
ctx.reg_alloc.HostCall(inst, {}, args[1]); |
|
|
if (ordered) { |
|
|
|
|
|
code.mfence(); |
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
if (ordered) code.mfence(); |
|
|
Devirtualize<callback>(conf.callbacks).EmitCall(code); |
|
|
Devirtualize<callback>(conf.callbacks).EmitCall(code); |
|
|
code.ZeroExtendFrom(bitsize, code.ABI_RETURN); |
|
|
code.ZeroExtendFrom(bitsize, code.ABI_RETURN); |
|
|
} |
|
|
} |
|
|
@ -154,9 +145,7 @@ void AxxEmitX64::EmitMemoryWrite(AxxEmitContext& ctx, IR::Inst* inst) { |
|
|
ctx.reg_alloc.HostCall(nullptr, {}, args[1], args[2]); |
|
|
ctx.reg_alloc.HostCall(nullptr, {}, args[1], args[2]); |
|
|
Devirtualize<callback>(conf.callbacks).EmitCall(code); |
|
|
Devirtualize<callback>(conf.callbacks).EmitCall(code); |
|
|
} |
|
|
} |
|
|
if (ordered) { |
|
|
|
|
|
code.mfence(); |
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
if (ordered) code.mfence(); |
|
|
EmitCheckMemoryAbort(ctx, inst); |
|
|
EmitCheckMemoryAbort(ctx, inst); |
|
|
return; |
|
|
return; |
|
|
} |
|
|
} |
|
|
@ -298,9 +287,7 @@ void AxxEmitX64::EmitExclusiveWriteMemory(AxxEmitContext& ctx, IR::Inst* inst) { |
|
|
return conf.global_monitor->DoExclusiveOperation<T>(conf.processor_id, vaddr, |
|
|
return conf.global_monitor->DoExclusiveOperation<T>(conf.processor_id, vaddr, |
|
|
[&](T expected) -> bool { |
|
|
[&](T expected) -> bool { |
|
|
return (conf.callbacks->*callback)(vaddr, value, expected); |
|
|
return (conf.callbacks->*callback)(vaddr, value, expected); |
|
|
}) |
|
|
|
|
|
? 0 |
|
|
|
|
|
: 1; |
|
|
|
|
|
|
|
|
}) ? 0 : 1; |
|
|
}); |
|
|
}); |
|
|
if (ordered) { |
|
|
if (ordered) { |
|
|
code.mfence(); |
|
|
code.mfence(); |
|
|
@ -314,9 +301,7 @@ void AxxEmitX64::EmitExclusiveWriteMemory(AxxEmitContext& ctx, IR::Inst* inst) { |
|
|
return conf.global_monitor->DoExclusiveOperation<Vector>(conf.processor_id, vaddr, |
|
|
return conf.global_monitor->DoExclusiveOperation<Vector>(conf.processor_id, vaddr, |
|
|
[&](Vector expected) -> bool { |
|
|
[&](Vector expected) -> bool { |
|
|
return (conf.callbacks->*callback)(vaddr, value, expected); |
|
|
return (conf.callbacks->*callback)(vaddr, value, expected); |
|
|
}) |
|
|
|
|
|
? 0 |
|
|
|
|
|
: 1; |
|
|
|
|
|
|
|
|
}) ? 0 : 1; |
|
|
}); |
|
|
}); |
|
|
if (ordered) { |
|
|
if (ordered) { |
|
|
code.mfence(); |
|
|
code.mfence(); |
|
|
@ -476,25 +461,20 @@ void AxxEmitX64::EmitExclusiveWriteMemoryInline(AxxEmitContext& ctx, IR::Inst* i |
|
|
const auto location = code.getCurr(); |
|
|
const auto location = code.getCurr(); |
|
|
|
|
|
|
|
|
if constexpr (bitsize == 128) { |
|
|
if constexpr (bitsize == 128) { |
|
|
code.lock(); |
|
|
|
|
|
code.cmpxchg16b(ptr[dest_ptr]); |
|
|
|
|
|
|
|
|
code.lock(); code.cmpxchg16b(ptr[dest_ptr]); |
|
|
} else { |
|
|
} else { |
|
|
switch (bitsize) { |
|
|
switch (bitsize) { |
|
|
case 8: |
|
|
case 8: |
|
|
code.lock(); |
|
|
|
|
|
code.cmpxchg(code.byte[dest_ptr], value.cvt8()); |
|
|
|
|
|
|
|
|
code.lock(); code.cmpxchg(code.byte[dest_ptr], value.cvt8()); |
|
|
break; |
|
|
break; |
|
|
case 16: |
|
|
case 16: |
|
|
code.lock(); |
|
|
|
|
|
code.cmpxchg(word[dest_ptr], value.cvt16()); |
|
|
|
|
|
|
|
|
code.lock(); code.cmpxchg(word[dest_ptr], value.cvt16()); |
|
|
break; |
|
|
break; |
|
|
case 32: |
|
|
case 32: |
|
|
code.lock(); |
|
|
|
|
|
code.cmpxchg(dword[dest_ptr], value.cvt32()); |
|
|
|
|
|
|
|
|
code.lock(); code.cmpxchg(dword[dest_ptr], value.cvt32()); |
|
|
break; |
|
|
break; |
|
|
case 64: |
|
|
case 64: |
|
|
code.lock(); |
|
|
|
|
|
code.cmpxchg(qword[dest_ptr], value.cvt64()); |
|
|
|
|
|
|
|
|
code.lock(); code.cmpxchg(qword[dest_ptr], value.cvt64()); |
|
|
break; |
|
|
break; |
|
|
default: |
|
|
default: |
|
|
UNREACHABLE(); |
|
|
UNREACHABLE(); |
|
|
|