|
|
|
@ -27,16 +27,16 @@ void SharedCasFunction(EmitContext& ctx, IR::Inst& inst, std::string_view offset |
|
|
|
void SsboCasFunction(EmitContext& ctx, IR::Inst& inst, const IR::Value& binding, |
|
|
|
const IR::Value& offset, std::string_view value, std::string_view function) { |
|
|
|
const auto ret{ctx.var_alloc.Define(inst, GlslVarType::U32)}; |
|
|
|
const std::string ssbo{ |
|
|
|
fmt::format("{}_ssbo{}[{}]", ctx.stage_name, binding.U32(), offset.U32())}; |
|
|
|
const std::string ssbo{fmt::format("{}_ssbo{}[{}>>2]", ctx.stage_name, binding.U32(), |
|
|
|
ctx.var_alloc.Consume(offset))}; |
|
|
|
ctx.Add(cas_loop.data(), ssbo, ret, ssbo, function, ssbo, value, ret); |
|
|
|
} |
|
|
|
|
|
|
|
void SsboCasFunctionF32(EmitContext& ctx, IR::Inst& inst, const IR::Value& binding, |
|
|
|
const IR::Value& offset, std::string_view value, |
|
|
|
std::string_view function) { |
|
|
|
const std::string ssbo{ |
|
|
|
fmt::format("{}_ssbo{}[{}]", ctx.stage_name, binding.U32(), offset.U32())}; |
|
|
|
const std::string ssbo{fmt::format("{}_ssbo{}[{}>>2]", ctx.stage_name, binding.U32(), |
|
|
|
ctx.var_alloc.Consume(offset))}; |
|
|
|
const auto ret{ctx.var_alloc.Define(inst, GlslVarType::U32)}; |
|
|
|
ctx.Add(cas_loop.data(), ssbo, ret, ssbo, function, ssbo, value, ret); |
|
|
|
ctx.AddF32("{}=uintBitsToFloat({});", inst, ret); |
|
|
|
@ -111,8 +111,8 @@ void EmitSharedAtomicExchange64(EmitContext& ctx, IR::Inst& inst, std::string_vi |
|
|
|
|
|
|
|
void EmitStorageAtomicIAdd32(EmitContext& ctx, IR::Inst& inst, const IR::Value& binding, |
|
|
|
const IR::Value& offset, std::string_view value) { |
|
|
|
ctx.AddU32("{}=atomicAdd({}_ssbo{}[{}],{});", inst, ctx.stage_name, binding.U32(), offset.U32(), |
|
|
|
value); |
|
|
|
ctx.AddU32("{}=atomicAdd({}_ssbo{}[{}>>2],{});", inst, ctx.stage_name, binding.U32(), |
|
|
|
ctx.var_alloc.Consume(offset), value); |
|
|
|
} |
|
|
|
|
|
|
|
void EmitStorageAtomicSMin32(EmitContext& ctx, IR::Inst& inst, const IR::Value& binding, |
|
|
|
@ -123,8 +123,8 @@ void EmitStorageAtomicSMin32(EmitContext& ctx, IR::Inst& inst, const IR::Value& |
|
|
|
|
|
|
|
void EmitStorageAtomicUMin32(EmitContext& ctx, IR::Inst& inst, const IR::Value& binding, |
|
|
|
const IR::Value& offset, std::string_view value) { |
|
|
|
ctx.AddU32("{}=atomicMin({}_ssbo{}[{}],{});", inst, ctx.stage_name, binding.U32(), offset.U32(), |
|
|
|
value); |
|
|
|
ctx.AddU32("{}=atomicMin({}_ssbo{}[{}>>2],{});", inst, ctx.stage_name, binding.U32(), |
|
|
|
ctx.var_alloc.Consume(offset), value); |
|
|
|
} |
|
|
|
|
|
|
|
void EmitStorageAtomicSMax32(EmitContext& ctx, IR::Inst& inst, const IR::Value& binding, |
|
|
|
@ -135,8 +135,8 @@ void EmitStorageAtomicSMax32(EmitContext& ctx, IR::Inst& inst, const IR::Value& |
|
|
|
|
|
|
|
void EmitStorageAtomicUMax32(EmitContext& ctx, IR::Inst& inst, const IR::Value& binding, |
|
|
|
const IR::Value& offset, std::string_view value) { |
|
|
|
ctx.AddU32("{}=atomicMax({}_ssbo{}[{}],{});", inst, ctx.stage_name, binding.U32(), offset.U32(), |
|
|
|
value); |
|
|
|
ctx.AddU32("{}=atomicMax({}_ssbo{}[{}>>2],{});", inst, ctx.stage_name, binding.U32(), |
|
|
|
ctx.var_alloc.Consume(offset), value); |
|
|
|
} |
|
|
|
|
|
|
|
void EmitStorageAtomicInc32(EmitContext& ctx, IR::Inst& inst, const IR::Value& binding, |
|
|
|
@ -151,116 +151,123 @@ void EmitStorageAtomicDec32(EmitContext& ctx, IR::Inst& inst, const IR::Value& b |
|
|
|
|
|
|
|
void EmitStorageAtomicAnd32(EmitContext& ctx, IR::Inst& inst, const IR::Value& binding, |
|
|
|
const IR::Value& offset, std::string_view value) { |
|
|
|
ctx.AddU32("{}=atomicAnd({}_ssbo{}[{}],{});", inst, ctx.stage_name, binding.U32(), offset.U32(), |
|
|
|
value); |
|
|
|
ctx.AddU32("{}=atomicAnd({}_ssbo{}[{}>>2],{});", inst, ctx.stage_name, binding.U32(), |
|
|
|
ctx.var_alloc.Consume(offset), value); |
|
|
|
} |
|
|
|
|
|
|
|
void EmitStorageAtomicOr32(EmitContext& ctx, IR::Inst& inst, const IR::Value& binding, |
|
|
|
const IR::Value& offset, std::string_view value) { |
|
|
|
ctx.AddU32("{}=atomicOr({}_ssbo{}[{}],{});", inst, ctx.stage_name, binding.U32(), offset.U32(), |
|
|
|
value); |
|
|
|
ctx.AddU32("{}=atomicOr({}_ssbo{}[{}>>2],{});", inst, ctx.stage_name, binding.U32(), |
|
|
|
ctx.var_alloc.Consume(offset), value); |
|
|
|
} |
|
|
|
|
|
|
|
void EmitStorageAtomicXor32(EmitContext& ctx, IR::Inst& inst, const IR::Value& binding, |
|
|
|
const IR::Value& offset, std::string_view value) { |
|
|
|
ctx.AddU32("{}=atomicXor({}_ssbo{}[{}],{});", inst, ctx.stage_name, binding.U32(), offset.U32(), |
|
|
|
value); |
|
|
|
ctx.AddU32("{}=atomicXor({}_ssbo{}[{}>>2],{});", inst, ctx.stage_name, binding.U32(), |
|
|
|
ctx.var_alloc.Consume(offset), value); |
|
|
|
} |
|
|
|
|
|
|
|
void EmitStorageAtomicExchange32(EmitContext& ctx, IR::Inst& inst, const IR::Value& binding, |
|
|
|
const IR::Value& offset, std::string_view value) { |
|
|
|
ctx.AddU32("{}=atomicExchange({}_ssbo{}[{}],{});", inst, ctx.stage_name, binding.U32(), |
|
|
|
offset.U32(), value); |
|
|
|
ctx.AddU32("{}=atomicExchange({}_ssbo{}[{}>>2],{});", inst, ctx.stage_name, binding.U32(), |
|
|
|
ctx.var_alloc.Consume(offset), value); |
|
|
|
} |
|
|
|
|
|
|
|
void EmitStorageAtomicIAdd64(EmitContext& ctx, IR::Inst& inst, const IR::Value& binding, |
|
|
|
const IR::Value& offset, std::string_view value) { |
|
|
|
// LOG_WARNING(..., "Op falling to non-atomic");
|
|
|
|
ctx.AddU64("{}=packUint2x32(uvec2({}_ssbo{}[{}],{}_ssbo{}[{}]));", inst, ctx.stage_name, |
|
|
|
binding.U32(), offset.U32(), ctx.stage_name, binding.U32(), offset.U32() + 1); |
|
|
|
ctx.Add("{}_ssbo{}[{}]+=unpackUint2x32({}).x;{}_ssbo{}[{}]+=unpackUint2x32({}).y;", |
|
|
|
ctx.stage_name, binding.U32(), offset.U32(), value, ctx.stage_name, binding.U32(), |
|
|
|
offset.U32() + 1, value); |
|
|
|
ctx.AddU64("{}=packUint2x32(uvec2({}_ssbo{}[{}>>2],{}_ssbo{}[({}>>2)+1]));", inst, |
|
|
|
ctx.stage_name, binding.U32(), ctx.var_alloc.Consume(offset), ctx.stage_name, |
|
|
|
binding.U32(), ctx.var_alloc.Consume(offset)); |
|
|
|
ctx.Add("{}_ssbo{}[{}>>2]+=unpackUint2x32({}).x;{}_ssbo{}[({}>>2)+1]+=unpackUint2x32({}).y;", |
|
|
|
ctx.stage_name, binding.U32(), ctx.var_alloc.Consume(offset), value, ctx.stage_name, |
|
|
|
binding.U32(), ctx.var_alloc.Consume(offset), value); |
|
|
|
} |
|
|
|
|
|
|
|
void EmitStorageAtomicSMin64(EmitContext& ctx, IR::Inst& inst, const IR::Value& binding, |
|
|
|
const IR::Value& offset, std::string_view value) { |
|
|
|
// LOG_WARNING(..., "Op falling to non-atomic");
|
|
|
|
ctx.AddS64("{}=packInt2x32(ivec2({}_ssbo{}[{}],{}_ssbo{}[{}]));", inst, ctx.stage_name, |
|
|
|
binding.U32(), offset.U32(), ctx.stage_name, binding.U32(), offset.U32() + 1); |
|
|
|
ctx.AddS64("{}=packInt2x32(ivec2({}_ssbo{}[{}>>2],{}_ssbo{}[({}>>2)+1]));", inst, |
|
|
|
ctx.stage_name, binding.U32(), ctx.var_alloc.Consume(offset), ctx.stage_name, |
|
|
|
binding.U32(), ctx.var_alloc.Consume(offset)); |
|
|
|
ctx.Add("for(int i=0;i<2;++i){{ " |
|
|
|
"{}_ssbo{}[{}+i]=uint(min(int({}_ssbo{}[{}+i]),unpackInt2x32(int64_t({}))[i]));}}", |
|
|
|
ctx.stage_name, binding.U32(), offset.U32(), ctx.stage_name, binding.U32(), |
|
|
|
offset.U32(), value); |
|
|
|
"{}_ssbo{}[({}>>2)+i]=uint(min(int({}_ssbo{}[({}>>2)+i]),unpackInt2x32(int64_t({}))[i])" |
|
|
|
");}}", |
|
|
|
ctx.stage_name, binding.U32(), ctx.var_alloc.Consume(offset), ctx.stage_name, |
|
|
|
binding.U32(), ctx.var_alloc.Consume(offset), value); |
|
|
|
} |
|
|
|
|
|
|
|
void EmitStorageAtomicUMin64(EmitContext& ctx, IR::Inst& inst, const IR::Value& binding, |
|
|
|
const IR::Value& offset, std::string_view value) { |
|
|
|
// LOG_WARNING(..., "Op falling to non-atomic");
|
|
|
|
ctx.AddU64("{}=packUint2x32(uvec2({}_ssbo{}[{}],{}_ssbo{}[{}]));", inst, ctx.stage_name, |
|
|
|
binding.U32(), offset.U32(), ctx.stage_name, binding.U32(), offset.U32() + 1); |
|
|
|
ctx.AddU64("{}=packUint2x32(uvec2({}_ssbo{}[{}>>2],{}_ssbo{}[({}>>2)+1]));", inst, |
|
|
|
ctx.stage_name, binding.U32(), ctx.var_alloc.Consume(offset), ctx.stage_name, |
|
|
|
binding.U32(), ctx.var_alloc.Consume(offset)); |
|
|
|
ctx.Add("for(int i=0;i<2;++i){{ " |
|
|
|
"{}_ssbo{}[{}+i]=min({}_ssbo{}[{}+i],unpackUint2x32(uint64_t({}))[i]);}}", |
|
|
|
ctx.stage_name, binding.U32(), offset.U32(), ctx.stage_name, binding.U32(), |
|
|
|
offset.U32(), value); |
|
|
|
"{}_ssbo{}[({}>>2)+i]=min({}_ssbo{}[({}>>2)+i],unpackUint2x32(uint64_t({}))[i]);}}", |
|
|
|
ctx.stage_name, binding.U32(), ctx.var_alloc.Consume(offset), ctx.stage_name, |
|
|
|
binding.U32(), ctx.var_alloc.Consume(offset), value); |
|
|
|
} |
|
|
|
|
|
|
|
void EmitStorageAtomicSMax64(EmitContext& ctx, IR::Inst& inst, const IR::Value& binding, |
|
|
|
const IR::Value& offset, std::string_view value) { |
|
|
|
// LOG_WARNING(..., "Op falling to non-atomic");
|
|
|
|
ctx.AddS64("{}=packInt2x32(ivec2({}_ssbo{}[{}],{}_ssbo{}[{}]));", inst, ctx.stage_name, |
|
|
|
binding.U32(), offset.U32(), ctx.stage_name, binding.U32(), offset.U32() + 1); |
|
|
|
ctx.AddS64("{}=packInt2x32(ivec2({}_ssbo{}[{}>>2],{}_ssbo{}[({}>>2)+1]));", inst, |
|
|
|
ctx.stage_name, binding.U32(), ctx.var_alloc.Consume(offset), ctx.stage_name, |
|
|
|
binding.U32(), ctx.var_alloc.Consume(offset)); |
|
|
|
ctx.Add("for(int i=0;i<2;++i){{ " |
|
|
|
"{}_ssbo{}[{}+i]=uint(max(int({}_ssbo{}[{}+i]),unpackInt2x32(int64_t({}))[i]));}}", |
|
|
|
ctx.stage_name, binding.U32(), offset.U32(), ctx.stage_name, binding.U32(), |
|
|
|
offset.U32(), value); |
|
|
|
"{}_ssbo{}[({}>>2)+i]=uint(max(int({}_ssbo{}[({}>>2)+i]),unpackInt2x32(int64_t({}))[i])" |
|
|
|
");}}", |
|
|
|
ctx.stage_name, binding.U32(), ctx.var_alloc.Consume(offset), ctx.stage_name, |
|
|
|
binding.U32(), ctx.var_alloc.Consume(offset), value); |
|
|
|
} |
|
|
|
|
|
|
|
void EmitStorageAtomicUMax64(EmitContext& ctx, IR::Inst& inst, const IR::Value& binding, |
|
|
|
const IR::Value& offset, std::string_view value) { |
|
|
|
// LOG_WARNING(..., "Op falling to non-atomic");
|
|
|
|
ctx.AddU64("{}=packUint2x32(uvec2({}_ssbo{}[{}],{}_ssbo{}[{}]));", inst, ctx.stage_name, |
|
|
|
binding.U32(), offset.U32(), ctx.stage_name, binding.U32(), offset.U32() + 1); |
|
|
|
|
|
|
|
ctx.AddU64("{}=packUint2x32(uvec2({}_ssbo{}[{}>>2],{}_ssbo{}[({}>>2)+1]));", inst, |
|
|
|
ctx.stage_name, binding.U32(), ctx.var_alloc.Consume(offset), ctx.stage_name, |
|
|
|
binding.U32(), ctx.var_alloc.Consume(offset)); |
|
|
|
ctx.Add("for(int " |
|
|
|
"i=0;i<2;++i){{{}_ssbo{}[{}+i]=max({}_ssbo{}[{}+i],unpackUint2x32(uint64_t({}))[i]);}}", |
|
|
|
ctx.stage_name, binding.U32(), offset.U32(), ctx.stage_name, binding.U32(), |
|
|
|
offset.U32(), value); |
|
|
|
"i=0;i<2;++i){{{}_ssbo{}[({}>>2)+i]=max({}_ssbo{}[({}>>2)+i],unpackUint2x32(uint64_t({}" |
|
|
|
"))[i]);}}", |
|
|
|
ctx.stage_name, binding.U32(), ctx.var_alloc.Consume(offset), ctx.stage_name, |
|
|
|
binding.U32(), ctx.var_alloc.Consume(offset), value); |
|
|
|
} |
|
|
|
|
|
|
|
void EmitStorageAtomicAnd64(EmitContext& ctx, IR::Inst& inst, const IR::Value& binding, |
|
|
|
const IR::Value& offset, std::string_view value) { |
|
|
|
ctx.AddU64("{}=packUint2x32(uvec2(atomicAnd({}_ssbo{}[{}],unpackUint2x32({}).x),atomicAnd({}_" |
|
|
|
"ssbo{}[{}]," |
|
|
|
"unpackUint2x32({}).y)));", |
|
|
|
inst, ctx.stage_name, binding.U32(), offset.U32(), value, ctx.stage_name, |
|
|
|
binding.U32(), offset.U32() + 1, value); |
|
|
|
ctx.AddU64( |
|
|
|
"{}=packUint2x32(uvec2(atomicAnd({}_ssbo{}[{}>>2],unpackUint2x32({}).x),atomicAnd({}_" |
|
|
|
"ssbo{}[({}>>2)+1],unpackUint2x32({}).y)));", |
|
|
|
inst, ctx.stage_name, binding.U32(), ctx.var_alloc.Consume(offset), value, ctx.stage_name, |
|
|
|
binding.U32(), ctx.var_alloc.Consume(offset), value); |
|
|
|
} |
|
|
|
|
|
|
|
void EmitStorageAtomicOr64(EmitContext& ctx, IR::Inst& inst, const IR::Value& binding, |
|
|
|
const IR::Value& offset, std::string_view value) { |
|
|
|
ctx.AddU64( |
|
|
|
"{}=packUint2x32(uvec2(atomicOr({}_ssbo{}[{}],unpackUint2x32({}).x),atomicOr({}_ssbo{}[{}]," |
|
|
|
"unpackUint2x32({}).y)));", |
|
|
|
inst, ctx.stage_name, binding.U32(), offset.U32(), value, ctx.stage_name, binding.U32(), |
|
|
|
offset.U32() + 1, value); |
|
|
|
ctx.AddU64("{}=packUint2x32(uvec2(atomicOr({}_ssbo{}[{}>>2],unpackUint2x32({}).x),atomicOr({}_" |
|
|
|
"ssbo{}[({}>>2)+1],unpackUint2x32({}).y)));", |
|
|
|
inst, ctx.stage_name, binding.U32(), ctx.var_alloc.Consume(offset), value, |
|
|
|
ctx.stage_name, binding.U32(), ctx.var_alloc.Consume(offset), value); |
|
|
|
} |
|
|
|
|
|
|
|
void EmitStorageAtomicXor64(EmitContext& ctx, IR::Inst& inst, const IR::Value& binding, |
|
|
|
const IR::Value& offset, std::string_view value) { |
|
|
|
ctx.AddU64("{}=packUint2x32(uvec2(atomicXor({}_ssbo{}[{}],unpackUint2x32({}).x),atomicXor({}_" |
|
|
|
"ssbo{}[{}]," |
|
|
|
"unpackUint2x32({}).y)));", |
|
|
|
inst, ctx.stage_name, binding.U32(), offset.U32(), value, ctx.stage_name, |
|
|
|
binding.U32(), offset.U32() + 1, value); |
|
|
|
ctx.AddU64( |
|
|
|
"{}=packUint2x32(uvec2(atomicXor({}_ssbo{}[{}>>2],unpackUint2x32({}).x),atomicXor({}_" |
|
|
|
"ssbo{}[({}>>2)+1],unpackUint2x32({}).y)));", |
|
|
|
inst, ctx.stage_name, binding.U32(), ctx.var_alloc.Consume(offset), value, ctx.stage_name, |
|
|
|
binding.U32(), ctx.var_alloc.Consume(offset), value); |
|
|
|
} |
|
|
|
|
|
|
|
void EmitStorageAtomicExchange64(EmitContext& ctx, IR::Inst& inst, const IR::Value& binding, |
|
|
|
const IR::Value& offset, std::string_view value) { |
|
|
|
ctx.AddU64( |
|
|
|
"{}=packUint2x32(uvec2(atomicExchange({}_ssbo{}[{}],unpackUint2x32({}).x),atomicExchange(" |
|
|
|
"{}_ssbo{}[{}],unpackUint2x32({}).y)));", |
|
|
|
inst, ctx.stage_name, binding.U32(), offset.U32(), value, ctx.stage_name, binding.U32(), |
|
|
|
offset.U32() + 1, value); |
|
|
|
ctx.AddU64("{}=packUint2x32(uvec2(atomicExchange({}_ssbo{}[{}>>2],unpackUint2x32({}).x)," |
|
|
|
"atomicExchange({}_ssbo{}[({}>>2)+1],unpackUint2x32({}).y)));", |
|
|
|
inst, ctx.stage_name, binding.U32(), ctx.var_alloc.Consume(offset), value, |
|
|
|
ctx.stage_name, binding.U32(), ctx.var_alloc.Consume(offset), value); |
|
|
|
} |
|
|
|
|
|
|
|
void EmitStorageAtomicAddF32(EmitContext& ctx, IR::Inst& inst, const IR::Value& binding, |
|
|
|
|