Add -hack-atomic-flag-barrier option (#1652)
Under the SPIR-V and Vulkan memory model specifications, OpAtomic*
instructions with acquire/release semantics already establish
happens-before ordering for regular memory accesses across threads.
However, some drivers (e.g. under the GLSL450 memory model) fail to
properly enforce this ordering for atomic_flag operations without
explicit memory barriers.
To work around such driver issues, add a new compiler option
-hack-atomic-flag-barrier:
- In replaceAtomicFlagClear: emit an OpMemoryBarrier with Release
semantics immediately prior to OpAtomicStore when release ordering
is requested (memory_order_release, memory_order_seq_cst, or default).
- In replaceAtomicFlagTestAndSet:
- Emit an OpMemoryBarrier with Release semantics immediately prior
to OpAtomicExchange when release ordering is requested
(memory_order_release, memory_order_acq_rel, memory_order_seq_cst,
or default).
- Emit an OpMemoryBarrier with Acquire semantics immediately after
OpAtomicExchange when acquire ordering is requested
(memory_order_acquire, memory_order_acq_rel, memory_order_seq_cst,
or default).
- Do not emit OpMemoryBarrier when memory_order_relaxed is specified,
as Vulkan SPIR-V forbids OpMemoryBarrier with semantics 0 (None).diff --git a/include/clspv/Option.h b/include/clspv/Option.h
index 4a486a2..6bda2ba 100644
--- a/include/clspv/Option.h
+++ b/include/clspv/Option.h
@@ -94,6 +94,10 @@
// requires components to be shuffled to match OpenCL specification.
bool HackImage1dBufferBGRA();
+// Returns true if OpMemoryBarrier should be emitted for non-relaxed atomic_flag
+// builtins. Works around driver bugs.
+bool HackAtomicFlagBarrier();
+
// Returns true if module-scope constants are to be collected into a single
// storage buffer. The binding for that buffer, and its intialization data
// are given in the descriptor map file.
diff --git a/lib/Option.cpp b/lib/Option.cpp
index eeb6043..3b3f947 100644
--- a/lib/Option.cpp
+++ b/lib/Option.cpp
@@ -171,6 +171,11 @@
llvm::cl::desc("Insert a dummy instruction after conversions to float to "
"avoid driver optimization getting rid of the conversion"));
+llvm::cl::opt<bool> hack_atomic_flag_barrier(
+ "hack-atomic-flag-barrier", llvm::cl::init(false),
+ llvm::cl::desc(
+ "Emit OpMemoryBarrier for non-relaxed atomic_flag builtins"));
+
llvm::cl::opt<bool>
pod_ubo("pod-ubo", llvm::cl::init(false),
llvm::cl::desc("POD kernel arguments are in uniform buffers"));
@@ -550,6 +555,7 @@
bool HackLogicalPtrtoint() { return hack_logical_ptrtoint; }
bool HackConvertToFloat() { return hack_convert_to_float; }
bool HackImage1dBufferBGRA() { return hack_image1d_buffer_bgra; }
+bool HackAtomicFlagBarrier() { return hack_atomic_flag_barrier; }
bool ModuleConstantsInStorageBuffer() {
return module_constants_in_storage_buffer;
}
diff --git a/lib/ReplaceOpenCLBuiltinPass.cpp b/lib/ReplaceOpenCLBuiltinPass.cpp
index 3c4549c..2902cc3 100644
--- a/lib/ReplaceOpenCLBuiltinPass.cpp
+++ b/lib/ReplaceOpenCLBuiltinPass.cpp
@@ -3884,6 +3884,42 @@
});
}
+namespace {
+bool OrderHasAcquire(Value *order_arg) {
+ if (order_arg == nullptr) {
+ return true;
+ }
+ if (auto const_order = dyn_cast<ConstantInt>(order_arg)) {
+ auto order_val =
+ static_cast<AtomicMemoryOrder>(const_order->getZExtValue());
+ return order_val == AtomicMemoryOrder::kMemoryOrderAcquire ||
+ order_val == AtomicMemoryOrder::kMemoryOrderAcqRel ||
+ order_val == AtomicMemoryOrder::kMemoryOrderSeqCst;
+ }
+ return true;
+}
+
+bool OrderHasRelease(Value *order_arg) {
+ if (order_arg == nullptr) {
+ return true;
+ }
+ if (auto const_order = dyn_cast<ConstantInt>(order_arg)) {
+ auto order_val =
+ static_cast<AtomicMemoryOrder>(const_order->getZExtValue());
+ return order_val == AtomicMemoryOrder::kMemoryOrderRelease ||
+ order_val == AtomicMemoryOrder::kMemoryOrderAcqRel ||
+ order_val == AtomicMemoryOrder::kMemoryOrderSeqCst;
+ }
+ return true;
+}
+
+void InsertMemoryBarrier(CallInst *Call, Value *scope, Value *semantics) {
+ Type *void_ty = Type::getVoidTy(Call->getContext());
+ InsertSPIRVOp(Call, spv::OpMemoryBarrier, {Attribute::Convergent}, void_ty,
+ {scope, semantics});
+}
+} // namespace
+
bool ReplaceOpenCLBuiltinPass::replaceAtomicFlagTestAndSet(Function &F) {
// convert
// %was_set = OpAtomicFlagTestAndSet %bool %flag %scope %semantics
@@ -3908,10 +3944,26 @@
auto scope = MemoryScope(scope_arg, is_global, Call);
IRBuilder<> builder(Call);
+ uint32_t storage = is_global ? spv::MemorySemanticsUniformMemoryMask
+ : spv::MemorySemanticsWorkgroupMemoryMask;
+
+ if (clspv::Option::HackAtomicFlagBarrier() && OrderHasRelease(order_arg)) {
+ auto release_semantics =
+ builder.getInt32(spv::MemorySemanticsReleaseMask | storage);
+ InsertMemoryBarrier(Call, scope, release_semantics);
+ }
+
auto set_value = builder.getInt32(1);
auto previous_value =
InsertSPIRVOp(Call, spv::OpAtomicExchange, {}, set_value->getType(),
{flag_pointer, scope, semantics, set_value});
+
+ if (clspv::Option::HackAtomicFlagBarrier() && OrderHasAcquire(order_arg)) {
+ auto acquire_semantics =
+ builder.getInt32(spv::MemorySemanticsAcquireMask | storage);
+ InsertMemoryBarrier(Call, scope, acquire_semantics);
+ }
+
return builder.CreateICmpEQ(previous_value, set_value);
});
}
@@ -3941,6 +3993,14 @@
auto scope = MemoryScope(scope_arg, is_global, Call);
IRBuilder<> builder(Call);
+ if (clspv::Option::HackAtomicFlagBarrier() && OrderHasRelease(order_arg)) {
+ uint32_t storage = is_global ? spv::MemorySemanticsUniformMemoryMask
+ : spv::MemorySemanticsWorkgroupMemoryMask;
+ auto release_semantics =
+ builder.getInt32(spv::MemorySemanticsReleaseMask | storage);
+ InsertMemoryBarrier(Call, scope, release_semantics);
+ }
+
auto clear_value = builder.getInt32(0);
return InsertSPIRVOp(Call, spv::OpAtomicStore, {}, builder.getVoidTy(),
{flag_pointer, scope, semantics, clear_value});
diff --git a/test/AtomicBuiltins/atomic_flag_hack_atomic_flag_barrier.cl b/test/AtomicBuiltins/atomic_flag_hack_atomic_flag_barrier.cl
new file mode 100644
index 0000000..c6fc6b4
--- /dev/null
+++ b/test/AtomicBuiltins/atomic_flag_hack_atomic_flag_barrier.cl
@@ -0,0 +1,368 @@
+// RUN: clspv %s --cl-std=CL3.0 --enable-feature-macros=__opencl_c_atomic_order_seq_cst,__opencl_c_atomic_scope_device -hack-atomic-flag-barrier -o %t.spv
+// RUN: spirv-val --target-env vulkan1.1 %t.spv
+// RUN: spirv-dis %t.spv | FileCheck %s
+// RUN: clspv %s --cl-std=CL2.0 -inline-entry-points -hack-atomic-flag-barrier -o %t.spv
+// RUN: spirv-val --target-env vulkan1.1 %t.spv
+// RUN: spirv-dis %t.spv | FileCheck %s
+
+// CHECK-DAG: OpEntryPoint GLCompute %[[flag_global:[a-zA-Z0-9_]+]] "flag_global"
+// CHECK-DAG: OpEntryPoint GLCompute %[[flag_local:[a-zA-Z0-9_]+]] "flag_local"
+// CHECK-DAG: OpEntryPoint GLCompute %[[flag_set_partial_explicit_global:[a-zA-Z0-9_]+]] "flag_set_partial_explicit_global"
+// CHECK-DAG: OpEntryPoint GLCompute %[[flag_set_partial_explicit_local:[a-zA-Z0-9_]+]] "flag_set_partial_explicit_local"
+// CHECK-DAG: OpEntryPoint GLCompute %[[flag_clear_partial_explicit_global:[a-zA-Z0-9_]+]] "flag_clear_partial_explicit_global"
+// CHECK-DAG: OpEntryPoint GLCompute %[[flag_clear_partial_explicit_local:[a-zA-Z0-9_]+]] "flag_clear_partial_explicit_local"
+// CHECK-DAG: OpEntryPoint GLCompute %[[flag_set_full_explicit_global:[a-zA-Z0-9_]+]] "flag_set_full_explicit_global"
+// CHECK-DAG: OpEntryPoint GLCompute %[[flag_set_full_explicit_local:[a-zA-Z0-9_]+]] "flag_set_full_explicit_local"
+// CHECK-DAG: OpEntryPoint GLCompute %[[flag_clear_full_explicit_global:[a-zA-Z0-9_]+]] "flag_clear_full_explicit_global"
+// CHECK-DAG: OpEntryPoint GLCompute %[[flag_clear_full_explicit_local:[a-zA-Z0-9_]+]] "flag_clear_full_explicit_local"
+
+// CHECK-DAG: %[[UINT:[a-zA-Z0-9_]+]] = OpTypeInt 32 0
+// CHECK-DAG: %[[BOOL:[a-zA-Z0-9_]+]] = OpTypeBool
+
+// 0 = Relaxed
+// CHECK-DAG: %[[UINT_0:[a-zA-Z0-9_]+]] = OpConstant %[[UINT]] 0
+
+// 1 = Device Scope
+// CHECK-DAG: %[[UINT_1:[a-zA-Z0-9_]+]] = OpConstant %[[UINT]] 1
+// 2 = Workgroup Scope
+// CHECK-DAG: %[[UINT_2:[a-zA-Z0-9_]+]] = OpConstant %[[UINT]] 2
+// 3 = Subgroup Scope
+// CHECK-DAG: %[[UINT_3:[a-zA-Z0-9_]+]] = OpConstant %[[UINT]] 3
+
+// dec 66 = hex 42 = Acquire & UniformMemory
+// CHECK-DAG: %[[UINT_66:[a-zA-Z0-9_]+]] = OpConstant %[[UINT]] 66
+// dec 68 = hex 44 = Release & UniformMemory
+// CHECK-DAG: %[[UINT_68:[a-zA-Z0-9_]+]] = OpConstant %[[UINT]] 68
+// dec 72 = hex 48 = AcquireRelease & UniformMemory
+// CHECK-DAG: %[[UINT_72:[a-zA-Z0-9_]+]] = OpConstant %[[UINT]] 72
+
+// dec 258 = hex 102 Acquire & WorkgroupMemory
+// CHECK-DAG: %[[UINT_258:[a-zA-Z0-9_]+]] = OpConstant %[[UINT]] 258
+// dec 260 = hex 104 Release & WorkgroupMemory
+// CHECK-DAG: %[[UINT_260:[a-zA-Z0-9_]+]] = OpConstant %[[UINT]] 260
+// dec 264 = hex 108 AcquireRelease & WorkgroupMemory
+// CHECK-DAG: %[[UINT_264:[a-zA-Z0-9_]+]] = OpConstant %[[UINT]] 264
+
+// Note: Device Scope is the default scope
+// Note: SequentiallyConsistent is the default order
+
+// CHECK: %[[flag_global]] = OpFunction %void
+kernel void flag_global(global int *out, global atomic_flag *flag) {
+// CHECK: OpMemoryBarrier %[[UINT_1]] %[[UINT_68]]
+// CHECK-NEXT: %[[previous_value:[a-zA-Z0-9_]+]] = OpAtomicExchange %[[UINT]] {{.*}} %[[UINT_1]] %[[UINT_72]] %[[UINT_1]]
+// CHECK-NEXT: OpMemoryBarrier %[[UINT_1]] %[[UINT_66]]
+// CHECK: OpIEqual %bool %[[previous_value]] %[[UINT_1]]
+ *out = atomic_flag_test_and_set(flag);
+
+// CHECK: OpMemoryBarrier %[[UINT_1]] %[[UINT_68]]
+// CHECK-NEXT: OpAtomicStore {{.*}} %[[UINT_1]] %[[UINT_68]] %[[UINT_0]]
+ atomic_flag_clear(flag);
+}
+
+// CHECK: %[[flag_local]] = OpFunction %void
+kernel void flag_local(global int *out, local atomic_flag *flag) {
+// CHECK: OpMemoryBarrier %[[UINT_2]] %[[UINT_260]]
+// CHECK-NEXT: %[[previous_value:[a-zA-Z0-9_]+]] = OpAtomicExchange %[[UINT]] {{.*}} %[[UINT_2]] %[[UINT_264]] %[[UINT_1]]
+// CHECK-NEXT: OpMemoryBarrier %[[UINT_2]] %[[UINT_258]]
+// CHECK: OpIEqual %bool %[[previous_value]] %[[UINT_1]]
+ *out = atomic_flag_test_and_set(flag);
+
+// CHECK: OpMemoryBarrier %[[UINT_2]] %[[UINT_260]]
+// CHECK-NEXT: OpAtomicStore {{.*}} %[[UINT_2]] %[[UINT_260]] %[[UINT_0]]
+ atomic_flag_clear(flag);
+}
+
+// explicit scope
+
+// CHECK: %[[flag_set_partial_explicit_global]] = OpFunction %void
+kernel void flag_set_partial_explicit_global(global int *out, global atomic_flag *flag) {
+
+// CHECK: OpMemoryBarrier %[[UINT_1]] %[[UINT_68]]
+// CHECK-NEXT: %[[previous_value:[a-zA-Z0-9_]+]] = OpAtomicExchange %[[UINT]] {{.*}} %[[UINT_1]] %[[UINT_72]] %[[UINT_1]]
+// CHECK-NEXT: OpMemoryBarrier %[[UINT_1]] %[[UINT_66]]
+// CHECK: OpIEqual %bool %[[previous_value]] %[[UINT_1]]
+ *out = atomic_flag_test_and_set_explicit(flag, memory_order_seq_cst);
+
+// CHECK: OpMemoryBarrier %[[UINT_1]] %[[UINT_68]]
+// CHECK-NEXT: %[[previous_value:[a-zA-Z0-9_]+]] = OpAtomicExchange %[[UINT]] {{.*}} %[[UINT_1]] %[[UINT_72]] %[[UINT_1]]
+// CHECK-NEXT: OpMemoryBarrier %[[UINT_1]] %[[UINT_66]]
+// CHECK: OpIEqual %bool %[[previous_value]] %[[UINT_1]]
+ *out = atomic_flag_test_and_set_explicit(flag, memory_order_acq_rel);
+
+// CHECK: %[[previous_value:[a-zA-Z0-9_]+]] = OpAtomicExchange %[[UINT]] {{.*}} %[[UINT_1]] %[[UINT_66]] %[[UINT_1]]
+// CHECK-NEXT: OpMemoryBarrier %[[UINT_1]] %[[UINT_66]]
+// CHECK: OpIEqual %bool %[[previous_value]] %[[UINT_1]]
+ *out = atomic_flag_test_and_set_explicit(flag, memory_order_acquire);
+
+// CHECK: OpMemoryBarrier %[[UINT_1]] %[[UINT_68]]
+// CHECK-NEXT: %[[previous_value:[a-zA-Z0-9_]+]] = OpAtomicExchange %[[UINT]] {{.*}} %[[UINT_1]] %[[UINT_68]] %[[UINT_1]]
+// CHECK: OpIEqual %bool %[[previous_value]] %[[UINT_1]]
+ *out = atomic_flag_test_and_set_explicit(flag, memory_order_release);
+
+// CHECK: %[[previous_value:[a-zA-Z0-9_]+]] = OpAtomicExchange %[[UINT]] {{.*}} %[[UINT_1]] %[[UINT_0]] %[[UINT_1]]
+// CHECK-NEXT: OpIEqual %bool %[[previous_value]] %[[UINT_1]]
+ *out = atomic_flag_test_and_set_explicit(flag, memory_order_relaxed);
+}
+
+// CHECK: %[[flag_set_partial_explicit_local]] = OpFunction %void
+kernel void flag_set_partial_explicit_local(global int *out, local atomic_flag *flag) {
+// CHECK: OpMemoryBarrier %[[UINT_2]] %[[UINT_260]]
+// CHECK-NEXT: %[[previous_value:[a-zA-Z0-9_]+]] = OpAtomicExchange %[[UINT]] {{.*}} %[[UINT_2]] %[[UINT_264]] %[[UINT_1]]
+// CHECK-NEXT: OpMemoryBarrier %[[UINT_2]] %[[UINT_258]]
+// CHECK: OpIEqual %bool %[[previous_value]] %[[UINT_1]]
+ *out = atomic_flag_test_and_set_explicit(flag, memory_order_seq_cst);
+
+// CHECK: OpMemoryBarrier %[[UINT_2]] %[[UINT_260]]
+// CHECK-NEXT: %[[previous_value:[a-zA-Z0-9_]+]] = OpAtomicExchange %[[UINT]] {{.*}} %[[UINT_2]] %[[UINT_264]] %[[UINT_1]]
+// CHECK-NEXT: OpMemoryBarrier %[[UINT_2]] %[[UINT_258]]
+// CHECK: OpIEqual %bool %[[previous_value]] %[[UINT_1]]
+ *out = atomic_flag_test_and_set_explicit(flag, memory_order_acq_rel);
+
+// CHECK: %[[previous_value:[a-zA-Z0-9_]+]] = OpAtomicExchange %[[UINT]] {{.*}} %[[UINT_2]] %[[UINT_258]] %[[UINT_1]]
+// CHECK-NEXT: OpMemoryBarrier %[[UINT_2]] %[[UINT_258]]
+// CHECK: OpIEqual %bool %[[previous_value]] %[[UINT_1]]
+ *out = atomic_flag_test_and_set_explicit(flag, memory_order_acquire);
+
+// CHECK: OpMemoryBarrier %[[UINT_2]] %[[UINT_260]]
+// CHECK-NEXT: %[[previous_value:[a-zA-Z0-9_]+]] = OpAtomicExchange %[[UINT]] {{.*}} %[[UINT_2]] %[[UINT_260]] %[[UINT_1]]
+// CHECK: OpIEqual %bool %[[previous_value]] %[[UINT_1]]
+ *out = atomic_flag_test_and_set_explicit(flag, memory_order_release);
+
+// CHECK: %[[previous_value:[a-zA-Z0-9_]+]] = OpAtomicExchange %[[UINT]] {{.*}} %[[UINT_2]] %[[UINT_0]] %[[UINT_1]]
+// CHECK-NEXT: OpIEqual %bool %[[previous_value]] %[[UINT_1]]
+ *out = atomic_flag_test_and_set_explicit(flag, memory_order_relaxed);
+}
+
+// CHECK: %[[flag_clear_partial_explicit_global]] = OpFunction %void
+kernel void flag_clear_partial_explicit_global(global atomic_flag *flag) {
+// CHECK: OpMemoryBarrier %[[UINT_1]] %[[UINT_68]]
+// CHECK-NEXT: OpAtomicStore {{.*}} %[[UINT_1]] %[[UINT_68]] %[[UINT_0]]
+ atomic_flag_clear_explicit(flag, memory_order_seq_cst);
+
+// CHECK: OpMemoryBarrier %[[UINT_1]] %[[UINT_68]]
+// CHECK-NEXT: OpAtomicStore {{.*}} %[[UINT_1]] %[[UINT_68]] %[[UINT_0]]
+ atomic_flag_clear_explicit(flag, memory_order_release);
+
+// CHECK: OpAtomicStore {{.*}} %[[UINT_1]] %[[UINT_0]] %[[UINT_0]]
+ atomic_flag_clear_explicit(flag, memory_order_relaxed);
+}
+
+// CHECK: %[[flag_clear_partial_explicit_local]] = OpFunction %void
+kernel void flag_clear_partial_explicit_local(local atomic_flag *flag) {
+// CHECK: OpMemoryBarrier %[[UINT_2]] %[[UINT_260]]
+// CHECK-NEXT: OpAtomicStore {{.*}} %[[UINT_2]] %[[UINT_260]] %[[UINT_0]]
+ atomic_flag_clear_explicit(flag, memory_order_seq_cst);
+
+// CHECK: OpMemoryBarrier %[[UINT_2]] %[[UINT_260]]
+// CHECK-NEXT: OpAtomicStore {{.*}} %[[UINT_2]] %[[UINT_260]] %[[UINT_0]]
+ atomic_flag_clear_explicit(flag, memory_order_release);
+
+// CHECK: OpAtomicStore {{.*}} %[[UINT_2]] %[[UINT_0]] %[[UINT_0]]
+ atomic_flag_clear_explicit(flag, memory_order_relaxed);
+}
+
+// explicit order and scope
+
+// CHECK: %[[flag_set_full_explicit_global]] = OpFunction %void
+kernel void flag_set_full_explicit_global(global int *out, global atomic_flag *flag) {
+
+// CHECK: OpMemoryBarrier %[[UINT_3]] %[[UINT_68]]
+// CHECK-NEXT: %[[previous_value:[a-zA-Z0-9_]+]] = OpAtomicExchange %[[UINT]] {{.*}} %[[UINT_3]] %[[UINT_72]] %[[UINT_1]]
+// CHECK-NEXT: OpMemoryBarrier %[[UINT_3]] %[[UINT_66]]
+// CHECK: OpIEqual %bool %[[previous_value]] %[[UINT_1]]
+ *out = atomic_flag_test_and_set_explicit(flag, memory_order_seq_cst, memory_scope_sub_group);
+// CHECK: OpMemoryBarrier %[[UINT_3]] %[[UINT_68]]
+// CHECK-NEXT: %[[previous_value:[a-zA-Z0-9_]+]] = OpAtomicExchange %[[UINT]] {{.*}} %[[UINT_3]] %[[UINT_72]] %[[UINT_1]]
+// CHECK-NEXT: OpMemoryBarrier %[[UINT_3]] %[[UINT_66]]
+// CHECK: OpIEqual %bool %[[previous_value]] %[[UINT_1]]
+ *out = atomic_flag_test_and_set_explicit(flag, memory_order_acq_rel, memory_scope_sub_group);
+// CHECK: %[[previous_value:[a-zA-Z0-9_]+]] = OpAtomicExchange %[[UINT]] {{.*}} %[[UINT_3]] %[[UINT_66]] %[[UINT_1]]
+// CHECK-NEXT: OpMemoryBarrier %[[UINT_3]] %[[UINT_66]]
+// CHECK: OpIEqual %bool %[[previous_value]] %[[UINT_1]]
+ *out = atomic_flag_test_and_set_explicit(flag, memory_order_acquire, memory_scope_sub_group);
+// CHECK: OpMemoryBarrier %[[UINT_3]] %[[UINT_68]]
+// CHECK-NEXT: %[[previous_value:[a-zA-Z0-9_]+]] = OpAtomicExchange %[[UINT]] {{.*}} %[[UINT_3]] %[[UINT_68]] %[[UINT_1]]
+// CHECK: OpIEqual %bool %[[previous_value]] %[[UINT_1]]
+ *out = atomic_flag_test_and_set_explicit(flag, memory_order_release, memory_scope_sub_group);
+// CHECK: %[[previous_value:[a-zA-Z0-9_]+]] = OpAtomicExchange %[[UINT]] {{.*}} %[[UINT_3]] %[[UINT_0]] %[[UINT_1]]
+// CHECK-NEXT: OpIEqual %bool %[[previous_value]] %[[UINT_1]]
+ *out = atomic_flag_test_and_set_explicit(flag, memory_order_relaxed, memory_scope_sub_group);
+
+// CHECK: OpMemoryBarrier %[[UINT_2]] %[[UINT_68]]
+// CHECK-NEXT: %[[previous_value:[a-zA-Z0-9_]+]] = OpAtomicExchange %[[UINT]] {{.*}} %[[UINT_2]] %[[UINT_72]] %[[UINT_1]]
+// CHECK-NEXT: OpMemoryBarrier %[[UINT_2]] %[[UINT_66]]
+// CHECK: OpIEqual %bool %[[previous_value]] %[[UINT_1]]
+ *out = atomic_flag_test_and_set_explicit(flag, memory_order_seq_cst, memory_scope_work_group);
+// CHECK: OpMemoryBarrier %[[UINT_2]] %[[UINT_68]]
+// CHECK-NEXT: %[[previous_value:[a-zA-Z0-9_]+]] = OpAtomicExchange %[[UINT]] {{.*}} %[[UINT_2]] %[[UINT_72]] %[[UINT_1]]
+// CHECK-NEXT: OpMemoryBarrier %[[UINT_2]] %[[UINT_66]]
+// CHECK: OpIEqual %bool %[[previous_value]] %[[UINT_1]]
+ *out = atomic_flag_test_and_set_explicit(flag, memory_order_acq_rel, memory_scope_work_group);
+// CHECK: %[[previous_value:[a-zA-Z0-9_]+]] = OpAtomicExchange %[[UINT]] {{.*}} %[[UINT_2]] %[[UINT_66]] %[[UINT_1]]
+// CHECK-NEXT: OpMemoryBarrier %[[UINT_2]] %[[UINT_66]]
+// CHECK: OpIEqual %bool %[[previous_value]] %[[UINT_1]]
+ *out = atomic_flag_test_and_set_explicit(flag, memory_order_acquire, memory_scope_work_group);
+// CHECK: OpMemoryBarrier %[[UINT_2]] %[[UINT_68]]
+// CHECK-NEXT: %[[previous_value:[a-zA-Z0-9_]+]] = OpAtomicExchange %[[UINT]] {{.*}} %[[UINT_2]] %[[UINT_68]] %[[UINT_1]]
+// CHECK: OpIEqual %bool %[[previous_value]] %[[UINT_1]]
+ *out = atomic_flag_test_and_set_explicit(flag, memory_order_release, memory_scope_work_group);
+// CHECK: %[[previous_value:[a-zA-Z0-9_]+]] = OpAtomicExchange %[[UINT]] {{.*}} %[[UINT_2]] %[[UINT_0]] %[[UINT_1]]
+// CHECK-NEXT: OpIEqual %bool %[[previous_value]] %[[UINT_1]]
+ *out = atomic_flag_test_and_set_explicit(flag, memory_order_relaxed, memory_scope_work_group);
+
+// CHECK: OpMemoryBarrier %[[UINT_1]] %[[UINT_68]]
+// CHECK-NEXT: %[[previous_value:[a-zA-Z0-9_]+]] = OpAtomicExchange %[[UINT]] {{.*}} %[[UINT_1]] %[[UINT_72]] %[[UINT_1]]
+// CHECK-NEXT: OpMemoryBarrier %[[UINT_1]] %[[UINT_66]]
+// CHECK: OpIEqual %bool %[[previous_value]] %[[UINT_1]]
+ *out = atomic_flag_test_and_set_explicit(flag, memory_order_seq_cst, memory_scope_device);
+// CHECK: OpMemoryBarrier %[[UINT_1]] %[[UINT_68]]
+// CHECK-NEXT: %[[previous_value:[a-zA-Z0-9_]+]] = OpAtomicExchange %[[UINT]] {{.*}} %[[UINT_1]] %[[UINT_72]] %[[UINT_1]]
+// CHECK-NEXT: OpMemoryBarrier %[[UINT_1]] %[[UINT_66]]
+// CHECK: OpIEqual %bool %[[previous_value]] %[[UINT_1]]
+ *out = atomic_flag_test_and_set_explicit(flag, memory_order_acq_rel, memory_scope_device);
+// CHECK: %[[previous_value:[a-zA-Z0-9_]+]] = OpAtomicExchange %[[UINT]] {{.*}} %[[UINT_1]] %[[UINT_66]] %[[UINT_1]]
+// CHECK-NEXT: OpMemoryBarrier %[[UINT_1]] %[[UINT_66]]
+// CHECK: OpIEqual %bool %[[previous_value]] %[[UINT_1]]
+ *out = atomic_flag_test_and_set_explicit(flag, memory_order_acquire, memory_scope_device);
+// CHECK: OpMemoryBarrier %[[UINT_1]] %[[UINT_68]]
+// CHECK-NEXT: %[[previous_value:[a-zA-Z0-9_]+]] = OpAtomicExchange %[[UINT]] {{.*}} %[[UINT_1]] %[[UINT_68]] %[[UINT_1]]
+// CHECK: OpIEqual %bool %[[previous_value]] %[[UINT_1]]
+ *out = atomic_flag_test_and_set_explicit(flag, memory_order_release, memory_scope_device);
+// CHECK: %[[previous_value:[a-zA-Z0-9_]+]] = OpAtomicExchange %[[UINT]] {{.*}} %[[UINT_1]] %[[UINT_0]] %[[UINT_1]]
+// CHECK-NEXT: OpIEqual %bool %[[previous_value]] %[[UINT_1]]
+ *out = atomic_flag_test_and_set_explicit(flag, memory_order_relaxed, memory_scope_device);
+
+}
+
+// CHECK: %[[flag_set_full_explicit_local]] = OpFunction %void
+kernel void flag_set_full_explicit_local(global int *out, local atomic_flag *flag) {
+
+// CHECK: OpMemoryBarrier %[[UINT_3]] %[[UINT_260]]
+// CHECK-NEXT: %[[previous_value:[a-zA-Z0-9_]+]] = OpAtomicExchange %[[UINT]] {{.*}} %[[UINT_3]] %[[UINT_264]] %[[UINT_1]]
+// CHECK-NEXT: OpMemoryBarrier %[[UINT_3]] %[[UINT_258]]
+// CHECK: OpIEqual %bool %[[previous_value]] %[[UINT_1]]
+ *out = atomic_flag_test_and_set_explicit(flag, memory_order_seq_cst, memory_scope_sub_group);
+// CHECK: OpMemoryBarrier %[[UINT_3]] %[[UINT_260]]
+// CHECK-NEXT: %[[previous_value:[a-zA-Z0-9_]+]] = OpAtomicExchange %[[UINT]] {{.*}} %[[UINT_3]] %[[UINT_264]] %[[UINT_1]]
+// CHECK-NEXT: OpMemoryBarrier %[[UINT_3]] %[[UINT_258]]
+// CHECK: OpIEqual %bool %[[previous_value]] %[[UINT_1]]
+ *out = atomic_flag_test_and_set_explicit(flag, memory_order_acq_rel, memory_scope_sub_group);
+// CHECK: %[[previous_value:[a-zA-Z0-9_]+]] = OpAtomicExchange %[[UINT]] {{.*}} %[[UINT_3]] %[[UINT_258]] %[[UINT_1]]
+// CHECK-NEXT: OpMemoryBarrier %[[UINT_3]] %[[UINT_258]]
+// CHECK: OpIEqual %bool %[[previous_value]] %[[UINT_1]]
+ *out = atomic_flag_test_and_set_explicit(flag, memory_order_acquire, memory_scope_sub_group);
+// CHECK: OpMemoryBarrier %[[UINT_3]] %[[UINT_260]]
+// CHECK-NEXT: %[[previous_value:[a-zA-Z0-9_]+]] = OpAtomicExchange %[[UINT]] {{.*}} %[[UINT_3]] %[[UINT_260]] %[[UINT_1]]
+// CHECK: OpIEqual %bool %[[previous_value]] %[[UINT_1]]
+ *out = atomic_flag_test_and_set_explicit(flag, memory_order_release, memory_scope_sub_group);
+// CHECK: %[[previous_value:[a-zA-Z0-9_]+]] = OpAtomicExchange %[[UINT]] {{.*}} %[[UINT_3]] %[[UINT_0]] %[[UINT_1]]
+// CHECK-NEXT: OpIEqual %bool %[[previous_value]] %[[UINT_1]]
+ *out = atomic_flag_test_and_set_explicit(flag, memory_order_relaxed, memory_scope_sub_group);
+
+// CHECK: OpMemoryBarrier %[[UINT_2]] %[[UINT_260]]
+// CHECK-NEXT: %[[previous_value:[a-zA-Z0-9_]+]] = OpAtomicExchange %[[UINT]] {{.*}} %[[UINT_2]] %[[UINT_264]] %[[UINT_1]]
+// CHECK-NEXT: OpMemoryBarrier %[[UINT_2]] %[[UINT_258]]
+// CHECK: OpIEqual %bool %[[previous_value]] %[[UINT_1]]
+ *out = atomic_flag_test_and_set_explicit(flag, memory_order_seq_cst, memory_scope_work_group);
+// CHECK: OpMemoryBarrier %[[UINT_2]] %[[UINT_260]]
+// CHECK-NEXT: %[[previous_value:[a-zA-Z0-9_]+]] = OpAtomicExchange %[[UINT]] {{.*}} %[[UINT_2]] %[[UINT_264]] %[[UINT_1]]
+// CHECK-NEXT: OpMemoryBarrier %[[UINT_2]] %[[UINT_258]]
+// CHECK: OpIEqual %bool %[[previous_value]] %[[UINT_1]]
+ *out = atomic_flag_test_and_set_explicit(flag, memory_order_acq_rel, memory_scope_work_group);
+// CHECK: %[[previous_value:[a-zA-Z0-9_]+]] = OpAtomicExchange %[[UINT]] {{.*}} %[[UINT_2]] %[[UINT_258]] %[[UINT_1]]
+// CHECK-NEXT: OpMemoryBarrier %[[UINT_2]] %[[UINT_258]]
+// CHECK: OpIEqual %bool %[[previous_value]] %[[UINT_1]]
+ *out = atomic_flag_test_and_set_explicit(flag, memory_order_acquire, memory_scope_work_group);
+// CHECK: OpMemoryBarrier %[[UINT_2]] %[[UINT_260]]
+// CHECK-NEXT: %[[previous_value:[a-zA-Z0-9_]+]] = OpAtomicExchange %[[UINT]] {{.*}} %[[UINT_2]] %[[UINT_260]] %[[UINT_1]]
+// CHECK: OpIEqual %bool %[[previous_value]] %[[UINT_1]]
+ *out = atomic_flag_test_and_set_explicit(flag, memory_order_release, memory_scope_work_group);
+// CHECK: %[[previous_value:[a-zA-Z0-9_]+]] = OpAtomicExchange %[[UINT]] {{.*}} %[[UINT_2]] %[[UINT_0]] %[[UINT_1]]
+// CHECK-NEXT: OpIEqual %bool %[[previous_value]] %[[UINT_1]]
+ *out = atomic_flag_test_and_set_explicit(flag, memory_order_relaxed, memory_scope_work_group);
+
+// CHECK: OpMemoryBarrier %[[UINT_1]] %[[UINT_260]]
+// CHECK-NEXT: %[[previous_value:[a-zA-Z0-9_]+]] = OpAtomicExchange %[[UINT]] {{.*}} %[[UINT_1]] %[[UINT_264]] %[[UINT_1]]
+// CHECK-NEXT: OpMemoryBarrier %[[UINT_1]] %[[UINT_258]]
+// CHECK: OpIEqual %bool %[[previous_value]] %[[UINT_1]]
+ *out = atomic_flag_test_and_set_explicit(flag, memory_order_seq_cst, memory_scope_device);
+// CHECK: OpMemoryBarrier %[[UINT_1]] %[[UINT_260]]
+// CHECK-NEXT: %[[previous_value:[a-zA-Z0-9_]+]] = OpAtomicExchange %[[UINT]] {{.*}} %[[UINT_1]] %[[UINT_264]] %[[UINT_1]]
+// CHECK-NEXT: OpMemoryBarrier %[[UINT_1]] %[[UINT_258]]
+// CHECK: OpIEqual %bool %[[previous_value]] %[[UINT_1]]
+ *out = atomic_flag_test_and_set_explicit(flag, memory_order_acq_rel, memory_scope_device);
+// CHECK: %[[previous_value:[a-zA-Z0-9_]+]] = OpAtomicExchange %[[UINT]] {{.*}} %[[UINT_1]] %[[UINT_258]] %[[UINT_1]]
+// CHECK-NEXT: OpMemoryBarrier %[[UINT_1]] %[[UINT_258]]
+// CHECK: OpIEqual %bool %[[previous_value]] %[[UINT_1]]
+ *out = atomic_flag_test_and_set_explicit(flag, memory_order_acquire, memory_scope_device);
+// CHECK: OpMemoryBarrier %[[UINT_1]] %[[UINT_260]]
+// CHECK-NEXT: %[[previous_value:[a-zA-Z0-9_]+]] = OpAtomicExchange %[[UINT]] {{.*}} %[[UINT_1]] %[[UINT_260]] %[[UINT_1]]
+// CHECK: OpIEqual %bool %[[previous_value]] %[[UINT_1]]
+ *out = atomic_flag_test_and_set_explicit(flag, memory_order_release, memory_scope_device);
+// CHECK: %[[previous_value:[a-zA-Z0-9_]+]] = OpAtomicExchange %[[UINT]] {{.*}} %[[UINT_1]] %[[UINT_0]] %[[UINT_1]]
+// CHECK-NEXT: OpIEqual %bool %[[previous_value]] %[[UINT_1]]
+ *out = atomic_flag_test_and_set_explicit(flag, memory_order_relaxed, memory_scope_device);
+
+}
+
+// CHECK: %[[flag_clear_full_explicit_global]] = OpFunction %void
+ kernel void flag_clear_full_explicit_global(global atomic_flag *flag) {
+// CHECK: OpMemoryBarrier %[[UINT_3]] %[[UINT_68]]
+// CHECK-NEXT: OpAtomicStore {{.*}} %[[UINT_3]] %[[UINT_68]] %[[UINT_0]]
+ atomic_flag_clear_explicit(flag, memory_order_seq_cst, memory_scope_sub_group);
+// CHECK: OpMemoryBarrier %[[UINT_3]] %[[UINT_68]]
+// CHECK-NEXT: OpAtomicStore {{.*}} %[[UINT_3]] %[[UINT_68]] %[[UINT_0]]
+ atomic_flag_clear_explicit(flag, memory_order_release, memory_scope_sub_group);
+// CHECK: OpAtomicStore {{.*}} %[[UINT_3]] %[[UINT_0]] %[[UINT_0]]
+ atomic_flag_clear_explicit(flag, memory_order_relaxed, memory_scope_sub_group);
+
+// CHECK: OpMemoryBarrier %[[UINT_2]] %[[UINT_68]]
+// CHECK-NEXT: OpAtomicStore {{.*}} %[[UINT_2]] %[[UINT_68]] %[[UINT_0]]
+ atomic_flag_clear_explicit(flag, memory_order_seq_cst, memory_scope_work_group);
+// CHECK: OpMemoryBarrier %[[UINT_2]] %[[UINT_68]]
+// CHECK-NEXT: OpAtomicStore {{.*}} %[[UINT_2]] %[[UINT_68]] %[[UINT_0]]
+ atomic_flag_clear_explicit(flag, memory_order_release, memory_scope_work_group);
+// CHECK: OpAtomicStore {{.*}} %[[UINT_2]] %[[UINT_0]] %[[UINT_0]]
+ atomic_flag_clear_explicit(flag, memory_order_relaxed, memory_scope_work_group);
+
+// CHECK: OpMemoryBarrier %[[UINT_1]] %[[UINT_68]]
+// CHECK-NEXT: OpAtomicStore {{.*}} %[[UINT_1]] %[[UINT_68]] %[[UINT_0]]
+ atomic_flag_clear_explicit(flag, memory_order_seq_cst, memory_scope_device);
+// CHECK: OpMemoryBarrier %[[UINT_1]] %[[UINT_68]]
+// CHECK-NEXT: OpAtomicStore {{.*}} %[[UINT_1]] %[[UINT_68]] %[[UINT_0]]
+ atomic_flag_clear_explicit(flag, memory_order_release, memory_scope_device);
+// CHECK: OpAtomicStore {{.*}} %[[UINT_1]] %[[UINT_0]] %[[UINT_0]]
+ atomic_flag_clear_explicit(flag, memory_order_relaxed, memory_scope_device);
+
+}
+
+// CHECK: %[[flag_clear_full_explicit_local]] = OpFunction %void
+ kernel void flag_clear_full_explicit_local(local atomic_flag *flag) {
+
+// CHECK: OpMemoryBarrier %[[UINT_3]] %[[UINT_260]]
+// CHECK-NEXT: OpAtomicStore {{.*}} %[[UINT_3]] %[[UINT_260]] %[[UINT_0]]
+ atomic_flag_clear_explicit(flag, memory_order_seq_cst, memory_scope_sub_group);
+// CHECK: OpMemoryBarrier %[[UINT_3]] %[[UINT_260]]
+// CHECK-NEXT: OpAtomicStore {{.*}} %[[UINT_3]] %[[UINT_260]] %[[UINT_0]]
+ atomic_flag_clear_explicit(flag, memory_order_release, memory_scope_sub_group);
+// CHECK: OpAtomicStore {{.*}} %[[UINT_3]] %[[UINT_0]] %[[UINT_0]]
+ atomic_flag_clear_explicit(flag, memory_order_relaxed, memory_scope_sub_group);
+
+// CHECK: OpMemoryBarrier %[[UINT_2]] %[[UINT_260]]
+// CHECK-NEXT: OpAtomicStore {{.*}} %[[UINT_2]] %[[UINT_260]] %[[UINT_0]]
+ atomic_flag_clear_explicit(flag, memory_order_seq_cst, memory_scope_work_group);
+// CHECK: OpMemoryBarrier %[[UINT_2]] %[[UINT_260]]
+// CHECK-NEXT: OpAtomicStore {{.*}} %[[UINT_2]] %[[UINT_260]] %[[UINT_0]]
+ atomic_flag_clear_explicit(flag, memory_order_release, memory_scope_work_group);
+// CHECK: OpAtomicStore {{.*}} %[[UINT_2]] %[[UINT_0]] %[[UINT_0]]
+ atomic_flag_clear_explicit(flag, memory_order_relaxed, memory_scope_work_group);
+
+// CHECK: OpMemoryBarrier %[[UINT_1]] %[[UINT_260]]
+// CHECK-NEXT: OpAtomicStore {{.*}} %[[UINT_1]] %[[UINT_260]] %[[UINT_0]]
+ atomic_flag_clear_explicit(flag, memory_order_seq_cst, memory_scope_device);
+// CHECK: OpMemoryBarrier %[[UINT_1]] %[[UINT_260]]
+// CHECK-NEXT: OpAtomicStore {{.*}} %[[UINT_1]] %[[UINT_260]] %[[UINT_0]]
+ atomic_flag_clear_explicit(flag, memory_order_release, memory_scope_device);
+// CHECK: OpAtomicStore {{.*}} %[[UINT_1]] %[[UINT_0]] %[[UINT_0]]
+ atomic_flag_clear_explicit(flag, memory_order_relaxed, memory_scope_device);
+}
