Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 20 additions & 5 deletions CUDACore/src/device/intrinsics/synchronization.jl
Original file line number Diff line number Diff line change
Expand Up @@ -72,11 +72,26 @@ export barrier_sync

export cluster_arrive, cluster_arrive_relaxed, cluster_wait

@device_functions begin
cluster_arrive() = ccall("llvm.nvvm.barrier.cluster.arrive", llvmcall, Cvoid, ())
cluster_arrive_relaxed() = ccall("llvm.nvvm.barrier.cluster.arrive.relaxed", llvmcall, Cvoid, ())
cluster_wait() = ccall("llvm.nvvm.barrier.cluster.wait", llvmcall, Cvoid, ())
end # @device_functions
# These intrinsics only exist in LLVM 17+, so `ccall(intr, llvmcall, ...)` cannot be
# used here: on older LLVM (Julia <= 1.11) the unknown name demotes to a runtime trap.
# A textual declaration passes through to the NVPTX back end on every version, but
# must spell out the attributes the loaded LLVM can't infer: `convergent`, plus
# `nomerge` to stop pre-17 SimplifyCFG from merging the calls across branches.
for (fn, barrier) in ["cluster_arrive" => "arrive",
"cluster_arrive_relaxed" => "arrive.relaxed",
"cluster_wait" => "wait"]
intr = "llvm.nvvm.barrier.cluster.$barrier"
mod = """
declare void @$intr() #0
define void @entry() #1 {
call void @$intr()
ret void
}
attributes #0 = { convergent nomerge nounwind }
attributes #1 = { alwaysinline }"""
@eval @device_function @inline $(Symbol(fn))() =
Base.llvmcall(($mod, "entry"), Cvoid, Tuple{})
end

## memory barriers (membar)

Expand Down
13 changes: 13 additions & 0 deletions test/core/device/intrinsics/clusters.jl
Original file line number Diff line number Diff line change
@@ -1,4 +1,17 @@
@testset "thread block clusters" begin

# the barriers must survive to device IR on every Julia; on LLVM <= 16 they
# used to demote to a silent trap stub (runs on any hardware)
@testset "barrier codegen" begin
@test @filecheck CUDA.code_llvm(Tuple{}) do
@check "llvm.nvvm.barrier.cluster.arrive"
@check "llvm.nvvm.barrier.cluster.wait"
@check_not "gpu_report_exception"
cluster_arrive()
cluster_wait()
end
end

if capability(device()) >= v"9.0" && VERSION >= v"1.11-"

###########################################################################################
Expand Down