[CUDA] added __nvvm_atom_{sys|cta}_* builtins.
These builtins are available on sm_60+ GPU only.
Differential Revision: https://reviews.llvm.org/D24944
llvm-svn: 282609
diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp
index 8cf2942..2f4fbcf 100644
--- a/clang/lib/CodeGen/CGBuiltin.cpp
+++ b/clang/lib/CodeGen/CGBuiltin.cpp
@@ -8124,7 +8124,13 @@
Ptr->getType()}),
{Ptr, ConstantInt::get(Builder.getInt32Ty(), Align.getQuantity())});
};
-
+ auto MakeScopedAtomic = [&](unsigned IntrinsicID) {
+ Value *Ptr = EmitScalarExpr(E->getArg(0));
+ return Builder.CreateCall(
+ CGM.getIntrinsic(IntrinsicID, {Ptr->getType()->getPointerElementType(),
+ Ptr->getType()}),
+ {Ptr, EmitScalarExpr(E->getArg(1))});
+ };
switch (BuiltinID) {
case NVPTX::BI__nvvm_atom_add_gen_i:
case NVPTX::BI__nvvm_atom_add_gen_l:
@@ -8243,6 +8249,109 @@
case NVPTX::BI__nvvm_ldg_d:
case NVPTX::BI__nvvm_ldg_d2:
return MakeLdg(Intrinsic::nvvm_ldg_global_f);
+
+ case NVPTX::BI__nvvm_atom_cta_add_gen_i:
+ case NVPTX::BI__nvvm_atom_cta_add_gen_l:
+ case NVPTX::BI__nvvm_atom_cta_add_gen_ll:
+ return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_gen_i_cta);
+ case NVPTX::BI__nvvm_atom_sys_add_gen_i:
+ case NVPTX::BI__nvvm_atom_sys_add_gen_l:
+ case NVPTX::BI__nvvm_atom_sys_add_gen_ll:
+ return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_gen_i_sys);
+ case NVPTX::BI__nvvm_atom_cta_add_gen_f:
+ case NVPTX::BI__nvvm_atom_cta_add_gen_d:
+ return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_gen_f_cta);
+ case NVPTX::BI__nvvm_atom_sys_add_gen_f:
+ case NVPTX::BI__nvvm_atom_sys_add_gen_d:
+ return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_gen_f_sys);
+ case NVPTX::BI__nvvm_atom_cta_xchg_gen_i:
+ case NVPTX::BI__nvvm_atom_cta_xchg_gen_l:
+ case NVPTX::BI__nvvm_atom_cta_xchg_gen_ll:
+ return MakeScopedAtomic(Intrinsic::nvvm_atomic_exch_gen_i_cta);
+ case NVPTX::BI__nvvm_atom_sys_xchg_gen_i:
+ case NVPTX::BI__nvvm_atom_sys_xchg_gen_l:
+ case NVPTX::BI__nvvm_atom_sys_xchg_gen_ll:
+ return MakeScopedAtomic(Intrinsic::nvvm_atomic_exch_gen_i_sys);
+ case NVPTX::BI__nvvm_atom_cta_max_gen_i:
+ case NVPTX::BI__nvvm_atom_cta_max_gen_ui:
+ case NVPTX::BI__nvvm_atom_cta_max_gen_l:
+ case NVPTX::BI__nvvm_atom_cta_max_gen_ul:
+ case NVPTX::BI__nvvm_atom_cta_max_gen_ll:
+ case NVPTX::BI__nvvm_atom_cta_max_gen_ull:
+ return MakeScopedAtomic(Intrinsic::nvvm_atomic_max_gen_i_cta);
+ case NVPTX::BI__nvvm_atom_sys_max_gen_i:
+ case NVPTX::BI__nvvm_atom_sys_max_gen_ui:
+ case NVPTX::BI__nvvm_atom_sys_max_gen_l:
+ case NVPTX::BI__nvvm_atom_sys_max_gen_ul:
+ case NVPTX::BI__nvvm_atom_sys_max_gen_ll:
+ case NVPTX::BI__nvvm_atom_sys_max_gen_ull:
+ return MakeScopedAtomic(Intrinsic::nvvm_atomic_max_gen_i_sys);
+ case NVPTX::BI__nvvm_atom_cta_min_gen_i:
+ case NVPTX::BI__nvvm_atom_cta_min_gen_ui:
+ case NVPTX::BI__nvvm_atom_cta_min_gen_l:
+ case NVPTX::BI__nvvm_atom_cta_min_gen_ul:
+ case NVPTX::BI__nvvm_atom_cta_min_gen_ll:
+ case NVPTX::BI__nvvm_atom_cta_min_gen_ull:
+ return MakeScopedAtomic(Intrinsic::nvvm_atomic_min_gen_i_cta);
+ case NVPTX::BI__nvvm_atom_sys_min_gen_i:
+ case NVPTX::BI__nvvm_atom_sys_min_gen_ui:
+ case NVPTX::BI__nvvm_atom_sys_min_gen_l:
+ case NVPTX::BI__nvvm_atom_sys_min_gen_ul:
+ case NVPTX::BI__nvvm_atom_sys_min_gen_ll:
+ case NVPTX::BI__nvvm_atom_sys_min_gen_ull:
+ return MakeScopedAtomic(Intrinsic::nvvm_atomic_min_gen_i_sys);
+ case NVPTX::BI__nvvm_atom_cta_inc_gen_ui:
+ return MakeScopedAtomic(Intrinsic::nvvm_atomic_inc_gen_i_cta);
+ case NVPTX::BI__nvvm_atom_cta_dec_gen_ui:
+ return MakeScopedAtomic(Intrinsic::nvvm_atomic_dec_gen_i_cta);
+ case NVPTX::BI__nvvm_atom_sys_inc_gen_ui:
+ return MakeScopedAtomic(Intrinsic::nvvm_atomic_inc_gen_i_sys);
+ case NVPTX::BI__nvvm_atom_sys_dec_gen_ui:
+ return MakeScopedAtomic(Intrinsic::nvvm_atomic_dec_gen_i_sys);
+ case NVPTX::BI__nvvm_atom_cta_and_gen_i:
+ case NVPTX::BI__nvvm_atom_cta_and_gen_l:
+ case NVPTX::BI__nvvm_atom_cta_and_gen_ll:
+ return MakeScopedAtomic(Intrinsic::nvvm_atomic_and_gen_i_cta);
+ case NVPTX::BI__nvvm_atom_sys_and_gen_i:
+ case NVPTX::BI__nvvm_atom_sys_and_gen_l:
+ case NVPTX::BI__nvvm_atom_sys_and_gen_ll:
+ return MakeScopedAtomic(Intrinsic::nvvm_atomic_and_gen_i_sys);
+ case NVPTX::BI__nvvm_atom_cta_or_gen_i:
+ case NVPTX::BI__nvvm_atom_cta_or_gen_l:
+ case NVPTX::BI__nvvm_atom_cta_or_gen_ll:
+ return MakeScopedAtomic(Intrinsic::nvvm_atomic_or_gen_i_cta);
+ case NVPTX::BI__nvvm_atom_sys_or_gen_i:
+ case NVPTX::BI__nvvm_atom_sys_or_gen_l:
+ case NVPTX::BI__nvvm_atom_sys_or_gen_ll:
+ return MakeScopedAtomic(Intrinsic::nvvm_atomic_or_gen_i_sys);
+ case NVPTX::BI__nvvm_atom_cta_xor_gen_i:
+ case NVPTX::BI__nvvm_atom_cta_xor_gen_l:
+ case NVPTX::BI__nvvm_atom_cta_xor_gen_ll:
+ return MakeScopedAtomic(Intrinsic::nvvm_atomic_xor_gen_i_cta);
+ case NVPTX::BI__nvvm_atom_sys_xor_gen_i:
+ case NVPTX::BI__nvvm_atom_sys_xor_gen_l:
+ case NVPTX::BI__nvvm_atom_sys_xor_gen_ll:
+ return MakeScopedAtomic(Intrinsic::nvvm_atomic_xor_gen_i_sys);
+ case NVPTX::BI__nvvm_atom_cta_cas_gen_i:
+ case NVPTX::BI__nvvm_atom_cta_cas_gen_l:
+ case NVPTX::BI__nvvm_atom_cta_cas_gen_ll: {
+ Value *Ptr = EmitScalarExpr(E->getArg(0));
+ return Builder.CreateCall(
+ CGM.getIntrinsic(
+ Intrinsic::nvvm_atomic_cas_gen_i_cta,
+ {Ptr->getType()->getPointerElementType(), Ptr->getType()}),
+ {Ptr, EmitScalarExpr(E->getArg(1)), EmitScalarExpr(E->getArg(2))});
+ }
+ case NVPTX::BI__nvvm_atom_sys_cas_gen_i:
+ case NVPTX::BI__nvvm_atom_sys_cas_gen_l:
+ case NVPTX::BI__nvvm_atom_sys_cas_gen_ll: {
+ Value *Ptr = EmitScalarExpr(E->getArg(0));
+ return Builder.CreateCall(
+ CGM.getIntrinsic(
+ Intrinsic::nvvm_atomic_cas_gen_i_sys,
+ {Ptr->getType()->getPointerElementType(), Ptr->getType()}),
+ {Ptr, EmitScalarExpr(E->getArg(1)), EmitScalarExpr(E->getArg(2))});
+ }
default:
return nullptr;
}