AMDGPU: Decompose all values to 32-bit pieces for calling conventions
This is the more natural lowering, and presents more opportunities to
reduce 64-bit ops to 32-bit.
This should also help avoid issues graphics shaders have had with
64-bit values, and simplify argument lowering in globalisel.
llvm-svn: 366578
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-amdgpu_vs.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-amdgpu_vs.ll
index 44b06b2..efeec0d 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-amdgpu_vs.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-amdgpu_vs.ll
@@ -1,3 +1,4 @@
+; XFAIL: *
; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -stop-after=irtranslator -global-isel %s -o - | FileCheck %s
diff --git a/llvm/test/CodeGen/AMDGPU/call-argument-types.ll b/llvm/test/CodeGen/AMDGPU/call-argument-types.ll
index 0e07a34..bb09c22 100644
--- a/llvm/test/CodeGen/AMDGPU/call-argument-types.ll
+++ b/llvm/test/CodeGen/AMDGPU/call-argument-types.ll
@@ -250,13 +250,11 @@
}
; GCN-LABEL: {{^}}test_call_external_void_func_i64_imm:
-; GCN-DAG: s_movk_i32 [[K0:s[0-9]+]], 0x7b{{$}}
-; GCN-DAG: s_mov_b32 [[K1:s[0-9]+]], 0{{$}}
-; GCN-DAG: v_mov_b32_e32 v0, [[K0]]
+; GCN-DAG: v_mov_b32_e32 v0, 0x7b{{$}}
+; GCN-DAG: v_mov_b32_e32 v1, 0{{$}}
; GCN-DAG: s_getpc_b64 s{{\[}}[[PC_LO:[0-9]+]]:[[PC_HI:[0-9]+]]{{\]}}
; GCN-DAG: s_add_u32 s[[PC_LO]], s[[PC_LO]], external_void_func_i64@rel32@lo+4
; GCN-DAG: s_addc_u32 s[[PC_HI]], s[[PC_HI]], external_void_func_i64@rel32@hi+4
-; GCN-DAG: v_mov_b32_e32 v1, [[K1]]
; GCN: s_swappc_b64 s[30:31], s{{\[}}[[PC_LO]]:[[PC_HI]]{{\]}}
; GCN-NEXT: s_endpgm
define amdgpu_kernel void @test_call_external_void_func_i64_imm() #0 {
@@ -756,8 +754,8 @@
}
; GCN-LABEL: {{^}}stack_passed_arg_alignment_v32i32_f64:
-; GCN: buffer_store_dword v{{[0-9]+}}, off, s{{\[[0-9]+:[0-9]+\]}}, s32 offset:4
; GCN: buffer_store_dword v{{[0-9]+}}, off, s{{\[[0-9]+:[0-9]+\]}}, s32{{$}}
+; GCN: buffer_store_dword v{{[0-9]+}}, off, s{{\[[0-9]+:[0-9]+\]}}, s32 offset:4
; GCN: s_swappc_b64
define amdgpu_kernel void @stack_passed_arg_alignment_v32i32_f64(<32 x i32> %val, double %tmp) #0 {
entry:
@@ -791,11 +789,11 @@
; GCN-NOT: s32
; GCN: buffer_store_dword v32, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
; GCN: buffer_store_dword v33, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
-; GCN: buffer_load_dword v32, off, s[0:3], s32{{$}}
-; GCN: buffer_load_dword v33, off, s[0:3], s32 offset:4
+; GCN: buffer_load_dword v32, off, s[0:3], s32 offset:4
+; GCN: buffer_load_dword v33, off, s[0:3], s32{{$}}
; GCN: s_getpc_b64
-; GCN: buffer_store_dword v32, off, s[0:3], s32{{$}}
-; GCN: buffer_store_dword v33, off, s[0:3], s32 offset:4
+; GCN: buffer_store_dword v33, off, s[0:3], s32{{$}}
+; GCN: buffer_store_dword v32, off, s[0:3], s32 offset:4
; GCN: buffer_load_dword v33, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
; GCN: buffer_load_dword v32, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
; GCN-NOT: s32
diff --git a/llvm/test/CodeGen/AMDGPU/debug-value2.ll b/llvm/test/CodeGen/AMDGPU/debug-value2.ll
index 3a13a33..18f5253 100644
--- a/llvm/test/CodeGen/AMDGPU/debug-value2.ll
+++ b/llvm/test/CodeGen/AMDGPU/debug-value2.ll
@@ -10,9 +10,13 @@
define <4 x float> @Scene_transformT(i32 %subshapeIdx, <4 x float> %v, float %time, i8 addrspace(1)* %gScene, i32 addrspace(1)* %gSceneOffsets) local_unnamed_addr !dbg !110 {
entry:
-; CHECK: ;DEBUG_VALUE: Scene_transformT:gScene <- [DW_OP_constu 1, DW_OP_swap, DW_OP_xderef] $vgpr6_vgpr7
+ ; CHECK: v_mov_b32_e32 v[[COPIED_ARG_PIECE:[0-9]+]], v9
+
+ ; CHECK: ;DEBUG_VALUE: Scene_transformT:gScene <- [DW_OP_constu 1, DW_OP_swap, DW_OP_xderef, DW_OP_LLVM_fragment 0 32] $vgpr6
+ ; CHECK: ;DEBUG_VALUE: Scene_transformT:gScene <- [DW_OP_constu 1, DW_OP_swap, DW_OP_xderef, DW_OP_LLVM_fragment 32 32] $vgpr7
call void @llvm.dbg.value(metadata i8 addrspace(1)* %gScene, metadata !120, metadata !DIExpression(DW_OP_constu, 1, DW_OP_swap, DW_OP_xderef)), !dbg !154
-; CHECK: ;DEBUG_VALUE: Scene_transformT:gSceneOffsets <- [DW_OP_constu 1, DW_OP_swap, DW_OP_xderef] $vgpr8_vgpr9
+ ; CHECK: ;DEBUG_VALUE: Scene_transformT:gSceneOffsets <- [DW_OP_constu 1, DW_OP_swap, DW_OP_xderef, DW_OP_LLVM_fragment 0 32] $vgpr8
+ ; CHECK: ;DEBUG_VALUE: Scene_transformT:gSceneOffsets <- [DW_OP_constu 1, DW_OP_swap, DW_OP_xderef, DW_OP_LLVM_fragment 32 32] $vgpr[[COPIED_ARG_PIECE]]
call void @llvm.dbg.value(metadata i32 addrspace(1)* %gSceneOffsets, metadata !121, metadata !DIExpression(DW_OP_constu, 1, DW_OP_swap, DW_OP_xderef)), !dbg !155
%call = tail call %struct.ShapeData addrspace(1)* @Scene_getSubShapeData(i32 %subshapeIdx, i8 addrspace(1)* %gScene, i32 addrspace(1)* %gSceneOffsets)
%m_linearMotion = getelementptr inbounds %struct.ShapeData, %struct.ShapeData addrspace(1)* %call, i64 0, i32 2
diff --git a/llvm/test/CodeGen/AMDGPU/implicit-def-muse.ll b/llvm/test/CodeGen/AMDGPU/implicit-def-muse.ll
index fb540c3..4e0b342 100644
--- a/llvm/test/CodeGen/AMDGPU/implicit-def-muse.ll
+++ b/llvm/test/CodeGen/AMDGPU/implicit-def-muse.ll
@@ -1,10 +1,11 @@
; RUN: llc -march=amdgcn -stop-after=amdgpu-isel -verify-machineinstrs -o - %s | FileCheck %s
; CHECK-LABEL: vcopy_i1_undef
-; CHECK: IMPLICIT_DEF
+; CHECK: [[IMPDEF0:%[0-9]+]]:vreg_1 = IMPLICIT_DEF
; CHECK-NOT: COPY
-; CHECK: IMPLICIT_DEF
-; CHECK-NOT: COPY
+; CHECK: [[IMPDEF1:%[0-9]+]]:vreg_1 = IMPLICIT_DEF
+; CHECK-NOT: COPY [[IMPDEF0]]
+; CHECK-NOT: COPY [[IMPDEF1]]
; CHECK: .false:
define <2 x float> @vcopy_i1_undef(<2 x float> addrspace(1)* %p) {
entry:
diff --git a/llvm/test/CodeGen/AMDGPU/reduce-build-vec-ext-to-ext-build-vec.ll b/llvm/test/CodeGen/AMDGPU/reduce-build-vec-ext-to-ext-build-vec.ll
index 79b817a..02eb400 100644
--- a/llvm/test/CodeGen/AMDGPU/reduce-build-vec-ext-to-ext-build-vec.ll
+++ b/llvm/test/CodeGen/AMDGPU/reduce-build-vec-ext-to-ext-build-vec.ll
@@ -8,9 +8,9 @@
; GCN-LABEL: {{^}}v2i16_to_i64:
; GFX9: s_waitcnt
-; GFX9-NEXT: v_pk_add_u16 v0, v0, v1
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX9-NEXT: v_pk_add_u16 v1, v0, v1
+; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v1
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_setpc_b64
define i64 @v2i16_to_i64(<2 x i16> %x, <2 x i16> %y) {
%x.add = add <2 x i16> %x, %y
diff --git a/llvm/test/CodeGen/AMDGPU/shift-i128.ll b/llvm/test/CodeGen/AMDGPU/shift-i128.ll
index 9fb3ede..2f3a3e5 100644
--- a/llvm/test/CodeGen/AMDGPU/shift-i128.ll
+++ b/llvm/test/CodeGen/AMDGPU/shift-i128.ll
@@ -15,12 +15,12 @@
; GCN-NEXT: v_lshl_b64 v[5:6], v[0:1], v5
; GCN-NEXT: v_cmp_gt_u32_e32 vcc, 64, v4
; GCN-NEXT: v_lshl_b64 v[0:1], v[0:1], v4
-; GCN-NEXT: v_cndmask_b32_e32 v6, v6, v8, vcc
; GCN-NEXT: v_cndmask_b32_e32 v5, v5, v7, vcc
-; GCN-NEXT: v_cndmask_b32_e64 v3, v6, v3, s[4:5]
; GCN-NEXT: v_cndmask_b32_e64 v2, v5, v2, s[4:5]
-; GCN-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
+; GCN-NEXT: v_cndmask_b32_e32 v5, v6, v8, vcc
+; GCN-NEXT: v_cndmask_b32_e64 v3, v5, v3, s[4:5]
; GCN-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
+; GCN-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
; GCN-NEXT: s_setpc_b64 s[30:31]
%shl = shl i128 %lhs, %rhs
ret i128 %shl
@@ -40,12 +40,12 @@
; GCN-NEXT: v_lshr_b64 v[5:6], v[2:3], v5
; GCN-NEXT: v_cmp_gt_u32_e32 vcc, 64, v4
; GCN-NEXT: v_lshr_b64 v[2:3], v[2:3], v4
-; GCN-NEXT: v_cndmask_b32_e32 v6, v6, v8, vcc
; GCN-NEXT: v_cndmask_b32_e32 v5, v5, v7, vcc
-; GCN-NEXT: v_cndmask_b32_e64 v1, v6, v1, s[4:5]
; GCN-NEXT: v_cndmask_b32_e64 v0, v5, v0, s[4:5]
-; GCN-NEXT: v_cndmask_b32_e32 v3, 0, v3, vcc
+; GCN-NEXT: v_cndmask_b32_e32 v5, v6, v8, vcc
+; GCN-NEXT: v_cndmask_b32_e64 v1, v5, v1, s[4:5]
; GCN-NEXT: v_cndmask_b32_e32 v2, 0, v2, vcc
+; GCN-NEXT: v_cndmask_b32_e32 v3, 0, v3, vcc
; GCN-NEXT: s_setpc_b64 s[30:31]
%shl = lshr i128 %lhs, %rhs
@@ -56,25 +56,23 @@
; GCN-LABEL: v_ashr_i128_vv:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_sub_i32_e32 v9, vcc, 64, v4
-; GCN-NEXT: v_lshr_b64 v[7:8], v[0:1], v4
-; GCN-NEXT: v_lshl_b64 v[9:10], v[2:3], v9
-; GCN-NEXT: v_ashrrev_i32_e32 v11, 31, v3
-; GCN-NEXT: v_or_b32_e32 v8, v8, v10
-; GCN-NEXT: v_subrev_i32_e32 v10, vcc, 64, v4
-; GCN-NEXT: v_ashr_i64 v[5:6], v[2:3], v4
-; GCN-NEXT: v_ashr_i64 v[2:3], v[2:3], v10
-; GCN-NEXT: v_cmp_gt_u32_e64 s[4:5], 64, v4
-; GCN-NEXT: v_or_b32_e32 v7, v7, v9
-; GCN-NEXT: v_cndmask_b32_e64 v5, v11, v5, s[4:5]
-; GCN-NEXT: v_cndmask_b32_e64 v3, v3, v8, s[4:5]
-; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v4
-; GCN-NEXT: v_cndmask_b32_e64 v2, v2, v7, s[4:5]
-; GCN-NEXT: v_cndmask_b32_e64 v6, v11, v6, s[4:5]
-; GCN-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
-; GCN-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc
-; GCN-NEXT: v_mov_b32_e32 v2, v5
-; GCN-NEXT: v_mov_b32_e32 v3, v6
+; GCN-NEXT: v_sub_i32_e32 v7, vcc, 64, v4
+; GCN-NEXT: v_lshr_b64 v[5:6], v[0:1], v4
+; GCN-NEXT: v_lshl_b64 v[7:8], v[2:3], v7
+; GCN-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v4
+; GCN-NEXT: v_or_b32_e32 v7, v5, v7
+; GCN-NEXT: v_subrev_i32_e32 v5, vcc, 64, v4
+; GCN-NEXT: v_or_b32_e32 v8, v6, v8
+; GCN-NEXT: v_ashr_i64 v[5:6], v[2:3], v5
+; GCN-NEXT: v_cmp_gt_u32_e32 vcc, 64, v4
+; GCN-NEXT: v_cndmask_b32_e32 v5, v5, v7, vcc
+; GCN-NEXT: v_cndmask_b32_e64 v0, v5, v0, s[4:5]
+; GCN-NEXT: v_cndmask_b32_e32 v5, v6, v8, vcc
+; GCN-NEXT: v_cndmask_b32_e64 v1, v5, v1, s[4:5]
+; GCN-NEXT: v_ashr_i64 v[4:5], v[2:3], v4
+; GCN-NEXT: v_ashrrev_i32_e32 v3, 31, v3
+; GCN-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GCN-NEXT: v_cndmask_b32_e32 v3, v3, v5, vcc
; GCN-NEXT: s_setpc_b64 s[30:31]
%shl = ashr i128 %lhs, %rhs
ret i128 %shl
@@ -85,10 +83,13 @@
; GCN-LABEL: v_shl_i128_vk:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_lshl_b64 v[2:3], v[2:3], 17
; GCN-NEXT: v_lshrrev_b32_e32 v4, 15, v1
-; GCN-NEXT: v_lshl_b64 v[0:1], v[0:1], 17
-; GCN-NEXT: v_or_b32_e32 v2, v2, v4
+; GCN-NEXT: v_lshlrev_b32_e32 v5, 17, v2
+; GCN-NEXT: v_or_b32_e32 v4, v5, v4
+; GCN-NEXT: v_alignbit_b32 v1, v1, v0, 15
+; GCN-NEXT: v_alignbit_b32 v3, v3, v2, 15
+; GCN-NEXT: v_lshlrev_b32_e32 v0, 17, v0
+; GCN-NEXT: v_mov_b32_e32 v2, v4
; GCN-NEXT: s_setpc_b64 s[30:31]
%shl = shl i128 %lhs, 17
ret i128 %shl
@@ -98,7 +99,8 @@
; GCN-LABEL: v_lshr_i128_vk:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_lshr_b64 v[0:1], v[2:3], 1
+; GCN-NEXT: v_alignbit_b32 v0, v3, v2, 1
+; GCN-NEXT: v_lshrrev_b32_e32 v1, 1, v3
; GCN-NEXT: v_mov_b32_e32 v2, 0
; GCN-NEXT: v_mov_b32_e32 v3, 0
; GCN-NEXT: s_setpc_b64 s[30:31]
@@ -110,12 +112,13 @@
; GCN-LABEL: v_ashr_i128_vk:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_lshl_b64 v[4:5], v[2:3], 31
-; GCN-NEXT: v_lshrrev_b32_e32 v0, 1, v1
-; GCN-NEXT: v_or_b32_e32 v4, v0, v4
-; GCN-NEXT: v_mov_b32_e32 v0, v4
-; GCN-NEXT: v_ashr_i64 v[2:3], v[2:3], 33
-; GCN-NEXT: v_mov_b32_e32 v1, v5
+; GCN-NEXT: v_ashr_i64 v[4:5], v[2:3], 33
+; GCN-NEXT: v_lshlrev_b32_e32 v0, 31, v2
+; GCN-NEXT: v_lshrrev_b32_e32 v1, 1, v1
+; GCN-NEXT: v_or_b32_e32 v0, v1, v0
+; GCN-NEXT: v_alignbit_b32 v1, v3, v2, 1
+; GCN-NEXT: v_mov_b32_e32 v2, v4
+; GCN-NEXT: v_mov_b32_e32 v3, v5
; GCN-NEXT: s_setpc_b64 s[30:31]
%shl = ashr i128 %lhs, 33
ret i128 %shl
@@ -126,18 +129,18 @@
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT: v_sub_i32_e32 v1, vcc, 64, v0
-; GCN-NEXT: v_subrev_i32_e32 v3, vcc, 64, v0
-; GCN-NEXT: v_lshr_b64 v[1:2], 17, v1
-; GCN-NEXT: v_lshl_b64 v[4:5], 17, v3
+; GCN-NEXT: v_lshr_b64 v[2:3], 17, v1
+; GCN-NEXT: v_subrev_i32_e32 v1, vcc, 64, v0
+; GCN-NEXT: v_lshl_b64 v[4:5], 17, v1
; GCN-NEXT: v_cmp_gt_u32_e32 vcc, 64, v0
-; GCN-NEXT: v_cndmask_b32_e32 v2, v5, v2, vcc
+; GCN-NEXT: v_cndmask_b32_e32 v1, v4, v2, vcc
; GCN-NEXT: v_cmp_ne_u32_e64 s[4:5], 0, v0
-; GCN-NEXT: v_cndmask_b32_e64 v3, 0, v2, s[4:5]
-; GCN-NEXT: v_cndmask_b32_e32 v2, v4, v1, vcc
+; GCN-NEXT: v_cndmask_b32_e64 v2, 0, v1, s[4:5]
; GCN-NEXT: v_lshl_b64 v[0:1], 17, v0
-; GCN-NEXT: v_cndmask_b32_e64 v2, 0, v2, s[4:5]
-; GCN-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
+; GCN-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
+; GCN-NEXT: v_cndmask_b32_e64 v3, 0, v3, s[4:5]
; GCN-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
+; GCN-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
; GCN-NEXT: s_setpc_b64 s[30:31]
%shl = shl i128 17, %rhs
ret i128 %shl
@@ -147,17 +150,17 @@
; GCN-LABEL: v_lshr_i128_kv:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: s_mov_b32 s7, 0
-; GCN-NEXT: s_movk_i32 s6, 0x41
-; GCN-NEXT: v_lshr_b64 v[2:3], s[6:7], v0
+; GCN-NEXT: s_movk_i32 s4, 0x41
+; GCN-NEXT: s_mov_b32 s5, 0
+; GCN-NEXT: v_lshr_b64 v[1:2], s[4:5], v0
; GCN-NEXT: v_cmp_gt_u32_e32 vcc, 64, v0
+; GCN-NEXT: v_mov_b32_e32 v3, s4
; GCN-NEXT: v_cmp_ne_u32_e64 s[4:5], 0, v0
-; GCN-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
-; GCN-NEXT: v_mov_b32_e32 v2, s6
-; GCN-NEXT: v_cndmask_b32_e32 v1, 0, v3, vcc
-; GCN-NEXT: v_cndmask_b32_e64 v0, v2, v0, s[4:5]
+; GCN-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
+; GCN-NEXT: s_and_b64 vcc, s[4:5], vcc
+; GCN-NEXT: v_cndmask_b32_e64 v0, v3, v1, s[4:5]
+; GCN-NEXT: v_cndmask_b32_e32 v1, 0, v2, vcc
; GCN-NEXT: v_mov_b32_e32 v2, 0
-; GCN-NEXT: v_cndmask_b32_e64 v1, 0, v1, s[4:5]
; GCN-NEXT: v_mov_b32_e32 v3, 0
; GCN-NEXT: s_setpc_b64 s[30:31]
%shl = lshr i128 65, %rhs
@@ -168,14 +171,14 @@
; GCN-LABEL: v_ashr_i128_kv:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_lshr_b64 v[2:3], 33, v0
+; GCN-NEXT: v_lshr_b64 v[1:2], 33, v0
; GCN-NEXT: v_cmp_gt_u32_e32 vcc, 64, v0
; GCN-NEXT: v_cmp_ne_u32_e64 s[4:5], 0, v0
-; GCN-NEXT: v_cndmask_b32_e32 v1, 0, v3, vcc
-; GCN-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc
+; GCN-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
+; GCN-NEXT: s_and_b64 vcc, s[4:5], vcc
+; GCN-NEXT: v_cndmask_b32_e64 v0, 33, v1, s[4:5]
+; GCN-NEXT: v_cndmask_b32_e32 v1, 0, v2, vcc
; GCN-NEXT: v_mov_b32_e32 v2, 0
-; GCN-NEXT: v_cndmask_b32_e64 v1, 0, v1, s[4:5]
-; GCN-NEXT: v_cndmask_b32_e64 v0, 33, v0, s[4:5]
; GCN-NEXT: v_mov_b32_e32 v3, 0
; GCN-NEXT: s_setpc_b64 s[30:31]
%shl = ashr i128 33, %rhs
@@ -312,33 +315,33 @@
; GCN-NEXT: v_lshl_b64 v[16:17], v[0:1], v9
; GCN-NEXT: s_and_b64 s[4:5], s[6:7], s[4:5]
; GCN-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[10:11]
-; GCN-NEXT: v_cndmask_b32_e64 v9, v17, v19, s[4:5]
-; GCN-NEXT: v_cndmask_b32_e32 v3, v9, v3, vcc
; GCN-NEXT: v_cndmask_b32_e64 v9, v16, v18, s[4:5]
+; GCN-NEXT: v_sub_i32_e64 v16, s[6:7], 64, v12
+; GCN-NEXT: v_cndmask_b32_e64 v11, v17, v19, s[4:5]
; GCN-NEXT: v_cndmask_b32_e32 v2, v9, v2, vcc
-; GCN-NEXT: v_sub_i32_e32 v11, vcc, 64, v12
; GCN-NEXT: v_lshl_b64 v[9:10], v[6:7], v12
-; GCN-NEXT: v_lshr_b64 v[16:17], v[4:5], v11
-; GCN-NEXT: v_cmp_gt_u64_e64 s[6:7], 64, v[12:13]
-; GCN-NEXT: v_or_b32_e32 v16, v9, v16
+; GCN-NEXT: v_lshr_b64 v[16:17], v[4:5], v16
; GCN-NEXT: v_cmp_eq_u64_e64 s[8:9], 0, v[14:15]
+; GCN-NEXT: v_cndmask_b32_e32 v3, v11, v3, vcc
+; GCN-NEXT: v_or_b32_e32 v16, v9, v16
+; GCN-NEXT: v_cmp_gt_u64_e64 s[6:7], 64, v[12:13]
; GCN-NEXT: v_subrev_i32_e32 v9, vcc, 64, v12
; GCN-NEXT: v_or_b32_e32 v11, v10, v17
; GCN-NEXT: v_lshl_b64 v[9:10], v[4:5], v9
+; GCN-NEXT: v_or_b32_e32 v15, v13, v15
+; GCN-NEXT: v_or_b32_e32 v14, v12, v14
; GCN-NEXT: s_and_b64 vcc, s[8:9], s[6:7]
-; GCN-NEXT: v_cndmask_b32_e32 v17, v10, v11, vcc
-; GCN-NEXT: v_or_b32_e32 v11, v13, v15
-; GCN-NEXT: v_or_b32_e32 v10, v12, v14
+; GCN-NEXT: v_cmp_eq_u64_e64 s[6:7], 0, v[14:15]
; GCN-NEXT: v_lshl_b64 v[0:1], v[0:1], v8
; GCN-NEXT: v_lshl_b64 v[4:5], v[4:5], v12
-; GCN-NEXT: v_cmp_eq_u64_e64 s[6:7], 0, v[10:11]
; GCN-NEXT: v_cndmask_b32_e32 v9, v9, v16, vcc
-; GCN-NEXT: v_cndmask_b32_e64 v7, v17, v7, s[6:7]
; GCN-NEXT: v_cndmask_b32_e64 v6, v9, v6, s[6:7]
-; GCN-NEXT: v_cndmask_b32_e64 v1, 0, v1, s[4:5]
+; GCN-NEXT: v_cndmask_b32_e32 v9, v10, v11, vcc
+; GCN-NEXT: v_cndmask_b32_e64 v7, v9, v7, s[6:7]
; GCN-NEXT: v_cndmask_b32_e64 v0, 0, v0, s[4:5]
-; GCN-NEXT: v_cndmask_b32_e32 v5, 0, v5, vcc
+; GCN-NEXT: v_cndmask_b32_e64 v1, 0, v1, s[4:5]
; GCN-NEXT: v_cndmask_b32_e32 v4, 0, v4, vcc
+; GCN-NEXT: v_cndmask_b32_e32 v5, 0, v5, vcc
; GCN-NEXT: s_setpc_b64 s[30:31]
%shl = shl <2 x i128> %lhs, %rhs
ret <2 x i128> %shl
@@ -361,33 +364,33 @@
; GCN-NEXT: v_lshr_b64 v[16:17], v[2:3], v9
; GCN-NEXT: s_and_b64 s[4:5], s[6:7], s[4:5]
; GCN-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[10:11]
-; GCN-NEXT: v_cndmask_b32_e64 v9, v17, v19, s[4:5]
-; GCN-NEXT: v_cndmask_b32_e32 v1, v9, v1, vcc
; GCN-NEXT: v_cndmask_b32_e64 v9, v16, v18, s[4:5]
+; GCN-NEXT: v_sub_i32_e64 v16, s[6:7], 64, v12
+; GCN-NEXT: v_cndmask_b32_e64 v11, v17, v19, s[4:5]
; GCN-NEXT: v_cndmask_b32_e32 v0, v9, v0, vcc
-; GCN-NEXT: v_sub_i32_e32 v11, vcc, 64, v12
; GCN-NEXT: v_lshr_b64 v[9:10], v[4:5], v12
-; GCN-NEXT: v_lshl_b64 v[16:17], v[6:7], v11
-; GCN-NEXT: v_cmp_gt_u64_e64 s[6:7], 64, v[12:13]
-; GCN-NEXT: v_or_b32_e32 v16, v9, v16
+; GCN-NEXT: v_lshl_b64 v[16:17], v[6:7], v16
; GCN-NEXT: v_cmp_eq_u64_e64 s[8:9], 0, v[14:15]
+; GCN-NEXT: v_cndmask_b32_e32 v1, v11, v1, vcc
+; GCN-NEXT: v_or_b32_e32 v16, v9, v16
+; GCN-NEXT: v_cmp_gt_u64_e64 s[6:7], 64, v[12:13]
; GCN-NEXT: v_subrev_i32_e32 v9, vcc, 64, v12
; GCN-NEXT: v_or_b32_e32 v11, v10, v17
; GCN-NEXT: v_lshr_b64 v[9:10], v[6:7], v9
+; GCN-NEXT: v_or_b32_e32 v15, v13, v15
+; GCN-NEXT: v_or_b32_e32 v14, v12, v14
; GCN-NEXT: s_and_b64 vcc, s[8:9], s[6:7]
-; GCN-NEXT: v_cndmask_b32_e32 v17, v10, v11, vcc
-; GCN-NEXT: v_or_b32_e32 v11, v13, v15
-; GCN-NEXT: v_or_b32_e32 v10, v12, v14
+; GCN-NEXT: v_cmp_eq_u64_e64 s[6:7], 0, v[14:15]
; GCN-NEXT: v_lshr_b64 v[2:3], v[2:3], v8
; GCN-NEXT: v_lshr_b64 v[6:7], v[6:7], v12
-; GCN-NEXT: v_cmp_eq_u64_e64 s[6:7], 0, v[10:11]
; GCN-NEXT: v_cndmask_b32_e32 v9, v9, v16, vcc
-; GCN-NEXT: v_cndmask_b32_e64 v5, v17, v5, s[6:7]
; GCN-NEXT: v_cndmask_b32_e64 v4, v9, v4, s[6:7]
-; GCN-NEXT: v_cndmask_b32_e64 v3, 0, v3, s[4:5]
+; GCN-NEXT: v_cndmask_b32_e32 v9, v10, v11, vcc
+; GCN-NEXT: v_cndmask_b32_e64 v5, v9, v5, s[6:7]
; GCN-NEXT: v_cndmask_b32_e64 v2, 0, v2, s[4:5]
-; GCN-NEXT: v_cndmask_b32_e32 v7, 0, v7, vcc
+; GCN-NEXT: v_cndmask_b32_e64 v3, 0, v3, s[4:5]
; GCN-NEXT: v_cndmask_b32_e32 v6, 0, v6, vcc
+; GCN-NEXT: v_cndmask_b32_e32 v7, 0, v7, vcc
; GCN-NEXT: s_setpc_b64 s[30:31]
%shl = lshr <2 x i128> %lhs, %rhs
ret <2 x i128> %shl
@@ -410,35 +413,35 @@
; GCN-NEXT: v_ashr_i64 v[16:17], v[2:3], v9
; GCN-NEXT: s_and_b64 s[4:5], s[6:7], s[4:5]
; GCN-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[10:11]
-; GCN-NEXT: v_cndmask_b32_e64 v9, v17, v19, s[4:5]
-; GCN-NEXT: v_cndmask_b32_e32 v1, v9, v1, vcc
; GCN-NEXT: v_cndmask_b32_e64 v9, v16, v18, s[4:5]
+; GCN-NEXT: v_sub_i32_e64 v16, s[6:7], 64, v12
+; GCN-NEXT: v_cndmask_b32_e64 v11, v17, v19, s[4:5]
; GCN-NEXT: v_cndmask_b32_e32 v0, v9, v0, vcc
-; GCN-NEXT: v_sub_i32_e32 v11, vcc, 64, v12
; GCN-NEXT: v_lshr_b64 v[9:10], v[4:5], v12
-; GCN-NEXT: v_lshl_b64 v[16:17], v[6:7], v11
-; GCN-NEXT: v_cmp_gt_u64_e64 s[6:7], 64, v[12:13]
-; GCN-NEXT: v_or_b32_e32 v16, v9, v16
+; GCN-NEXT: v_lshl_b64 v[16:17], v[6:7], v16
; GCN-NEXT: v_cmp_eq_u64_e64 s[8:9], 0, v[14:15]
+; GCN-NEXT: v_cndmask_b32_e32 v1, v11, v1, vcc
+; GCN-NEXT: v_or_b32_e32 v16, v9, v16
+; GCN-NEXT: v_cmp_gt_u64_e64 s[6:7], 64, v[12:13]
; GCN-NEXT: v_subrev_i32_e32 v9, vcc, 64, v12
; GCN-NEXT: v_or_b32_e32 v11, v10, v17
; GCN-NEXT: v_ashr_i64 v[9:10], v[6:7], v9
+; GCN-NEXT: v_or_b32_e32 v15, v13, v15
+; GCN-NEXT: v_or_b32_e32 v14, v12, v14
; GCN-NEXT: s_and_b64 vcc, s[8:9], s[6:7]
-; GCN-NEXT: v_cndmask_b32_e32 v17, v10, v11, vcc
-; GCN-NEXT: v_or_b32_e32 v11, v13, v15
-; GCN-NEXT: v_or_b32_e32 v10, v12, v14
-; GCN-NEXT: v_cmp_eq_u64_e64 s[6:7], 0, v[10:11]
+; GCN-NEXT: v_cmp_eq_u64_e64 s[6:7], 0, v[14:15]
; GCN-NEXT: v_cndmask_b32_e32 v9, v9, v16, vcc
; GCN-NEXT: v_cndmask_b32_e64 v4, v9, v4, s[6:7]
+; GCN-NEXT: v_cndmask_b32_e32 v9, v10, v11, vcc
+; GCN-NEXT: v_cndmask_b32_e64 v5, v9, v5, s[6:7]
; GCN-NEXT: v_ashr_i64 v[8:9], v[2:3], v8
-; GCN-NEXT: v_ashrrev_i32_e32 v2, 31, v3
-; GCN-NEXT: v_cndmask_b32_e64 v3, v2, v9, s[4:5]
-; GCN-NEXT: v_cndmask_b32_e64 v2, v2, v8, s[4:5]
+; GCN-NEXT: v_ashrrev_i32_e32 v3, 31, v3
+; GCN-NEXT: v_cndmask_b32_e64 v2, v3, v8, s[4:5]
+; GCN-NEXT: v_cndmask_b32_e64 v3, v3, v9, s[4:5]
; GCN-NEXT: v_ashr_i64 v[8:9], v[6:7], v12
-; GCN-NEXT: v_ashrrev_i32_e32 v6, 31, v7
-; GCN-NEXT: v_cndmask_b32_e32 v7, v6, v9, vcc
-; GCN-NEXT: v_cndmask_b32_e64 v5, v17, v5, s[6:7]
-; GCN-NEXT: v_cndmask_b32_e32 v6, v6, v8, vcc
+; GCN-NEXT: v_ashrrev_i32_e32 v7, 31, v7
+; GCN-NEXT: v_cndmask_b32_e32 v6, v7, v8, vcc
+; GCN-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc
; GCN-NEXT: s_setpc_b64 s[30:31]
%shl = ashr <2 x i128> %lhs, %rhs
ret <2 x i128> %shl
diff --git a/llvm/test/CodeGen/AMDGPU/si-scheduler.ll b/llvm/test/CodeGen/AMDGPU/si-scheduler.ll
index b76eb7c..d4087a3 100644
--- a/llvm/test/CodeGen/AMDGPU/si-scheduler.ll
+++ b/llvm/test/CodeGen/AMDGPU/si-scheduler.ll
@@ -10,8 +10,8 @@
; CHECK-LABEL: {{^}}main:
; CHECK: s_wqm
-; CHECK: s_load_dwordx4
; CHECK: s_load_dwordx8
+; CHECK: s_load_dwordx4
; CHECK: s_waitcnt lgkmcnt(0)
; CHECK: image_sample
; CHECK: s_waitcnt vmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/split-arg-dbg-value.ll b/llvm/test/CodeGen/AMDGPU/split-arg-dbg-value.ll
index a3452c6..12af3ce 100644
--- a/llvm/test/CodeGen/AMDGPU/split-arg-dbg-value.ll
+++ b/llvm/test/CodeGen/AMDGPU/split-arg-dbg-value.ll
@@ -76,9 +76,8 @@
; GCN-NEXT: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT: .Ltmp10:
-; GCN-NEXT: ;DEBUG_VALUE: split_f64_arg:arg <- [DW_OP_constu 1, DW_OP_swap, DW_OP_xderef] $vgpr0_vgpr1
-; GCN-NEXT: ;DEBUG_VALUE: split_f64_arg:arg <- [DW_OP_constu 1, DW_OP_swap, DW_OP_xderef] $vgpr0_vgpr1
-; GCN-NEXT: ;DEBUG_VALUE: split_f64_arg:arg <- [DW_OP_constu 1, DW_OP_swap, DW_OP_xderef] $vgpr0_vgpr1
+; GCN-NEXT: ;DEBUG_VALUE: split_f64_arg:arg <- [DW_OP_constu 1, DW_OP_swap, DW_OP_xderef, DW_OP_LLVM_fragment 32 32] $vgpr1
+; GCN-NEXT: ;DEBUG_VALUE: split_f64_arg:arg <- [DW_OP_constu 1, DW_OP_swap, DW_OP_xderef, DW_OP_LLVM_fragment 0 32] $vgpr0
; GCN-NEXT: .loc 0 16 5 prologue_end ; /tmp/dbg.cl:16:5
; GCN-NEXT: s_setpc_b64 s[30:31]
; GCN-NEXT: .Ltmp11:
@@ -111,9 +110,8 @@
; GCN-NEXT: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT: .Ltmp14:
-; GCN-NEXT: ;DEBUG_VALUE: split_i64_arg:arg <- [DW_OP_constu 1, DW_OP_swap, DW_OP_xderef] $vgpr0_vgpr1
-; GCN-NEXT: ;DEBUG_VALUE: split_i64_arg:arg <- [DW_OP_constu 1, DW_OP_swap, DW_OP_xderef] $vgpr0_vgpr1
-; GCN-NEXT: ;DEBUG_VALUE: split_i64_arg:arg <- [DW_OP_constu 1, DW_OP_swap, DW_OP_xderef] $vgpr0_vgpr1
+; GCN-NEXT: ;DEBUG_VALUE: split_i64_arg:arg <- [DW_OP_constu 1, DW_OP_swap, DW_OP_xderef, DW_OP_LLVM_fragment 32 32] $vgpr1
+; GCN-NEXT: ;DEBUG_VALUE: split_i64_arg:arg <- [DW_OP_constu 1, DW_OP_swap, DW_OP_xderef, DW_OP_LLVM_fragment 0 32] $vgpr0
; GCN-NEXT: .loc 0 24 5 prologue_end ; /tmp/dbg.cl:24:5
; GCN-NEXT: s_setpc_b64 s[30:31]
; GCN-NEXT: .Ltmp15:
@@ -128,9 +126,8 @@
; GCN-NEXT: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT: .Ltmp16:
-; GCN-NEXT: ;DEBUG_VALUE: split_ptr_arg:arg <- [DW_OP_constu 1, DW_OP_swap, DW_OP_xderef] $vgpr0_vgpr1
-; GCN-NEXT: ;DEBUG_VALUE: split_ptr_arg:arg <- [DW_OP_constu 1, DW_OP_swap, DW_OP_xderef] $vgpr0_vgpr1
-; GCN-NEXT: ;DEBUG_VALUE: split_ptr_arg:arg <- [DW_OP_constu 1, DW_OP_swap, DW_OP_xderef] $vgpr0_vgpr1
+; GCN-NEXT: ;DEBUG_VALUE: split_ptr_arg:arg <- [DW_OP_constu 1, DW_OP_swap, DW_OP_xderef, DW_OP_LLVM_fragment 32 32] $vgpr1
+; GCN-NEXT: ;DEBUG_VALUE: split_ptr_arg:arg <- [DW_OP_constu 1, DW_OP_swap, DW_OP_xderef, DW_OP_LLVM_fragment 0 32] $vgpr0
; GCN-NEXT: .loc 0 28 5 prologue_end ; /tmp/dbg.cl:28:5
; GCN-NEXT: s_setpc_b64 s[30:31]
; GCN-NEXT: .Ltmp17:
diff --git a/llvm/test/CodeGen/AMDGPU/vector_shuffle.packed.ll b/llvm/test/CodeGen/AMDGPU/vector_shuffle.packed.ll
index 50f06f9..145a181 100644
--- a/llvm/test/CodeGen/AMDGPU/vector_shuffle.packed.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector_shuffle.packed.ll
@@ -19,11 +19,11 @@
; GFX9-LABEL: shuffle_v4f16_234u:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_load_dwordx2 v[4:5], v[0:1], off
-; GFX9-NEXT: global_load_dwordx2 v[1:2], v[2:3], off
-; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_mov_b32_e32 v0, v5
+; GFX9-NEXT: global_load_dwordx2 v[2:3], v[2:3], off
+; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off
; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_mov_b32_e32 v0, v1
+; GFX9-NEXT: v_mov_b32_e32 v1, v2
; GFX9-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0
%val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1
@@ -96,12 +96,12 @@
; GFX9-LABEL: shuffle_v4f16_3uu7:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: global_load_dword v2, v[2:3], off offset:4
; GFX9-NEXT: global_load_dword v0, v[0:1], off offset:4
-; GFX9-NEXT: global_load_dword v1, v[2:3], off offset:4
; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v2
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0
@@ -181,10 +181,10 @@
; GFX9-LABEL: shuffle_v4f16_0145:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: global_load_dwordx2 v[2:3], v[2:3], off
; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: global_load_dwordx2 v[1:2], v[2:3], off
-; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_mov_b32_e32 v1, v2
; GFX9-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0
%val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1
@@ -196,11 +196,10 @@
; GFX9-LABEL: shuffle_v4f16_0167:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: global_load_dwordx2 v[2:3], v[2:3], off
; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: global_load_dwordx2 v[1:2], v[2:3], off
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v1, v2
+; GFX9-NEXT: v_mov_b32_e32 v1, v3
; GFX9-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0
%val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1
@@ -240,11 +239,11 @@
; GFX9-LABEL: shuffle_v4f16_2345:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_load_dwordx2 v[4:5], v[0:1], off
-; GFX9-NEXT: global_load_dwordx2 v[1:2], v[2:3], off
-; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_mov_b32_e32 v0, v5
+; GFX9-NEXT: global_load_dwordx2 v[2:3], v[2:3], off
+; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off
; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_mov_b32_e32 v0, v1
+; GFX9-NEXT: v_mov_b32_e32 v1, v2
; GFX9-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0
%val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1
@@ -256,10 +255,11 @@
; GFX9-LABEL: shuffle_v4f16_2367:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_load_dwordx2 v[4:5], v[0:1], off
-; GFX9-NEXT: global_load_dwordx2 v[0:1], v[2:3], off
+; GFX9-NEXT: global_load_dwordx2 v[2:3], v[2:3], off
+; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v0, v5
+; GFX9-NEXT: v_mov_b32_e32 v0, v1
+; GFX9-NEXT: v_mov_b32_e32 v1, v3
; GFX9-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0
%val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1
@@ -271,10 +271,11 @@
; GFX9-LABEL: shuffle_v4f16_4501:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_load_dwordx2 v[4:5], v[0:1], off
-; GFX9-NEXT: global_load_dwordx2 v[0:1], v[2:3], off
+; GFX9-NEXT: global_load_dwordx2 v[3:4], v[2:3], off
+; GFX9-NEXT: global_load_dwordx2 v[1:2], v[0:1], off
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_mov_b32_e32 v0, v3
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v1, v4
; GFX9-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0
%val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1
@@ -286,8 +287,8 @@
; GFX9-LABEL: shuffle_v4f16_4523:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off
; GFX9-NEXT: global_load_dwordx2 v[2:3], v[2:3], off
+; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v0, v2
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -328,11 +329,11 @@
; GFX9-LABEL: shuffle_v4f16_6701:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_load_dwordx2 v[4:5], v[0:1], off
-; GFX9-NEXT: global_load_dwordx2 v[0:1], v[2:3], off
+; GFX9-NEXT: global_load_dwordx2 v[2:3], v[2:3], off
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v0, v1
-; GFX9-NEXT: v_mov_b32_e32 v1, v4
+; GFX9-NEXT: global_load_dwordx2 v[1:2], v[0:1], off
+; GFX9-NEXT: v_mov_b32_e32 v0, v3
+; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0
%val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1
@@ -344,8 +345,8 @@
; GFX9-LABEL: shuffle_v4f16_6723:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off
; GFX9-NEXT: global_load_dwordx2 v[2:3], v[2:3], off
+; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v0, v3
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -387,12 +388,13 @@
; GFX9-LABEL: shuffle_v4f16_2356:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: global_load_dwordx2 v[2:3], v[2:3], off
; GFX9-NEXT: global_load_dwordx2 v[4:5], v[0:1], off
-; GFX9-NEXT: global_load_dwordx2 v[0:1], v[2:3], off
-; GFX9-NEXT: v_mov_b32_e32 v2, 0xffff
+; GFX9-NEXT: v_mov_b32_e32 v6, 0xffff
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_and_b32_sdwa v0, v6, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_lshl_or_b32 v1, v3, 16, v0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_and_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v1, v1, 16, v0
; GFX9-NEXT: v_mov_b32_e32 v0, v5
; GFX9-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0
@@ -405,12 +407,11 @@
; GFX9-LABEL: shuffle_v4f16_5623:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off
; GFX9-NEXT: global_load_dwordx2 v[2:3], v[2:3], off
-; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_mov_b32_e32 v0, 0xffff
+; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off
+; GFX9-NEXT: v_mov_b32_e32 v4, 0xffff
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_and_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_and_b32_sdwa v0, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_lshl_or_b32 v0, v3, 16, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0
@@ -463,16 +464,16 @@
; GFX9-LABEL: shuffle_v4f16_5734:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_load_dword v0, v[0:1], off offset:4
-; GFX9-NEXT: global_load_dwordx2 v[1:2], v[2:3], off
-; GFX9-NEXT: v_mov_b32_e32 v3, 0xffff
+; GFX9-NEXT: global_load_dword v4, v[0:1], off offset:4
+; GFX9-NEXT: global_load_dwordx2 v[0:1], v[2:3], off
+; GFX9-NEXT: v_mov_b32_e32 v2, 0xffff
; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_and_b32_sdwa v4, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_and_b32_sdwa v3, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v2
-; GFX9-NEXT: v_and_b32_sdwa v2, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, v2
-; GFX9-NEXT: v_lshl_or_b32 v1, v1, 16, v4
+; GFX9-NEXT: v_lshrrev_b32_e32 v4, 16, v1
+; GFX9-NEXT: v_and_b32_sdwa v2, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_lshl_or_b32 v1, v0, 16, v3
+; GFX9-NEXT: v_lshl_or_b32 v0, v4, 16, v2
; GFX9-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0
%val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1
@@ -484,12 +485,13 @@
; GFX9-LABEL: shuffle_v4i16_2356:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: global_load_dwordx2 v[2:3], v[2:3], off
; GFX9-NEXT: global_load_dwordx2 v[4:5], v[0:1], off
-; GFX9-NEXT: global_load_dwordx2 v[0:1], v[2:3], off
-; GFX9-NEXT: v_mov_b32_e32 v2, 0xffff
+; GFX9-NEXT: v_mov_b32_e32 v6, 0xffff
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_and_b32_sdwa v0, v6, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_lshl_or_b32 v1, v3, 16, v0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_and_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v1, v1, 16, v0
; GFX9-NEXT: v_mov_b32_e32 v0, v5
; GFX9-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x i16>, <4 x i16> addrspace(1)* %arg0
@@ -502,11 +504,10 @@
; GFX9-LABEL: shuffle_v4i16_0167:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: global_load_dwordx2 v[2:3], v[2:3], off
; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: global_load_dwordx2 v[1:2], v[2:3], off
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v1, v2
+; GFX9-NEXT: v_mov_b32_e32 v1, v3
; GFX9-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x i16>, <4 x i16> addrspace(1)* %arg0
%val1 = load <4 x i16>, <4 x i16> addrspace(1)* %arg1
@@ -552,13 +553,13 @@
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off
+; GFX9-NEXT: v_mov_b32_e32 v2, 0xffff
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v1, 0xffff
-; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; GFX9-NEXT: v_and_b32_e32 v3, v1, v0
-; GFX9-NEXT: v_and_b32_e32 v4, v1, v2
-; GFX9-NEXT: v_lshl_or_b32 v1, v0, 16, v3
-; GFX9-NEXT: v_lshl_or_b32 v0, v2, 16, v4
+; GFX9-NEXT: v_and_b32_e32 v1, v2, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX9-NEXT: v_lshl_or_b32 v1, v0, 16, v1
+; GFX9-NEXT: v_and_b32_e32 v0, v2, v3
+; GFX9-NEXT: v_lshl_or_b32 v0, v3, 16, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0
%val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1
@@ -650,8 +651,10 @@
; GFX9-LABEL: shuffle_v8f16_4589:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: global_load_dword v2, v[2:3], off
; GFX9-NEXT: global_load_dword v0, v[0:1], off offset:8
-; GFX9-NEXT: global_load_dword v1, v[2:3], off
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_mov_b32_e32 v1, v2
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
%val0 = load <8 x half>, <8 x half> addrspace(1)* %arg0
@@ -664,8 +667,10 @@
; GFX9-LABEL: shuffle_v8f16_10_11_2_3:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: global_load_dword v2, v[2:3], off offset:4
; GFX9-NEXT: global_load_dword v1, v[0:1], off offset:4
-; GFX9-NEXT: global_load_dword v0, v[2:3], off offset:4
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_mov_b32_e32 v0, v2
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
%val0 = load <8 x half>, <8 x half> addrspace(1)* %arg0
@@ -678,12 +683,13 @@
; GFX9-LABEL: shuffle_v8f16_13_14_2_3:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_load_dword v1, v[0:1], off offset:4
; GFX9-NEXT: global_load_dwordx4 v[2:5], v[2:3], off
-; GFX9-NEXT: v_mov_b32_e32 v0, 0xffff
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_and_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: global_load_dword v1, v[0:1], off offset:4
+; GFX9-NEXT: v_mov_b32_e32 v6, 0xffff
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_and_b32_sdwa v0, v6, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_lshl_or_b32 v0, v5, 16, v0
+; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
%val0 = load <8 x half>, <8 x half> addrspace(1)* %arg0
%val1 = load <8 x half>, <8 x half> addrspace(1)* %arg1
@@ -726,13 +732,10 @@
; GFX9-LABEL: shuffle_v6f16_452367:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v4, v3
-; GFX9-NEXT: v_mov_b32_e32 v3, v2
+; GFX9-NEXT: global_load_dword v3, v[2:3], off
; GFX9-NEXT: global_load_dwordx3 v[0:2], v[0:1], off
-; GFX9-NEXT: global_load_dword v3, v[3:4], off
-; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_mov_b32_e32 v0, v2
; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_mov_b32_e32 v0, v2
; GFX9-NEXT: v_mov_b32_e32 v2, v3
; GFX9-NEXT: s_setpc_b64 s[30:31]
%val0 = load <6 x half>, <6 x half> addrspace(1)* %arg0
@@ -800,15 +803,14 @@
; GFX9-LABEL: shuffle_v4f16_0456:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: global_load_dwordx2 v[2:3], v[2:3], off
; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: global_load_dwordx2 v[1:2], v[2:3], off
-; GFX9-NEXT: v_mov_b32_e32 v3, 0xffff
-; GFX9-NEXT: v_and_b32_e32 v0, v3, v0
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_and_b32_sdwa v3, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v3
+; GFX9-NEXT: v_mov_b32_e32 v1, 0xffff
+; GFX9-NEXT: v_and_b32_sdwa v4, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_and_b32_e32 v0, v1, v0
+; GFX9-NEXT: v_lshl_or_b32 v0, v2, 16, v0
+; GFX9-NEXT: v_lshl_or_b32 v1, v3, 16, v4
; GFX9-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0
%val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1
diff --git a/llvm/test/CodeGen/AMDGPU/wait.ll b/llvm/test/CodeGen/AMDGPU/wait.ll
index 12dc7cc..1f0a66f 100644
--- a/llvm/test/CodeGen/AMDGPU/wait.ll
+++ b/llvm/test/CodeGen/AMDGPU/wait.ll
@@ -7,10 +7,14 @@
; DEFAULT-LABEL: {{^}}main:
; DEFAULT: s_load_dwordx4
; DEFAULT: s_load_dwordx4
+; DEFAULT: s_waitcnt lgkmcnt(0)
+; DEFAULT: buffer_load_format_xyzw
+; DEFAULT: s_waitcnt vmcnt(0)
+; DEFAULT: buffer_load_format_xyzw
; DEFAULT: s_waitcnt vmcnt(0)
; DEFAULT: exp
-; DEFAULT: s_waitcnt lgkmcnt(0)
-; DEFAULT: s_endpgm
+; DEFAULT-NEXT: exp
+; DEFAULT-NEXT: s_endpgm
define amdgpu_vs void @main(<16 x i8> addrspace(4)* inreg %arg, <16 x i8> addrspace(4)* inreg %arg1, <32 x i8> addrspace(4)* inreg %arg2, <16 x i8> addrspace(4)* inreg %arg3, <16 x i8> addrspace(4)* inreg %arg4, i32 inreg %arg5, i32 %arg6, i32 %arg7, i32 %arg8, i32 %arg9, float addrspace(4)* inreg %constptr) #0 {
main_body:
%tmp = getelementptr <16 x i8>, <16 x i8> addrspace(4)* %arg3, i32 0
diff --git a/llvm/test/CodeGen/AMDGPU/wwm-reserved.ll b/llvm/test/CodeGen/AMDGPU/wwm-reserved.ll
index 3588883..1b1118e 100644
--- a/llvm/test/CodeGen/AMDGPU/wwm-reserved.ll
+++ b/llvm/test/CodeGen/AMDGPU/wwm-reserved.ll
@@ -136,8 +136,8 @@
; GFX9-O0: buffer_store_dword v1
; GFX9: s_swappc_b64
%tmp134 = call i64 @called_i64(i64 %tmp107)
-; GFX9-O0: buffer_load_dword v3
-; GFX9-O0: buffer_load_dword v4
+; GFX9-O0: buffer_load_dword v6
+; GFX9-O0: buffer_load_dword v7
%tmp136 = add i64 %tmp134, %tmp107
%tmp137 = tail call i64 @llvm.amdgcn.wwm.i64(i64 %tmp136)
%tmp138 = bitcast i64 %tmp137 to <2 x i32>
@@ -183,6 +183,6 @@
declare void @llvm.amdgcn.raw.buffer.store.i32(i32, <4 x i32>, i32, i32, i32)
declare void @llvm.amdgcn.raw.buffer.store.v2i32(<2 x i32>, <4 x i32>, i32, i32, i32)
declare void @llvm.amdgcn.raw.buffer.store.v2f32(<2 x float>, <4 x i32>, i32, i32, i32)
-declare void @llvm.amdgcn.raw.buffer.store.v4f32(<4 x float>, <4 x i32>, i32, i32, i32)
+declare void @llvm.amdgcn.raw.buffer.store.v4f32(<4 x float>, <4 x i32>, i32, i32, i32)
declare <2 x i32> @llvm.amdgcn.s.buffer.load.v2i32(<4 x i32>, i32, i32)
declare <4 x i32> @llvm.amdgcn.s.buffer.load.v4i32(<4 x i32>, i32, i32)