blob: 41a075756ee1615840b52b8d4565c140317e4167 [file] [log] [blame]
Tom Stellard45bb48e2015-06-13 03:28:10 +00001//===-- AMDGPUTargetMachine.cpp - TargetMachine for hw codegen targets-----===//
2//
Chandler Carruth2946cd72019-01-19 08:50:56 +00003// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
Tom Stellard45bb48e2015-06-13 03:28:10 +00006//
7//===----------------------------------------------------------------------===//
8//
9/// \file
Adrian Prantl5f8f34e42018-05-01 15:54:18 +000010/// The AMDGPU target machine contains all of the hardware specific
Tom Stellard45bb48e2015-06-13 03:28:10 +000011/// information needed to emit code for R600 and SI GPUs.
12//
13//===----------------------------------------------------------------------===//
14
15#include "AMDGPUTargetMachine.h"
16#include "AMDGPU.h"
Stanislav Mekhanoshin8e45acf2017-03-17 23:56:58 +000017#include "AMDGPUAliasAnalysis.h"
Matt Arsenaulteb9025d2016-06-28 17:42:09 +000018#include "AMDGPUCallLowering.h"
Tom Stellardca166212017-01-30 21:56:46 +000019#include "AMDGPUInstructionSelector.h"
20#include "AMDGPULegalizerInfo.h"
Matt Arsenault9aa45f02017-07-06 20:57:05 +000021#include "AMDGPUMacroFusion.h"
Matt Arsenaulteb9025d2016-06-28 17:42:09 +000022#include "AMDGPUTargetObjectFile.h"
Tom Stellard45bb48e2015-06-13 03:28:10 +000023#include "AMDGPUTargetTransformInfo.h"
Valery Pykhtinfd4c4102017-03-21 13:15:46 +000024#include "GCNIterativeScheduler.h"
Tom Stellard0d23ebe2016-08-29 19:42:52 +000025#include "GCNSchedStrategy.h"
Tom Stellard45bb48e2015-06-13 03:28:10 +000026#include "R600MachineScheduler.h"
Matt Arsenaultbc6d07c2019-03-14 22:54:43 +000027#include "SIMachineFunctionInfo.h"
Matt Arsenault2ffe8fd2016-08-11 19:18:50 +000028#include "SIMachineScheduler.h"
Richard Trieu8ce2ee92019-05-14 21:54:37 +000029#include "TargetInfo/AMDGPUTargetInfo.h"
Tom Stellard000c5af2016-04-14 19:09:28 +000030#include "llvm/CodeGen/GlobalISel/IRTranslator.h"
Chandler Carruth6bda14b2017-06-06 11:49:48 +000031#include "llvm/CodeGen/GlobalISel/InstructionSelect.h"
Tom Stellardca166212017-01-30 21:56:46 +000032#include "llvm/CodeGen/GlobalISel/Legalizer.h"
33#include "llvm/CodeGen/GlobalISel/RegBankSelect.h"
Matt Arsenaultbc6d07c2019-03-14 22:54:43 +000034#include "llvm/CodeGen/MIRParser/MIParser.h"
Tom Stellard45bb48e2015-06-13 03:28:10 +000035#include "llvm/CodeGen/Passes.h"
Matthias Braun31d19d42016-05-10 03:21:59 +000036#include "llvm/CodeGen/TargetPassConfig.h"
Eugene Zelenko6a9226d2016-12-12 22:23:53 +000037#include "llvm/IR/Attributes.h"
38#include "llvm/IR/Function.h"
Stanislav Mekhanoshin50ea93a2016-12-08 19:46:04 +000039#include "llvm/IR/LegacyPassManager.h"
Eugene Zelenko6a9226d2016-12-12 22:23:53 +000040#include "llvm/Pass.h"
41#include "llvm/Support/CommandLine.h"
42#include "llvm/Support/Compiler.h"
Chandler Carruth6bda14b2017-06-06 11:49:48 +000043#include "llvm/Support/TargetRegistry.h"
David Blaikie6054e652018-03-23 23:58:19 +000044#include "llvm/Target/TargetLoweringObjectFile.h"
Chandler Carruth6bda14b2017-06-06 11:49:48 +000045#include "llvm/Transforms/IPO.h"
46#include "llvm/Transforms/IPO/AlwaysInliner.h"
47#include "llvm/Transforms/IPO/PassManagerBuilder.h"
48#include "llvm/Transforms/Scalar.h"
49#include "llvm/Transforms/Scalar/GVN.h"
Sameer Sahasrabuddheb4f2d1c2018-09-25 09:39:21 +000050#include "llvm/Transforms/Utils.h"
Chandler Carruth6bda14b2017-06-06 11:49:48 +000051#include "llvm/Transforms/Vectorize.h"
Eugene Zelenko6a9226d2016-12-12 22:23:53 +000052#include <memory>
Tom Stellard45bb48e2015-06-13 03:28:10 +000053
54using namespace llvm;
55
Matt Arsenaultc5816112016-06-24 06:30:22 +000056static cl::opt<bool> EnableR600StructurizeCFG(
57 "r600-ir-structurize",
58 cl::desc("Use StructurizeCFG IR pass"),
59 cl::init(true));
60
Matt Arsenault03d85842016-06-27 20:32:13 +000061static cl::opt<bool> EnableSROA(
62 "amdgpu-sroa",
63 cl::desc("Run SROA after promote alloca pass"),
64 cl::ReallyHidden,
65 cl::init(true));
66
Matt Arsenault9f5e0ef2017-01-25 04:25:02 +000067static cl::opt<bool>
68EnableEarlyIfConversion("amdgpu-early-ifcvt", cl::Hidden,
69 cl::desc("Run early if-conversion"),
70 cl::init(false));
71
Matt Arsenault4d47ac32019-03-27 16:58:30 +000072static cl::opt<bool>
73OptExecMaskPreRA("amdgpu-opt-exec-mask-pre-ra", cl::Hidden,
74 cl::desc("Run pre-RA exec mask optimizations"),
75 cl::init(true));
76
Matt Arsenault03d85842016-06-27 20:32:13 +000077static cl::opt<bool> EnableR600IfConvert(
78 "r600-if-convert",
79 cl::desc("Use if conversion pass"),
80 cl::ReallyHidden,
81 cl::init(true));
82
Matt Arsenault908b9e22016-07-01 03:33:52 +000083// Option to disable vectorizer for tests.
84static cl::opt<bool> EnableLoadStoreVectorizer(
85 "amdgpu-load-store-vectorizer",
86 cl::desc("Enable load store vectorizer"),
Matt Arsenault0efdd062016-09-09 22:29:28 +000087 cl::init(true),
Matt Arsenault908b9e22016-07-01 03:33:52 +000088 cl::Hidden);
89
Hiroshi Inouec8e92452018-01-29 05:17:03 +000090// Option to control global loads scalarization
Alexander Timofeev18009562016-12-08 17:28:47 +000091static cl::opt<bool> ScalarizeGlobal(
92 "amdgpu-scalarize-global-loads",
93 cl::desc("Enable global load scalarization"),
Alexander Timofeev982aee62017-07-04 17:32:00 +000094 cl::init(true),
Alexander Timofeev18009562016-12-08 17:28:47 +000095 cl::Hidden);
96
Stanislav Mekhanoshina3b72792017-01-30 21:05:18 +000097// Option to run internalize pass.
98static cl::opt<bool> InternalizeSymbols(
99 "amdgpu-internalize-symbols",
100 cl::desc("Enable elimination of non-kernel functions and unused globals"),
101 cl::init(false),
102 cl::Hidden);
103
Stanislav Mekhanoshin9053f222017-03-28 18:23:24 +0000104// Option to inline all early.
105static cl::opt<bool> EarlyInlineAll(
106 "amdgpu-early-inline-all",
107 cl::desc("Inline all functions early"),
108 cl::init(false),
109 cl::Hidden);
110
Sam Koltonf60ad582017-03-21 12:51:34 +0000111static cl::opt<bool> EnableSDWAPeephole(
112 "amdgpu-sdwa-peephole",
113 cl::desc("Enable SDWA peepholer"),
Sam Kolton9fa16962017-04-06 15:03:28 +0000114 cl::init(true));
Sam Koltonf60ad582017-03-21 12:51:34 +0000115
Valery Pykhtin3d9afa22018-11-30 14:21:56 +0000116static cl::opt<bool> EnableDPPCombine(
117 "amdgpu-dpp-combine",
118 cl::desc("Enable DPP combiner"),
Valery Pykhtinded96df2019-02-11 11:15:03 +0000119 cl::init(true));
Valery Pykhtin3d9afa22018-11-30 14:21:56 +0000120
Stanislav Mekhanoshin8e45acf2017-03-17 23:56:58 +0000121// Enable address space based alias analysis
122static cl::opt<bool> EnableAMDGPUAliasAnalysis("enable-amdgpu-aa", cl::Hidden,
123 cl::desc("Enable AMDGPU Alias Analysis"),
124 cl::init(true));
125
Jan Sjodina06bfe02017-05-15 20:18:37 +0000126// Option to run late CFG structurizer
Matt Arsenaultcc852232017-10-10 20:22:07 +0000127static cl::opt<bool, true> LateCFGStructurize(
Jan Sjodina06bfe02017-05-15 20:18:37 +0000128 "amdgpu-late-structurize",
129 cl::desc("Enable late CFG structurization"),
Matt Arsenaultcc852232017-10-10 20:22:07 +0000130 cl::location(AMDGPUTargetMachine::EnableLateStructurizeCFG),
Jan Sjodina06bfe02017-05-15 20:18:37 +0000131 cl::Hidden);
132
Matt Arsenault5d567dc2019-02-28 00:40:32 +0000133static cl::opt<bool, true> EnableAMDGPUFunctionCallsOpt(
Matt Arsenaultb62a4eb2017-08-01 19:54:18 +0000134 "amdgpu-function-calls",
Matt Arsenaultb62a4eb2017-08-01 19:54:18 +0000135 cl::desc("Enable AMDGPU function call support"),
Matt Arsenaulta6801992018-07-10 14:03:41 +0000136 cl::location(AMDGPUTargetMachine::EnableFunctionCalls),
Matt Arsenault5d567dc2019-02-28 00:40:32 +0000137 cl::init(true),
Matt Arsenaulta6801992018-07-10 14:03:41 +0000138 cl::Hidden);
Matt Arsenaultb62a4eb2017-08-01 19:54:18 +0000139
Stanislav Mekhanoshin7f377942017-08-11 16:42:09 +0000140// Enable lib calls simplifications
141static cl::opt<bool> EnableLibCallSimplify(
142 "amdgpu-simplify-libcall",
Matt Arsenault2e4d3382018-05-29 19:35:46 +0000143 cl::desc("Enable amdgpu library simplifications"),
Stanislav Mekhanoshin7f377942017-08-11 16:42:09 +0000144 cl::init(true),
145 cl::Hidden);
146
Matt Arsenault8c4a3522018-06-26 19:10:00 +0000147static cl::opt<bool> EnableLowerKernelArguments(
148 "amdgpu-ir-lower-kernel-arguments",
149 cl::desc("Lower kernel argument loads in IR pass"),
150 cl::init(true),
151 cl::Hidden);
152
Stanislav Mekhanoshinc29d4912019-05-01 16:40:49 +0000153static cl::opt<bool> EnableRegReassign(
154 "amdgpu-reassign-regs",
155 cl::desc("Enable register reassign optimizations on gfx10+"),
156 cl::init(true),
157 cl::Hidden);
158
Neil Henning66416572018-10-08 15:49:19 +0000159// Enable atomic optimization
160static cl::opt<bool> EnableAtomicOptimizations(
161 "amdgpu-atomic-optimizations",
162 cl::desc("Enable atomic optimizations"),
163 cl::init(false),
164 cl::Hidden);
165
Tim Corringham4c4d2fe2018-12-10 12:06:10 +0000166// Enable Mode register optimization
167static cl::opt<bool> EnableSIModeRegisterPass(
168 "amdgpu-mode-register",
169 cl::desc("Enable mode register pass"),
170 cl::init(true),
171 cl::Hidden);
172
Stanislav Mekhanoshinc8f78f82019-04-05 20:11:32 +0000173// Option is used in lit tests to prevent deadcoding of patterns inspected.
174static cl::opt<bool>
175EnableDCEInRA("amdgpu-dce-in-ra",
176 cl::init(true), cl::Hidden,
177 cl::desc("Enable machine DCE inside regalloc"));
178
Nikita Popov3db93ac2019-04-07 17:22:16 +0000179static cl::opt<bool> EnableScalarIRPasses(
180 "amdgpu-scalar-ir-passes",
181 cl::desc("Enable scalar IR passes"),
182 cl::init(true),
183 cl::Hidden);
184
Tom Stellard4b0b2612019-06-11 03:21:13 +0000185extern "C" void LLVMInitializeAMDGPUTarget() {
Tom Stellard45bb48e2015-06-13 03:28:10 +0000186 // Register the target
Mehdi Aminif42454b2016-10-09 23:00:34 +0000187 RegisterTargetMachine<R600TargetMachine> X(getTheAMDGPUTarget());
188 RegisterTargetMachine<GCNTargetMachine> Y(getTheGCNTarget());
Matt Arsenaultb87fc222015-10-01 22:10:03 +0000189
190 PassRegistry *PR = PassRegistry::getPassRegistry();
Tom Stellarda2f57be2017-08-02 22:19:45 +0000191 initializeR600ClauseMergePassPass(*PR);
192 initializeR600ControlFlowFinalizerPass(*PR);
193 initializeR600PacketizerPass(*PR);
194 initializeR600ExpandSpecialInstrsPassPass(*PR);
195 initializeR600VectorRegMergerPass(*PR);
Tom Stellarde753c522018-04-09 16:09:13 +0000196 initializeGlobalISel(*PR);
Matt Arsenault7016f132017-08-03 22:30:46 +0000197 initializeAMDGPUDAGToDAGISelPass(*PR);
Valery Pykhtin3d9afa22018-11-30 14:21:56 +0000198 initializeGCNDPPCombinePass(*PR);
Matt Arsenault8c0ef8b2015-10-12 17:43:59 +0000199 initializeSILowerI1CopiesPass(*PR);
Matt Arsenault782c03b2015-11-03 22:30:13 +0000200 initializeSIFixSGPRCopiesPass(*PR);
Stanislav Mekhanoshin22a56f22017-01-24 17:46:17 +0000201 initializeSIFixVGPRCopiesPass(*PR);
Ron Liebermancac749a2018-11-16 01:13:34 +0000202 initializeSIFixupVectorISelPass(*PR);
Matt Arsenault8c0ef8b2015-10-12 17:43:59 +0000203 initializeSIFoldOperandsPass(*PR);
Sam Koltonf60ad582017-03-21 12:51:34 +0000204 initializeSIPeepholeSDWAPass(*PR);
Matt Arsenaultc3a01ec2016-06-09 23:18:47 +0000205 initializeSIShrinkInstructionsPass(*PR);
Stanislav Mekhanoshin37e7f952017-08-01 23:14:32 +0000206 initializeSIOptimizeExecMaskingPreRAPass(*PR);
Matt Arsenault187276f2015-10-07 00:42:53 +0000207 initializeSILoadStoreOptimizerPass(*PR);
Scott Linder11ef7982018-10-26 13:18:36 +0000208 initializeAMDGPUFixFunctionBitcastsPass(*PR);
Matt Arsenault746e0652017-06-02 18:02:42 +0000209 initializeAMDGPUAlwaysInlinePass(*PR);
Matt Arsenault39319482015-11-06 18:01:57 +0000210 initializeAMDGPUAnnotateKernelFeaturesPass(*PR);
Tom Stellarda6f24c62015-12-15 20:55:55 +0000211 initializeAMDGPUAnnotateUniformValuesPass(*PR);
Matt Arsenault7016f132017-08-03 22:30:46 +0000212 initializeAMDGPUArgumentUsageInfoPass(*PR);
Neil Henning66416572018-10-08 15:49:19 +0000213 initializeAMDGPUAtomicOptimizerPass(*PR);
Matt Arsenault8c4a3522018-06-26 19:10:00 +0000214 initializeAMDGPULowerKernelArgumentsPass(*PR);
Matt Arsenault372d7962018-05-18 21:35:00 +0000215 initializeAMDGPULowerKernelAttributesPass(*PR);
Matt Arsenault0699ef32017-02-09 22:00:42 +0000216 initializeAMDGPULowerIntrinsicsPass(*PR);
Yaxun Liude4b88d2017-10-10 19:39:48 +0000217 initializeAMDGPUOpenCLEnqueuedBlockLoweringPass(*PR);
Matt Arsenaulte0132462016-01-30 05:19:45 +0000218 initializeAMDGPUPromoteAllocaPass(*PR);
Matt Arsenault86de4862016-06-24 07:07:55 +0000219 initializeAMDGPUCodeGenPreparePass(*PR);
Stanislav Mekhanoshinad04e7a2019-06-17 17:47:28 +0000220 initializeAMDGPUPropagateAttributesEarlyPass(*PR);
221 initializeAMDGPUPropagateAttributesLatePass(*PR);
Matt Arsenaultc06574f2017-07-28 18:40:05 +0000222 initializeAMDGPURewriteOutArgumentsPass(*PR);
Stanislav Mekhanoshin50ea93a2016-12-08 19:46:04 +0000223 initializeAMDGPUUnifyMetadataPass(*PR);
Tom Stellard77a17772016-01-20 15:48:27 +0000224 initializeSIAnnotateControlFlowPass(*PR);
Kannan Narayananacb089e2017-04-12 03:25:12 +0000225 initializeSIInsertWaitcntsPass(*PR);
Tim Corringham4c4d2fe2018-12-10 12:06:10 +0000226 initializeSIModeRegisterPass(*PR);
Nicolai Haehnle213e87f2016-03-21 20:28:33 +0000227 initializeSIWholeQuadModePass(*PR);
Matt Arsenault55d49cf2016-02-12 02:16:10 +0000228 initializeSILowerControlFlowPass(*PR);
Matt Arsenault78fc9da2016-08-22 19:33:16 +0000229 initializeSIInsertSkipsPass(*PR);
Konstantin Zhuravlyove9a5a772017-07-21 21:19:23 +0000230 initializeSIMemoryLegalizerPass(*PR);
Matt Arsenaulte6740752016-09-29 01:44:16 +0000231 initializeSIOptimizeExecMaskingPass(*PR);
Neil Henning0a30f332019-04-01 15:19:52 +0000232 initializeSIPreAllocateWWMRegsPass(*PR);
Stanislav Mekhanoshin739174c2018-05-31 20:13:51 +0000233 initializeSIFormMemoryClausesPass(*PR);
Matt Arsenaultb8f8dbc2017-03-24 19:52:05 +0000234 initializeAMDGPUUnifyDivergentExitNodesPass(*PR);
Stanislav Mekhanoshin8e45acf2017-03-17 23:56:58 +0000235 initializeAMDGPUAAWrapperPassPass(*PR);
Matt Arsenault8ba740a2018-11-07 20:26:42 +0000236 initializeAMDGPUExternalAAWrapperPass(*PR);
Stanislav Mekhanoshin7f377942017-08-11 16:42:09 +0000237 initializeAMDGPUUseNativeCallsPass(*PR);
238 initializeAMDGPUSimplifyLibCallsPass(*PR);
Stanislav Mekhanoshin5670e6d2017-09-20 04:25:58 +0000239 initializeAMDGPUInlinerPass(*PR);
Stanislav Mekhanoshin3b7925f2019-05-01 16:49:31 +0000240 initializeGCNRegBankReassignPass(*PR);
Stanislav Mekhanoshinc29d4912019-05-01 16:40:49 +0000241 initializeGCNNSAReassignPass(*PR);
Tom Stellard45bb48e2015-06-13 03:28:10 +0000242}
243
Tom Stellarde135ffd2015-09-25 21:41:28 +0000244static std::unique_ptr<TargetLoweringObjectFile> createTLOF(const Triple &TT) {
Eugene Zelenko6a9226d2016-12-12 22:23:53 +0000245 return llvm::make_unique<AMDGPUTargetObjectFile>();
Tom Stellarde135ffd2015-09-25 21:41:28 +0000246}
247
Tom Stellard45bb48e2015-06-13 03:28:10 +0000248static ScheduleDAGInstrs *createR600MachineScheduler(MachineSchedContext *C) {
Eugene Zelenko6a9226d2016-12-12 22:23:53 +0000249 return new ScheduleDAGMILive(C, llvm::make_unique<R600SchedStrategy>());
Tom Stellard45bb48e2015-06-13 03:28:10 +0000250}
251
Matt Arsenault2ffe8fd2016-08-11 19:18:50 +0000252static ScheduleDAGInstrs *createSIMachineScheduler(MachineSchedContext *C) {
253 return new SIScheduleDAGMI(C);
254}
255
Tom Stellard0d23ebe2016-08-29 19:42:52 +0000256static ScheduleDAGInstrs *
257createGCNMaxOccupancyMachineScheduler(MachineSchedContext *C) {
258 ScheduleDAGMILive *DAG =
Stanislav Mekhanoshin582a5232017-02-15 17:19:50 +0000259 new GCNScheduleDAGMILive(C, make_unique<GCNMaxOccupancySchedStrategy>(C));
Matthias Braun115efcd2016-11-28 20:11:54 +0000260 DAG->addMutation(createLoadClusterDAGMutation(DAG->TII, DAG->TRI));
261 DAG->addMutation(createStoreClusterDAGMutation(DAG->TII, DAG->TRI));
Matt Arsenault9aa45f02017-07-06 20:57:05 +0000262 DAG->addMutation(createAMDGPUMacroFusionDAGMutation());
Tom Stellard0d23ebe2016-08-29 19:42:52 +0000263 return DAG;
264}
265
Valery Pykhtinfd4c4102017-03-21 13:15:46 +0000266static ScheduleDAGInstrs *
267createIterativeGCNMaxOccupancyMachineScheduler(MachineSchedContext *C) {
268 auto DAG = new GCNIterativeScheduler(C,
269 GCNIterativeScheduler::SCHEDULE_LEGACYMAXOCCUPANCY);
270 DAG->addMutation(createLoadClusterDAGMutation(DAG->TII, DAG->TRI));
271 DAG->addMutation(createStoreClusterDAGMutation(DAG->TII, DAG->TRI));
272 return DAG;
273}
274
275static ScheduleDAGInstrs *createMinRegScheduler(MachineSchedContext *C) {
276 return new GCNIterativeScheduler(C,
277 GCNIterativeScheduler::SCHEDULE_MINREGFORCED);
278}
279
Valery Pykhtinf2fe9722017-11-20 14:35:53 +0000280static ScheduleDAGInstrs *
281createIterativeILPMachineScheduler(MachineSchedContext *C) {
282 auto DAG = new GCNIterativeScheduler(C,
283 GCNIterativeScheduler::SCHEDULE_ILP);
284 DAG->addMutation(createLoadClusterDAGMutation(DAG->TII, DAG->TRI));
285 DAG->addMutation(createStoreClusterDAGMutation(DAG->TII, DAG->TRI));
286 DAG->addMutation(createAMDGPUMacroFusionDAGMutation());
287 return DAG;
288}
289
Tom Stellard45bb48e2015-06-13 03:28:10 +0000290static MachineSchedRegistry
Nicolai Haehnle02c32912016-01-13 16:10:10 +0000291R600SchedRegistry("r600", "Run R600's custom scheduler",
292 createR600MachineScheduler);
293
294static MachineSchedRegistry
295SISchedRegistry("si", "Run SI's custom scheduler",
296 createSIMachineScheduler);
Tom Stellard45bb48e2015-06-13 03:28:10 +0000297
Tom Stellard0d23ebe2016-08-29 19:42:52 +0000298static MachineSchedRegistry
299GCNMaxOccupancySchedRegistry("gcn-max-occupancy",
300 "Run GCN scheduler to maximize occupancy",
301 createGCNMaxOccupancyMachineScheduler);
302
Valery Pykhtinfd4c4102017-03-21 13:15:46 +0000303static MachineSchedRegistry
304IterativeGCNMaxOccupancySchedRegistry("gcn-max-occupancy-experimental",
305 "Run GCN scheduler to maximize occupancy (experimental)",
306 createIterativeGCNMaxOccupancyMachineScheduler);
307
308static MachineSchedRegistry
309GCNMinRegSchedRegistry("gcn-minreg",
310 "Run GCN iterative scheduler for minimal register usage (experimental)",
311 createMinRegScheduler);
312
Valery Pykhtinf2fe9722017-11-20 14:35:53 +0000313static MachineSchedRegistry
314GCNILPSchedRegistry("gcn-ilp",
315 "Run GCN iterative scheduler for ILP scheduling (experimental)",
316 createIterativeILPMachineScheduler);
317
Matt Arsenaultec30eb52016-05-31 16:57:45 +0000318static StringRef computeDataLayout(const Triple &TT) {
319 if (TT.getArch() == Triple::r600) {
320 // 32-bit pointers.
Yaxun Liucc56a8b2017-11-06 14:32:33 +0000321 return "e-p:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128"
Matt Arsenault95329f82018-03-27 19:26:40 +0000322 "-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5";
Tom Stellard45bb48e2015-06-13 03:28:10 +0000323 }
324
Matt Arsenaultec30eb52016-05-31 16:57:45 +0000325 // 32-bit private, local, and region pointers. 64-bit global, constant and
Neil Henning523dab02019-03-18 14:44:28 +0000326 // flat, non-integral buffer fat pointers.
Yaxun Liu0124b542018-02-13 18:00:25 +0000327 return "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32"
Matt Arsenaultec30eb52016-05-31 16:57:45 +0000328 "-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128"
Neil Henning523dab02019-03-18 14:44:28 +0000329 "-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
330 "-ni:7";
Tom Stellard45bb48e2015-06-13 03:28:10 +0000331}
332
Matt Arsenaultb22828f2016-01-27 02:17:49 +0000333LLVM_READNONE
334static StringRef getGPUOrDefault(const Triple &TT, StringRef GPU) {
335 if (!GPU.empty())
336 return GPU;
337
Matt Arsenaulte0c1f9e2019-03-17 21:31:35 +0000338 // Need to default to a target with flat support for HSA.
Matt Arsenaultb22828f2016-01-27 02:17:49 +0000339 if (TT.getArch() == Triple::amdgcn)
Matt Arsenaulte0c1f9e2019-03-17 21:31:35 +0000340 return TT.getOS() == Triple::AMDHSA ? "generic-hsa" : "generic";
Matt Arsenaultb22828f2016-01-27 02:17:49 +0000341
Matt Arsenault8e001942016-06-02 18:37:16 +0000342 return "r600";
Matt Arsenaultb22828f2016-01-27 02:17:49 +0000343}
344
Rafael Espindola8c34dd82016-05-18 22:04:49 +0000345static Reloc::Model getEffectiveRelocModel(Optional<Reloc::Model> RM) {
Tom Stellard418beb72016-07-13 14:23:33 +0000346 // The AMDGPU toolchain only supports generating shared objects, so we
347 // must always use PIC.
348 return Reloc::PIC_;
Rafael Espindola8c34dd82016-05-18 22:04:49 +0000349}
350
Tom Stellard45bb48e2015-06-13 03:28:10 +0000351AMDGPUTargetMachine::AMDGPUTargetMachine(const Target &T, const Triple &TT,
352 StringRef CPU, StringRef FS,
Rafael Espindola8c34dd82016-05-18 22:04:49 +0000353 TargetOptions Options,
354 Optional<Reloc::Model> RM,
Rafael Espindola79e238a2017-08-03 02:16:21 +0000355 Optional<CodeModel::Model> CM,
Tom Stellard45bb48e2015-06-13 03:28:10 +0000356 CodeGenOpt::Level OptLevel)
Matthias Braunbb8507e2017-10-12 22:57:28 +0000357 : LLVMTargetMachine(T, computeDataLayout(TT), TT, getGPUOrDefault(TT, CPU),
358 FS, Options, getEffectiveRelocModel(RM),
David Greenca29c272018-12-07 12:10:23 +0000359 getEffectiveCodeModel(CM, CodeModel::Small), OptLevel),
Rafael Espindola79e238a2017-08-03 02:16:21 +0000360 TLOF(createTLOF(getTargetTriple())) {
Tom Stellard45bb48e2015-06-13 03:28:10 +0000361 initAsmInfo();
362}
363
Vlad Tsyrklevich688e7522018-07-10 00:46:07 +0000364bool AMDGPUTargetMachine::EnableLateStructurizeCFG = false;
Matt Arsenaulta6801992018-07-10 14:03:41 +0000365bool AMDGPUTargetMachine::EnableFunctionCalls = false;
366
367AMDGPUTargetMachine::~AMDGPUTargetMachine() = default;
Vlad Tsyrklevich688e7522018-07-10 00:46:07 +0000368
Matt Arsenault59c0ffa2016-06-27 20:48:03 +0000369StringRef AMDGPUTargetMachine::getGPUName(const Function &F) const {
370 Attribute GPUAttr = F.getFnAttribute("target-cpu");
371 return GPUAttr.hasAttribute(Attribute::None) ?
372 getTargetCPU() : GPUAttr.getValueAsString();
373}
374
375StringRef AMDGPUTargetMachine::getFeatureString(const Function &F) const {
376 Attribute FSAttr = F.getFnAttribute("target-features");
377
378 return FSAttr.hasAttribute(Attribute::None) ?
379 getTargetFeatureString() :
380 FSAttr.getValueAsString();
381}
382
Matt Arsenaulte745d992017-09-19 07:40:11 +0000383/// Predicate for Internalize pass.
Benjamin Kramerf9ab3dd2017-10-31 23:21:30 +0000384static bool mustPreserveGV(const GlobalValue &GV) {
Matt Arsenaulte745d992017-09-19 07:40:11 +0000385 if (const Function *F = dyn_cast<Function>(&GV))
386 return F->isDeclaration() || AMDGPU::isEntryFunctionCC(F->getCallingConv());
387
388 return !GV.use_empty();
389}
390
Stanislav Mekhanoshin81598112017-01-26 16:49:08 +0000391void AMDGPUTargetMachine::adjustPassManager(PassManagerBuilder &Builder) {
Stanislav Mekhanoshinee2dd782017-03-17 17:13:41 +0000392 Builder.DivergentTarget = true;
393
Stanislav Mekhanoshin7f377942017-08-11 16:42:09 +0000394 bool EnableOpt = getOptLevel() > CodeGenOpt::None;
Matt Arsenaulte745d992017-09-19 07:40:11 +0000395 bool Internalize = InternalizeSymbols;
Matt Arsenault5d567dc2019-02-28 00:40:32 +0000396 bool EarlyInline = EarlyInlineAll && EnableOpt && !EnableFunctionCalls;
Stanislav Mekhanoshin7f377942017-08-11 16:42:09 +0000397 bool AMDGPUAA = EnableAMDGPUAliasAnalysis && EnableOpt;
398 bool LibCallSimplify = EnableLibCallSimplify && EnableOpt;
Stanislav Mekhanoshina27b2ca2017-03-24 18:01:14 +0000399
Matt Arsenault5d567dc2019-02-28 00:40:32 +0000400 if (EnableFunctionCalls) {
Stanislav Mekhanoshin2e3bf372017-09-20 06:34:28 +0000401 delete Builder.Inliner;
Stanislav Mekhanoshin56418202017-09-20 06:10:15 +0000402 Builder.Inliner = createAMDGPUFunctionInliningPass();
Stanislav Mekhanoshin2e3bf372017-09-20 06:34:28 +0000403 }
Stanislav Mekhanoshin5670e6d2017-09-20 04:25:58 +0000404
Stanislav Mekhanoshin81598112017-01-26 16:49:08 +0000405 Builder.addExtension(
Stanislav Mekhanoshinf6c1feb2017-01-27 16:38:10 +0000406 PassManagerBuilder::EP_ModuleOptimizerEarly,
Stanislav Mekhanoshinad04e7a2019-06-17 17:47:28 +0000407 [Internalize, EarlyInline, AMDGPUAA, this](const PassManagerBuilder &,
408 legacy::PassManagerBase &PM) {
Stanislav Mekhanoshina27b2ca2017-03-24 18:01:14 +0000409 if (AMDGPUAA) {
410 PM.add(createAMDGPUAAWrapperPass());
411 PM.add(createAMDGPUExternalAAWrapperPass());
412 }
Stanislav Mekhanoshin81598112017-01-26 16:49:08 +0000413 PM.add(createAMDGPUUnifyMetadataPass());
Stanislav Mekhanoshinad04e7a2019-06-17 17:47:28 +0000414 PM.add(createAMDGPUPropagateAttributesLatePass(this));
Stanislav Mekhanoshina3b72792017-01-30 21:05:18 +0000415 if (Internalize) {
Matt Arsenaulte745d992017-09-19 07:40:11 +0000416 PM.add(createInternalizePass(mustPreserveGV));
Stanislav Mekhanoshina3b72792017-01-30 21:05:18 +0000417 PM.add(createGlobalDCEPass());
418 }
Stanislav Mekhanoshin9053f222017-03-28 18:23:24 +0000419 if (EarlyInline)
Stanislav Mekhanoshin89653df2017-03-30 20:16:02 +0000420 PM.add(createAMDGPUAlwaysInlinePass(false));
Stanislav Mekhanoshina3b72792017-01-30 21:05:18 +0000421 });
Stanislav Mekhanoshina27b2ca2017-03-24 18:01:14 +0000422
Stanislav Mekhanoshin1d8cf2b2017-09-29 23:40:19 +0000423 const auto &Opt = Options;
Stanislav Mekhanoshina27b2ca2017-03-24 18:01:14 +0000424 Builder.addExtension(
425 PassManagerBuilder::EP_EarlyAsPossible,
Stanislav Mekhanoshinad04e7a2019-06-17 17:47:28 +0000426 [AMDGPUAA, LibCallSimplify, &Opt, this](const PassManagerBuilder &,
427 legacy::PassManagerBase &PM) {
Stanislav Mekhanoshina27b2ca2017-03-24 18:01:14 +0000428 if (AMDGPUAA) {
429 PM.add(createAMDGPUAAWrapperPass());
430 PM.add(createAMDGPUExternalAAWrapperPass());
431 }
Stanislav Mekhanoshinad04e7a2019-06-17 17:47:28 +0000432 PM.add(llvm::createAMDGPUPropagateAttributesEarlyPass(this));
Stanislav Mekhanoshin7f377942017-08-11 16:42:09 +0000433 PM.add(llvm::createAMDGPUUseNativeCallsPass());
434 if (LibCallSimplify)
Stanislav Mekhanoshin1d8cf2b2017-09-29 23:40:19 +0000435 PM.add(llvm::createAMDGPUSimplifyLibCallsPass(Opt));
Stanislav Mekhanoshina27b2ca2017-03-24 18:01:14 +0000436 });
Stanislav Mekhanoshin50c2f252017-06-19 23:17:36 +0000437
438 Builder.addExtension(
439 PassManagerBuilder::EP_CGSCCOptimizerLate,
440 [](const PassManagerBuilder &, legacy::PassManagerBase &PM) {
441 // Add infer address spaces pass to the opt pipeline after inlining
442 // but before SROA to increase SROA opportunities.
443 PM.add(createInferAddressSpacesPass());
Matt Arsenault372d7962018-05-18 21:35:00 +0000444
445 // This should run after inlining to have any chance of doing anything,
446 // and before other cleanup optimizations.
447 PM.add(createAMDGPULowerKernelAttributesPass());
Stanislav Mekhanoshin50c2f252017-06-19 23:17:36 +0000448 });
Stanislav Mekhanoshin50ea93a2016-12-08 19:46:04 +0000449}
450
Tom Stellard45bb48e2015-06-13 03:28:10 +0000451//===----------------------------------------------------------------------===//
452// R600 Target Machine (R600 -> Cayman)
453//===----------------------------------------------------------------------===//
454
455R600TargetMachine::R600TargetMachine(const Target &T, const Triple &TT,
Tom Stellard5dde1d22016-02-05 18:29:17 +0000456 StringRef CPU, StringRef FS,
Rafael Espindola8c34dd82016-05-18 22:04:49 +0000457 TargetOptions Options,
458 Optional<Reloc::Model> RM,
Rafael Espindola79e238a2017-08-03 02:16:21 +0000459 Optional<CodeModel::Model> CM,
460 CodeGenOpt::Level OL, bool JIT)
461 : AMDGPUTargetMachine(T, TT, CPU, FS, Options, RM, CM, OL) {
Matt Arsenaultad55ee52016-12-06 01:02:51 +0000462 setRequiresStructuredCFG(true);
Matt Arsenault5d567dc2019-02-28 00:40:32 +0000463
Matt Arsenault09a09ef2019-02-28 00:52:33 +0000464 // Override the default since calls aren't supported for r600.
Matt Arsenault5d567dc2019-02-28 00:40:32 +0000465 if (EnableFunctionCalls &&
466 EnableAMDGPUFunctionCallsOpt.getNumOccurrences() == 0)
467 EnableFunctionCalls = false;
Matt Arsenaultad55ee52016-12-06 01:02:51 +0000468}
Matt Arsenault59c0ffa2016-06-27 20:48:03 +0000469
470const R600Subtarget *R600TargetMachine::getSubtargetImpl(
471 const Function &F) const {
472 StringRef GPU = getGPUName(F);
473 StringRef FS = getFeatureString(F);
474
475 SmallString<128> SubtargetKey(GPU);
476 SubtargetKey.append(FS);
477
478 auto &I = SubtargetMap[SubtargetKey];
479 if (!I) {
480 // This needs to be done before we create a new subtarget since any
481 // creation will depend on the TM and the code generation flags on the
482 // function that reside in TargetOptions.
483 resetTargetOptions(F);
484 I = llvm::make_unique<R600Subtarget>(TargetTriple, GPU, FS, *this);
485 }
486
487 return I.get();
488}
Tom Stellard45bb48e2015-06-13 03:28:10 +0000489
Tom Stellardc7624312018-05-30 22:55:35 +0000490TargetTransformInfo
491R600TargetMachine::getTargetTransformInfo(const Function &F) {
492 return TargetTransformInfo(R600TTIImpl(this, F));
493}
494
Tom Stellard45bb48e2015-06-13 03:28:10 +0000495//===----------------------------------------------------------------------===//
496// GCN Target Machine (SI+)
497//===----------------------------------------------------------------------===//
498
499GCNTargetMachine::GCNTargetMachine(const Target &T, const Triple &TT,
Tom Stellard5dde1d22016-02-05 18:29:17 +0000500 StringRef CPU, StringRef FS,
Rafael Espindola8c34dd82016-05-18 22:04:49 +0000501 TargetOptions Options,
502 Optional<Reloc::Model> RM,
Rafael Espindola79e238a2017-08-03 02:16:21 +0000503 Optional<CodeModel::Model> CM,
504 CodeGenOpt::Level OL, bool JIT)
505 : AMDGPUTargetMachine(T, TT, CPU, FS, Options, RM, CM, OL) {}
Matt Arsenault59c0ffa2016-06-27 20:48:03 +0000506
Tom Stellard5bfbae52018-07-11 20:59:01 +0000507const GCNSubtarget *GCNTargetMachine::getSubtargetImpl(const Function &F) const {
Matt Arsenault59c0ffa2016-06-27 20:48:03 +0000508 StringRef GPU = getGPUName(F);
509 StringRef FS = getFeatureString(F);
510
511 SmallString<128> SubtargetKey(GPU);
512 SubtargetKey.append(FS);
513
514 auto &I = SubtargetMap[SubtargetKey];
515 if (!I) {
516 // This needs to be done before we create a new subtarget since any
517 // creation will depend on the TM and the code generation flags on the
518 // function that reside in TargetOptions.
519 resetTargetOptions(F);
Tom Stellard5bfbae52018-07-11 20:59:01 +0000520 I = llvm::make_unique<GCNSubtarget>(TargetTriple, GPU, FS, *this);
Matt Arsenault59c0ffa2016-06-27 20:48:03 +0000521 }
522
Alexander Timofeev18009562016-12-08 17:28:47 +0000523 I->setScalarizeGlobalBehavior(ScalarizeGlobal);
524
Matt Arsenault59c0ffa2016-06-27 20:48:03 +0000525 return I.get();
526}
Tom Stellard45bb48e2015-06-13 03:28:10 +0000527
Tom Stellardc7624312018-05-30 22:55:35 +0000528TargetTransformInfo
529GCNTargetMachine::getTargetTransformInfo(const Function &F) {
530 return TargetTransformInfo(GCNTTIImpl(this, F));
531}
532
Tom Stellard45bb48e2015-06-13 03:28:10 +0000533//===----------------------------------------------------------------------===//
534// AMDGPU Pass Setup
535//===----------------------------------------------------------------------===//
536
537namespace {
Tom Stellardcc7067a62016-03-03 03:53:29 +0000538
Tom Stellard45bb48e2015-06-13 03:28:10 +0000539class AMDGPUPassConfig : public TargetPassConfig {
540public:
Matthias Braunbb8507e2017-10-12 22:57:28 +0000541 AMDGPUPassConfig(LLVMTargetMachine &TM, PassManagerBase &PM)
Matt Arsenault0a109002015-09-25 17:41:20 +0000542 : TargetPassConfig(TM, PM) {
Matt Arsenault0a109002015-09-25 17:41:20 +0000543 // Exceptions and StackMaps are not supported, so these passes will never do
544 // anything.
545 disablePass(&StackMapLivenessID);
546 disablePass(&FuncletLayoutID);
547 }
Tom Stellard45bb48e2015-06-13 03:28:10 +0000548
549 AMDGPUTargetMachine &getAMDGPUTargetMachine() const {
550 return getTM<AMDGPUTargetMachine>();
551 }
552
Matthias Braun115efcd2016-11-28 20:11:54 +0000553 ScheduleDAGInstrs *
554 createMachineScheduler(MachineSchedContext *C) const override {
555 ScheduleDAGMILive *DAG = createGenericSchedLive(C);
556 DAG->addMutation(createLoadClusterDAGMutation(DAG->TII, DAG->TRI));
557 DAG->addMutation(createStoreClusterDAGMutation(DAG->TII, DAG->TRI));
558 return DAG;
559 }
560
Matt Arsenaultf42c6922016-06-15 00:11:01 +0000561 void addEarlyCSEOrGVNPass();
562 void addStraightLineScalarOptimizationPasses();
Tom Stellard45bb48e2015-06-13 03:28:10 +0000563 void addIRPasses() override;
Matt Arsenault908b9e22016-07-01 03:33:52 +0000564 void addCodeGenPrepare() override;
Matt Arsenault0a109002015-09-25 17:41:20 +0000565 bool addPreISel() override;
566 bool addInstSelector() override;
567 bool addGCPasses() override;
Amara Emersond1896802019-04-15 04:53:46 +0000568
569 std::unique_ptr<CSEConfigBase> getCSEConfig() const override;
Tom Stellard45bb48e2015-06-13 03:28:10 +0000570};
571
Amara Emersond1896802019-04-15 04:53:46 +0000572std::unique_ptr<CSEConfigBase> AMDGPUPassConfig::getCSEConfig() const {
573 return getStandardCSEConfigForOpt(TM->getOptLevel());
574}
575
Matt Arsenault6b6a2c32016-03-11 08:00:27 +0000576class R600PassConfig final : public AMDGPUPassConfig {
Tom Stellard45bb48e2015-06-13 03:28:10 +0000577public:
Matthias Braunbb8507e2017-10-12 22:57:28 +0000578 R600PassConfig(LLVMTargetMachine &TM, PassManagerBase &PM)
Eugene Zelenko6a9226d2016-12-12 22:23:53 +0000579 : AMDGPUPassConfig(TM, PM) {}
Tom Stellard45bb48e2015-06-13 03:28:10 +0000580
Matt Arsenault43e92fe2016-06-24 06:30:11 +0000581 ScheduleDAGInstrs *createMachineScheduler(
582 MachineSchedContext *C) const override {
583 return createR600MachineScheduler(C);
584 }
585
Tom Stellard45bb48e2015-06-13 03:28:10 +0000586 bool addPreISel() override;
Tom Stellard20287692017-08-08 04:57:55 +0000587 bool addInstSelector() override;
Tom Stellard45bb48e2015-06-13 03:28:10 +0000588 void addPreRegAlloc() override;
589 void addPreSched2() override;
590 void addPreEmitPass() override;
591};
592
Matt Arsenault6b6a2c32016-03-11 08:00:27 +0000593class GCNPassConfig final : public AMDGPUPassConfig {
Tom Stellard45bb48e2015-06-13 03:28:10 +0000594public:
Matthias Braunbb8507e2017-10-12 22:57:28 +0000595 GCNPassConfig(LLVMTargetMachine &TM, PassManagerBase &PM)
Matt Arsenault6ed7b9b2017-08-02 01:31:28 +0000596 : AMDGPUPassConfig(TM, PM) {
Matt Arsenaulta2025382017-08-03 23:24:05 +0000597 // It is necessary to know the register usage of the entire call graph. We
598 // allow calls without EnableAMDGPUFunctionCalls if they are marked
599 // noinline, so this is always required.
600 setRequiresCodeGenSCCOrder(true);
Matt Arsenault6ed7b9b2017-08-02 01:31:28 +0000601 }
Matt Arsenault43e92fe2016-06-24 06:30:11 +0000602
603 GCNTargetMachine &getGCNTargetMachine() const {
604 return getTM<GCNTargetMachine>();
605 }
606
607 ScheduleDAGInstrs *
Matt Arsenault03d85842016-06-27 20:32:13 +0000608 createMachineScheduler(MachineSchedContext *C) const override;
Matt Arsenault43e92fe2016-06-24 06:30:11 +0000609
Tom Stellard45bb48e2015-06-13 03:28:10 +0000610 bool addPreISel() override;
Matt Arsenault3d1c1de2016-04-14 21:58:24 +0000611 void addMachineSSAOptimization() override;
Matt Arsenault9f5e0ef2017-01-25 04:25:02 +0000612 bool addILPOpts() override;
Tom Stellard45bb48e2015-06-13 03:28:10 +0000613 bool addInstSelector() override;
Tom Stellard000c5af2016-04-14 19:09:28 +0000614 bool addIRTranslator() override;
Tim Northover33b07d62016-07-22 20:03:43 +0000615 bool addLegalizeMachineIR() override;
Tom Stellard000c5af2016-04-14 19:09:28 +0000616 bool addRegBankSelect() override;
Ahmed Bougacha6756a2c2016-07-27 14:31:55 +0000617 bool addGlobalInstructionSelect() override;
Matt Arsenaultcf55a652019-03-19 19:33:12 +0000618 void addFastRegAlloc() override;
619 void addOptimizedRegAlloc() override;
Tom Stellard45bb48e2015-06-13 03:28:10 +0000620 void addPreRegAlloc() override;
Stanislav Mekhanoshinc29d4912019-05-01 16:40:49 +0000621 bool addPreRewrite() override;
Matt Arsenaulte6740752016-09-29 01:44:16 +0000622 void addPostRegAlloc() override;
Tom Stellard45bb48e2015-06-13 03:28:10 +0000623 void addPreSched2() override;
624 void addPreEmitPass() override;
625};
626
Eugene Zelenko6a9226d2016-12-12 22:23:53 +0000627} // end anonymous namespace
Tom Stellard45bb48e2015-06-13 03:28:10 +0000628
Matt Arsenaultf42c6922016-06-15 00:11:01 +0000629void AMDGPUPassConfig::addEarlyCSEOrGVNPass() {
630 if (getOptLevel() == CodeGenOpt::Aggressive)
631 addPass(createGVNPass());
632 else
633 addPass(createEarlyCSEPass());
634}
635
636void AMDGPUPassConfig::addStraightLineScalarOptimizationPasses() {
Stanislav Mekhanoshin20d47952018-06-29 16:26:53 +0000637 addPass(createLICMPass());
Matt Arsenaultf42c6922016-06-15 00:11:01 +0000638 addPass(createSeparateConstOffsetFromGEPPass());
639 addPass(createSpeculativeExecutionPass());
640 // ReassociateGEPs exposes more opportunites for SLSR. See
641 // the example in reassociate-geps-and-slsr.ll.
642 addPass(createStraightLineStrengthReducePass());
643 // SeparateConstOffsetFromGEP and SLSR creates common expressions which GVN or
644 // EarlyCSE can reuse.
645 addEarlyCSEOrGVNPass();
646 // Run NaryReassociate after EarlyCSE/GVN to be more effective.
647 addPass(createNaryReassociatePass());
648 // NaryReassociate on GEPs creates redundant common expressions, so run
649 // EarlyCSE after it.
650 addPass(createEarlyCSEPass());
651}
652
Tom Stellard45bb48e2015-06-13 03:28:10 +0000653void AMDGPUPassConfig::addIRPasses() {
Stanislav Mekhanoshinc90347d2017-04-12 20:48:56 +0000654 const AMDGPUTargetMachine &TM = getAMDGPUTargetMachine();
655
Matt Arsenaultbde80342016-05-18 15:41:07 +0000656 // There is no reason to run these.
657 disablePass(&StackMapLivenessID);
658 disablePass(&FuncletLayoutID);
659 disablePass(&PatchableFunctionID);
660
Stanislav Mekhanoshinad04e7a2019-06-17 17:47:28 +0000661 // A call to propagate attributes pass in the backend in case opt was not run.
662 addPass(createAMDGPUPropagateAttributesEarlyPass(&TM));
663
Matt Arsenaultab411932018-10-02 03:50:56 +0000664 addPass(createAtomicExpandPass());
Scott Linder11ef7982018-10-26 13:18:36 +0000665
666 // This must occur before inlining, as the inliner will not look through
667 // bitcast calls.
668 addPass(createAMDGPUFixFunctionBitcastsPass());
669
Francis Visoiu Mistrih8b617642017-05-18 17:21:13 +0000670 addPass(createAMDGPULowerIntrinsicsPass());
Matt Arsenault0699ef32017-02-09 22:00:42 +0000671
Matt Arsenault635d4792018-10-03 02:47:25 +0000672 // Function calls are not supported, so make sure we inline everything.
673 addPass(createAMDGPUAlwaysInlinePass());
674 addPass(createAlwaysInlinerLegacyPass());
675 // We need to add the barrier noop pass, otherwise adding the function
676 // inlining pass will cause all of the PassConfigs passes to be run
677 // one function at a time, which means if we have a nodule with two
678 // functions, then we will generate code for the first function
679 // without ever running any passes on the second.
680 addPass(createBarrierNoopPass());
Matt Arsenault39319482015-11-06 18:01:57 +0000681
Matt Arsenault0c329382017-01-30 18:40:29 +0000682 if (TM.getTargetTriple().getArch() == Triple::amdgcn) {
683 // TODO: May want to move later or split into an early and late one.
684
Francis Visoiu Mistrih8b617642017-05-18 17:21:13 +0000685 addPass(createAMDGPUCodeGenPreparePass());
Matt Arsenault0c329382017-01-30 18:40:29 +0000686 }
687
Tom Stellardfd253952015-08-07 23:19:30 +0000688 // Handle uses of OpenCL image2d_t, image3d_t and sampler_t arguments.
Matt Arsenault432aaea2018-05-13 10:04:48 +0000689 if (TM.getTargetTriple().getArch() == Triple::r600)
690 addPass(createR600OpenCLImageTypeLoweringPass());
Matt Arsenault39319482015-11-06 18:01:57 +0000691
Yaxun Liude4b88d2017-10-10 19:39:48 +0000692 // Replace OpenCL enqueued block function pointers with global variables.
693 addPass(createAMDGPUOpenCLEnqueuedBlockLoweringPass());
694
Matt Arsenault03d85842016-06-27 20:32:13 +0000695 if (TM.getOptLevel() > CodeGenOpt::None) {
Matt Arsenault417e0072017-02-08 06:16:04 +0000696 addPass(createInferAddressSpacesPass());
Francis Visoiu Mistrih8b617642017-05-18 17:21:13 +0000697 addPass(createAMDGPUPromoteAlloca());
Matt Arsenault03d85842016-06-27 20:32:13 +0000698
699 if (EnableSROA)
700 addPass(createSROAPass());
Matt Arsenaultf42c6922016-06-15 00:11:01 +0000701
Nikita Popov3db93ac2019-04-07 17:22:16 +0000702 if (EnableScalarIRPasses)
703 addStraightLineScalarOptimizationPasses();
Stanislav Mekhanoshin8e45acf2017-03-17 23:56:58 +0000704
705 if (EnableAMDGPUAliasAnalysis) {
706 addPass(createAMDGPUAAWrapperPass());
707 addPass(createExternalAAWrapperPass([](Pass &P, Function &,
708 AAResults &AAR) {
709 if (auto *WrapperPass = P.getAnalysisIfAvailable<AMDGPUAAWrapperPass>())
710 AAR.addAAResult(WrapperPass->getResult());
711 }));
712 }
Konstantin Zhuravlyov4658e5f2016-09-30 16:39:24 +0000713 }
Matt Arsenaultf42c6922016-06-15 00:11:01 +0000714
715 TargetPassConfig::addIRPasses();
716
717 // EarlyCSE is not always strong enough to clean up what LSR produces. For
718 // example, GVN can combine
719 //
720 // %0 = add %a, %b
721 // %1 = add %b, %a
722 //
723 // and
724 //
725 // %0 = shl nsw %a, 2
726 // %1 = shl %a, 2
727 //
728 // but EarlyCSE can do neither of them.
Nikita Popov3db93ac2019-04-07 17:22:16 +0000729 if (getOptLevel() != CodeGenOpt::None && EnableScalarIRPasses)
Matt Arsenaultf42c6922016-06-15 00:11:01 +0000730 addEarlyCSEOrGVNPass();
Tom Stellard45bb48e2015-06-13 03:28:10 +0000731}
732
Matt Arsenault908b9e22016-07-01 03:33:52 +0000733void AMDGPUPassConfig::addCodeGenPrepare() {
Aakanksha Patilc56d2af2019-03-07 00:54:04 +0000734 if (TM->getTargetTriple().getArch() == Triple::amdgcn)
735 addPass(createAMDGPUAnnotateKernelFeaturesPass());
736
Matt Arsenault8c4a3522018-06-26 19:10:00 +0000737 if (TM->getTargetTriple().getArch() == Triple::amdgcn &&
738 EnableLowerKernelArguments)
739 addPass(createAMDGPULowerKernelArgumentsPass());
740
Matt Arsenault908b9e22016-07-01 03:33:52 +0000741 TargetPassConfig::addCodeGenPrepare();
742
743 if (EnableLoadStoreVectorizer)
744 addPass(createLoadStoreVectorizerPass());
745}
746
Matt Arsenault43e92fe2016-06-24 06:30:11 +0000747bool AMDGPUPassConfig::addPreISel() {
Sameer Sahasrabuddheb4f2d1c2018-09-25 09:39:21 +0000748 addPass(createLowerSwitchPass());
Tom Stellard45bb48e2015-06-13 03:28:10 +0000749 addPass(createFlattenCFGPass());
Tom Stellard45bb48e2015-06-13 03:28:10 +0000750 return false;
751}
752
753bool AMDGPUPassConfig::addInstSelector() {
Matt Arsenault7016f132017-08-03 22:30:46 +0000754 addPass(createAMDGPUISelDag(&getAMDGPUTargetMachine(), getOptLevel()));
Tom Stellard45bb48e2015-06-13 03:28:10 +0000755 return false;
756}
757
Matt Arsenault0a109002015-09-25 17:41:20 +0000758bool AMDGPUPassConfig::addGCPasses() {
759 // Do nothing. GC is not supported.
760 return false;
761}
762
Tom Stellard45bb48e2015-06-13 03:28:10 +0000763//===----------------------------------------------------------------------===//
764// R600 Pass Setup
765//===----------------------------------------------------------------------===//
766
767bool R600PassConfig::addPreISel() {
768 AMDGPUPassConfig::addPreISel();
Matt Arsenaultc5816112016-06-24 06:30:22 +0000769
770 if (EnableR600StructurizeCFG)
Tom Stellardbc4497b2016-02-12 23:45:29 +0000771 addPass(createStructurizeCFGPass());
Tom Stellard45bb48e2015-06-13 03:28:10 +0000772 return false;
773}
774
Tom Stellard20287692017-08-08 04:57:55 +0000775bool R600PassConfig::addInstSelector() {
776 addPass(createR600ISelDag(&getAMDGPUTargetMachine(), getOptLevel()));
777 return false;
778}
779
Tom Stellard45bb48e2015-06-13 03:28:10 +0000780void R600PassConfig::addPreRegAlloc() {
Francis Visoiu Mistrih8b617642017-05-18 17:21:13 +0000781 addPass(createR600VectorRegMerger());
Tom Stellard45bb48e2015-06-13 03:28:10 +0000782}
783
784void R600PassConfig::addPreSched2() {
Tom Stellard45bb48e2015-06-13 03:28:10 +0000785 addPass(createR600EmitClauseMarkers(), false);
Matt Arsenault03d85842016-06-27 20:32:13 +0000786 if (EnableR600IfConvert)
Tom Stellard45bb48e2015-06-13 03:28:10 +0000787 addPass(&IfConverterID, false);
Francis Visoiu Mistrih8b617642017-05-18 17:21:13 +0000788 addPass(createR600ClauseMergePass(), false);
Tom Stellard45bb48e2015-06-13 03:28:10 +0000789}
790
791void R600PassConfig::addPreEmitPass() {
792 addPass(createAMDGPUCFGStructurizerPass(), false);
Francis Visoiu Mistrih8b617642017-05-18 17:21:13 +0000793 addPass(createR600ExpandSpecialInstrsPass(), false);
Tom Stellard45bb48e2015-06-13 03:28:10 +0000794 addPass(&FinalizeMachineBundlesID, false);
Francis Visoiu Mistrih8b617642017-05-18 17:21:13 +0000795 addPass(createR600Packetizer(), false);
796 addPass(createR600ControlFlowFinalizer(), false);
Tom Stellard45bb48e2015-06-13 03:28:10 +0000797}
798
799TargetPassConfig *R600TargetMachine::createPassConfig(PassManagerBase &PM) {
Matthias Braun5e394c32017-05-30 21:36:41 +0000800 return new R600PassConfig(*this, PM);
Tom Stellard45bb48e2015-06-13 03:28:10 +0000801}
802
803//===----------------------------------------------------------------------===//
804// GCN Pass Setup
805//===----------------------------------------------------------------------===//
806
Matt Arsenault03d85842016-06-27 20:32:13 +0000807ScheduleDAGInstrs *GCNPassConfig::createMachineScheduler(
808 MachineSchedContext *C) const {
Tom Stellard5bfbae52018-07-11 20:59:01 +0000809 const GCNSubtarget &ST = C->MF->getSubtarget<GCNSubtarget>();
Matt Arsenault03d85842016-06-27 20:32:13 +0000810 if (ST.enableSIScheduler())
811 return createSIMachineScheduler(C);
Tom Stellard0d23ebe2016-08-29 19:42:52 +0000812 return createGCNMaxOccupancyMachineScheduler(C);
Matt Arsenault03d85842016-06-27 20:32:13 +0000813}
814
Tom Stellard45bb48e2015-06-13 03:28:10 +0000815bool GCNPassConfig::addPreISel() {
816 AMDGPUPassConfig::addPreISel();
Matt Arsenault39319482015-11-06 18:01:57 +0000817
Neil Henning66416572018-10-08 15:49:19 +0000818 if (EnableAtomicOptimizations) {
819 addPass(createAMDGPUAtomicOptimizerPass());
820 }
821
Matt Arsenault39319482015-11-06 18:01:57 +0000822 // FIXME: We need to run a pass to propagate the attributes when calls are
823 // supported.
Matt Arsenaultb8f8dbc2017-03-24 19:52:05 +0000824
825 // Merge divergent exit nodes. StructurizeCFG won't recognize the multi-exit
826 // regions formed by them.
827 addPass(&AMDGPUUnifyDivergentExitNodesID);
Jan Sjodina06bfe02017-05-15 20:18:37 +0000828 if (!LateCFGStructurize) {
829 addPass(createStructurizeCFGPass(true)); // true -> SkipUniformRegions
830 }
Tom Stellard45bb48e2015-06-13 03:28:10 +0000831 addPass(createSinkingPass());
Tom Stellarda6f24c62015-12-15 20:55:55 +0000832 addPass(createAMDGPUAnnotateUniformValues());
Jan Sjodina06bfe02017-05-15 20:18:37 +0000833 if (!LateCFGStructurize) {
834 addPass(createSIAnnotateControlFlowPass());
835 }
Tom Stellarda6f24c62015-12-15 20:55:55 +0000836
Tom Stellard45bb48e2015-06-13 03:28:10 +0000837 return false;
838}
839
Matt Arsenault3d1c1de2016-04-14 21:58:24 +0000840void GCNPassConfig::addMachineSSAOptimization() {
841 TargetPassConfig::addMachineSSAOptimization();
842
843 // We want to fold operands after PeepholeOptimizer has run (or as part of
844 // it), because it will eliminate extra copies making it easier to fold the
845 // real source operand. We want to eliminate dead instructions after, so that
846 // we see fewer uses of the copies. We then need to clean up the dead
847 // instructions leftover after the operands are folded as well.
848 //
849 // XXX - Can we get away without running DeadMachineInstructionElim again?
850 addPass(&SIFoldOperandsID);
Valery Pykhtin3d9afa22018-11-30 14:21:56 +0000851 if (EnableDPPCombine)
852 addPass(&GCNDPPCombineID);
Matt Arsenault3d1c1de2016-04-14 21:58:24 +0000853 addPass(&DeadMachineInstructionElimID);
Tom Stellardc2ff0eb2016-08-29 19:15:22 +0000854 addPass(&SILoadStoreOptimizerID);
Sam Kolton6e795292017-04-07 10:53:12 +0000855 if (EnableSDWAPeephole) {
856 addPass(&SIPeepholeSDWAID);
Matthias Braun4a7c8e72018-01-19 06:46:10 +0000857 addPass(&EarlyMachineLICMID);
Stanislav Mekhanoshin56ea4882017-05-30 16:49:24 +0000858 addPass(&MachineCSEID);
859 addPass(&SIFoldOperandsID);
Sam Kolton6e795292017-04-07 10:53:12 +0000860 addPass(&DeadMachineInstructionElimID);
861 }
Stanislav Mekhanoshin03306602017-06-03 17:39:47 +0000862 addPass(createSIShrinkInstructionsPass());
Matt Arsenault3d1c1de2016-04-14 21:58:24 +0000863}
864
Matt Arsenault9f5e0ef2017-01-25 04:25:02 +0000865bool GCNPassConfig::addILPOpts() {
866 if (EnableEarlyIfConversion)
867 addPass(&EarlyIfConverterID);
868
869 TargetPassConfig::addILPOpts();
870 return false;
871}
872
Tom Stellard45bb48e2015-06-13 03:28:10 +0000873bool GCNPassConfig::addInstSelector() {
874 AMDGPUPassConfig::addInstSelector();
Matt Arsenault782c03b2015-11-03 22:30:13 +0000875 addPass(&SIFixSGPRCopiesID);
Nicolai Haehnle814abb52018-10-31 13:27:08 +0000876 addPass(createSILowerI1CopiesPass());
Ron Liebermancac749a2018-11-16 01:13:34 +0000877 addPass(createSIFixupVectorISelPass());
David Stuttardf77079f2019-01-14 11:55:24 +0000878 addPass(createSIAddIMGInitPass());
Tom Stellard45bb48e2015-06-13 03:28:10 +0000879 return false;
880}
881
Tom Stellard000c5af2016-04-14 19:09:28 +0000882bool GCNPassConfig::addIRTranslator() {
883 addPass(new IRTranslator());
884 return false;
885}
886
Tim Northover33b07d62016-07-22 20:03:43 +0000887bool GCNPassConfig::addLegalizeMachineIR() {
Tom Stellardca166212017-01-30 21:56:46 +0000888 addPass(new Legalizer());
Tim Northover33b07d62016-07-22 20:03:43 +0000889 return false;
890}
891
Tom Stellard000c5af2016-04-14 19:09:28 +0000892bool GCNPassConfig::addRegBankSelect() {
Tom Stellardca166212017-01-30 21:56:46 +0000893 addPass(new RegBankSelect());
Tom Stellard000c5af2016-04-14 19:09:28 +0000894 return false;
895}
Ahmed Bougacha6756a2c2016-07-27 14:31:55 +0000896
897bool GCNPassConfig::addGlobalInstructionSelect() {
Tom Stellardca166212017-01-30 21:56:46 +0000898 addPass(new InstructionSelect());
Ahmed Bougacha6756a2c2016-07-27 14:31:55 +0000899 return false;
900}
Tom Stellardca166212017-01-30 21:56:46 +0000901
Tom Stellard45bb48e2015-06-13 03:28:10 +0000902void GCNPassConfig::addPreRegAlloc() {
Jan Sjodina06bfe02017-05-15 20:18:37 +0000903 if (LateCFGStructurize) {
904 addPass(createAMDGPUMachineCFGStructurizerPass());
905 }
Nicolai Haehnle213e87f2016-03-21 20:28:33 +0000906 addPass(createSIWholeQuadModePass());
Matt Arsenaultb87fc222015-10-01 22:10:03 +0000907}
908
Matt Arsenaultcf55a652019-03-19 19:33:12 +0000909void GCNPassConfig::addFastRegAlloc() {
Matt Arsenault78fc9da2016-08-22 19:33:16 +0000910 // FIXME: We have to disable the verifier here because of PHIElimination +
911 // TwoAddressInstructions disabling it.
Matt Arsenaulte6740752016-09-29 01:44:16 +0000912
913 // This must be run immediately after phi elimination and before
914 // TwoAddressInstructions, otherwise the processing of the tied operand of
915 // SI_ELSE will introduce a copy of the tied operand source after the else.
916 insertPass(&PHIEliminationID, &SILowerControlFlowID, false);
Matt Arsenault78fc9da2016-08-22 19:33:16 +0000917
Neil Henning0a30f332019-04-01 15:19:52 +0000918 // This must be run just after RegisterCoalescing.
919 insertPass(&RegisterCoalescerID, &SIPreAllocateWWMRegsID, false);
Connor Abbott92638ab2017-08-04 18:36:52 +0000920
Matt Arsenaultcf55a652019-03-19 19:33:12 +0000921 TargetPassConfig::addFastRegAlloc();
Matt Arsenaultb87fc222015-10-01 22:10:03 +0000922}
923
Matt Arsenaultcf55a652019-03-19 19:33:12 +0000924void GCNPassConfig::addOptimizedRegAlloc() {
Matt Arsenault4d47ac32019-03-27 16:58:30 +0000925 if (OptExecMaskPreRA) {
926 insertPass(&MachineSchedulerID, &SIOptimizeExecMaskingPreRAID);
927 insertPass(&SIOptimizeExecMaskingPreRAID, &SIFormMemoryClausesID);
928 } else {
929 insertPass(&MachineSchedulerID, &SIFormMemoryClausesID);
930 }
Stanislav Mekhanoshin739174c2018-05-31 20:13:51 +0000931
Matt Arsenaulte6740752016-09-29 01:44:16 +0000932 // This must be run immediately after phi elimination and before
933 // TwoAddressInstructions, otherwise the processing of the tied operand of
934 // SI_ELSE will introduce a copy of the tied operand source after the else.
935 insertPass(&PHIEliminationID, &SILowerControlFlowID, false);
Matt Arsenault78fc9da2016-08-22 19:33:16 +0000936
Neil Henning0a30f332019-04-01 15:19:52 +0000937 // This must be run just after RegisterCoalescing.
938 insertPass(&RegisterCoalescerID, &SIPreAllocateWWMRegsID, false);
Connor Abbott92638ab2017-08-04 18:36:52 +0000939
Stanislav Mekhanoshinc8f78f82019-04-05 20:11:32 +0000940 if (EnableDCEInRA)
941 insertPass(&RenameIndependentSubregsID, &DeadMachineInstructionElimID);
942
Matt Arsenaultcf55a652019-03-19 19:33:12 +0000943 TargetPassConfig::addOptimizedRegAlloc();
Tom Stellard45bb48e2015-06-13 03:28:10 +0000944}
945
Stanislav Mekhanoshinc29d4912019-05-01 16:40:49 +0000946bool GCNPassConfig::addPreRewrite() {
947 if (EnableRegReassign) {
948 addPass(&GCNNSAReassignID);
Stanislav Mekhanoshin3b7925f2019-05-01 16:49:31 +0000949 addPass(&GCNRegBankReassignID);
Stanislav Mekhanoshinc29d4912019-05-01 16:40:49 +0000950 }
951 return true;
952}
953
Matt Arsenaulte6740752016-09-29 01:44:16 +0000954void GCNPassConfig::addPostRegAlloc() {
Stanislav Mekhanoshin22a56f22017-01-24 17:46:17 +0000955 addPass(&SIFixVGPRCopiesID);
Matt Arsenault105fc1a2018-11-26 17:02:02 +0000956 if (getOptLevel() > CodeGenOpt::None)
957 addPass(&SIOptimizeExecMaskingID);
Matt Arsenaulte6740752016-09-29 01:44:16 +0000958 TargetPassConfig::addPostRegAlloc();
959}
960
Tom Stellard45bb48e2015-06-13 03:28:10 +0000961void GCNPassConfig::addPreSched2() {
Tom Stellard45bb48e2015-06-13 03:28:10 +0000962}
963
964void GCNPassConfig::addPreEmitPass() {
Mark Searles72da47d2018-07-16 10:02:41 +0000965 addPass(createSIMemoryLegalizerPass());
966 addPass(createSIInsertWaitcntsPass());
967 addPass(createSIShrinkInstructionsPass());
Tim Corringham4c4d2fe2018-12-10 12:06:10 +0000968 addPass(createSIModeRegisterPass());
Mark Searles72da47d2018-07-16 10:02:41 +0000969
Tom Stellardcb6ba622016-04-30 00:23:06 +0000970 // The hazard recognizer that runs as part of the post-ra scheduler does not
Matt Arsenault254a6452016-06-28 16:59:53 +0000971 // guarantee to be able handle all hazards correctly. This is because if there
972 // are multiple scheduling regions in a basic block, the regions are scheduled
973 // bottom up, so when we begin to schedule a region we don't know what
974 // instructions were emitted directly before it.
Tom Stellardcb6ba622016-04-30 00:23:06 +0000975 //
Matt Arsenault254a6452016-06-28 16:59:53 +0000976 // Here we add a stand-alone hazard recognizer pass which can handle all
977 // cases.
Mark Searles72da47d2018-07-16 10:02:41 +0000978 //
979 // FIXME: This stand-alone pass will emit indiv. S_NOP 0, as needed. It would
980 // be better for it to emit S_NOP <N> when possible.
Tom Stellardcb6ba622016-04-30 00:23:06 +0000981 addPass(&PostRAHazardRecognizerID);
982
Matt Arsenault78fc9da2016-08-22 19:33:16 +0000983 addPass(&SIInsertSkipsPassID);
Matt Arsenault6bc43d82016-10-06 16:20:41 +0000984 addPass(&BranchRelaxationPassID);
Tom Stellard45bb48e2015-06-13 03:28:10 +0000985}
986
987TargetPassConfig *GCNTargetMachine::createPassConfig(PassManagerBase &PM) {
Matthias Braun5e394c32017-05-30 21:36:41 +0000988 return new GCNPassConfig(*this, PM);
Tom Stellard45bb48e2015-06-13 03:28:10 +0000989}
Matt Arsenaultbc6d07c2019-03-14 22:54:43 +0000990
991yaml::MachineFunctionInfo *GCNTargetMachine::createDefaultFuncInfoYAML() const {
992 return new yaml::SIMachineFunctionInfo();
993}
994
995yaml::MachineFunctionInfo *
996GCNTargetMachine::convertFuncInfoToYAML(const MachineFunction &MF) const {
997 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
998 return new yaml::SIMachineFunctionInfo(*MFI,
999 *MF.getSubtarget().getRegisterInfo());
1000}
1001
1002bool GCNTargetMachine::parseMachineFunctionInfo(
1003 const yaml::MachineFunctionInfo &MFI_, PerFunctionMIParsingState &PFS,
1004 SMDiagnostic &Error, SMRange &SourceRange) const {
1005 const yaml::SIMachineFunctionInfo &YamlMFI =
1006 reinterpret_cast<const yaml::SIMachineFunctionInfo &>(MFI_);
1007 MachineFunction &MF = PFS.MF;
1008 SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
1009
1010 MFI->initializeBaseYamlFields(YamlMFI);
1011
1012 auto parseRegister = [&](const yaml::StringValue &RegName, unsigned &RegVal) {
1013 if (parseNamedRegisterReference(PFS, RegVal, RegName.Value, Error)) {
1014 SourceRange = RegName.SourceRange;
1015 return true;
1016 }
1017
1018 return false;
1019 };
1020
1021 auto diagnoseRegisterClass = [&](const yaml::StringValue &RegName) {
1022 // Create a diagnostic for a the register string literal.
1023 const MemoryBuffer &Buffer =
1024 *PFS.SM->getMemoryBuffer(PFS.SM->getMainFileID());
1025 Error = SMDiagnostic(*PFS.SM, SMLoc(), Buffer.getBufferIdentifier(), 1,
1026 RegName.Value.size(), SourceMgr::DK_Error,
1027 "incorrect register class for field", RegName.Value,
1028 None, None);
1029 SourceRange = RegName.SourceRange;
1030 return true;
1031 };
1032
1033 if (parseRegister(YamlMFI.ScratchRSrcReg, MFI->ScratchRSrcReg) ||
1034 parseRegister(YamlMFI.ScratchWaveOffsetReg, MFI->ScratchWaveOffsetReg) ||
1035 parseRegister(YamlMFI.FrameOffsetReg, MFI->FrameOffsetReg) ||
1036 parseRegister(YamlMFI.StackPtrOffsetReg, MFI->StackPtrOffsetReg))
1037 return true;
1038
1039 if (MFI->ScratchRSrcReg != AMDGPU::PRIVATE_RSRC_REG &&
1040 !AMDGPU::SReg_128RegClass.contains(MFI->ScratchRSrcReg)) {
1041 return diagnoseRegisterClass(YamlMFI.ScratchRSrcReg);
1042 }
1043
1044 if (MFI->ScratchWaveOffsetReg != AMDGPU::SCRATCH_WAVE_OFFSET_REG &&
1045 !AMDGPU::SGPR_32RegClass.contains(MFI->ScratchWaveOffsetReg)) {
1046 return diagnoseRegisterClass(YamlMFI.ScratchWaveOffsetReg);
1047 }
1048
1049 if (MFI->FrameOffsetReg != AMDGPU::FP_REG &&
1050 !AMDGPU::SGPR_32RegClass.contains(MFI->FrameOffsetReg)) {
1051 return diagnoseRegisterClass(YamlMFI.FrameOffsetReg);
1052 }
1053
1054 if (MFI->StackPtrOffsetReg != AMDGPU::SP_REG &&
1055 !AMDGPU::SGPR_32RegClass.contains(MFI->StackPtrOffsetReg)) {
1056 return diagnoseRegisterClass(YamlMFI.StackPtrOffsetReg);
1057 }
1058
1059 return false;
1060}