blob: e8f7476dd76ef45bc07073d1b7fe4dc80c8fc4b6 [file] [log] [blame]
Tom Stellard75aadc22012-12-11 21:25:42 +00001//===-- AMDGPU.h - MachineFunction passes hw codegen --------------*- C++ -*-=//
2//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8/// \file
9//===----------------------------------------------------------------------===//
10
Matt Arsenault6b6a2c32016-03-11 08:00:27 +000011#ifndef LLVM_LIB_TARGET_AMDGPU_AMDGPU_H
12#define LLVM_LIB_TARGET_AMDGPU_AMDGPU_H
Tom Stellard75aadc22012-12-11 21:25:42 +000013
Matt Arsenault678e1112017-04-10 17:58:06 +000014#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
Konstantin Zhuravlyov60a83732016-10-03 18:47:26 +000015#include "llvm/Target/TargetMachine.h"
16
Tom Stellard75aadc22012-12-11 21:25:42 +000017namespace llvm {
18
Tom Stellard75aadc22012-12-11 21:25:42 +000019class AMDGPUTargetMachine;
Tom Stellarda6c6e1b2013-06-07 20:37:48 +000020class FunctionPass;
Matt Arsenaulta1fe17c2016-07-19 23:16:53 +000021class GCNTargetMachine;
Matt Arsenault2ffe8fd2016-08-11 19:18:50 +000022class ModulePass;
23class Pass;
Tom Stellarda6c6e1b2013-06-07 20:37:48 +000024class Target;
25class TargetMachine;
Stanislav Mekhanoshin1d8cf2b2017-09-29 23:40:19 +000026class TargetOptions;
Matt Arsenault2ffe8fd2016-08-11 19:18:50 +000027class PassRegistry;
Yaxun Liu1a14bfa2017-03-27 14:04:01 +000028class Module;
Tom Stellard75aadc22012-12-11 21:25:42 +000029
30// R600 Passes
Francis Visoiu Mistrih8b617642017-05-18 17:21:13 +000031FunctionPass *createR600VectorRegMerger();
32FunctionPass *createR600ExpandSpecialInstrsPass();
Tom Stellard1de55822013-12-11 17:51:41 +000033FunctionPass *createR600EmitClauseMarkers();
Francis Visoiu Mistrih8b617642017-05-18 17:21:13 +000034FunctionPass *createR600ClauseMergePass();
35FunctionPass *createR600Packetizer();
36FunctionPass *createR600ControlFlowFinalizer();
Tom Stellardf2ba9722013-12-11 17:51:47 +000037FunctionPass *createAMDGPUCFGStructurizerPass();
Tom Stellard20287692017-08-08 04:57:55 +000038FunctionPass *createR600ISelDag(TargetMachine *TM, CodeGenOpt::Level OptLevel);
Tom Stellard75aadc22012-12-11 21:25:42 +000039
40// SI Passes
Tom Stellardf8794352012-12-19 22:10:31 +000041FunctionPass *createSIAnnotateControlFlowPass();
Tom Stellard6596ba72014-11-21 22:06:37 +000042FunctionPass *createSIFoldOperandsPass();
Sam Koltonf60ad582017-03-21 12:51:34 +000043FunctionPass *createSIPeepholeSDWAPass();
Tom Stellard1bd80722014-04-30 15:31:33 +000044FunctionPass *createSILowerI1CopiesPass();
Tom Stellard1aaad692014-07-21 16:55:33 +000045FunctionPass *createSIShrinkInstructionsPass();
Francis Visoiu Mistrih8b617642017-05-18 17:21:13 +000046FunctionPass *createSILoadStoreOptimizerPass();
Nicolai Haehnle213e87f2016-03-21 20:28:33 +000047FunctionPass *createSIWholeQuadModePass();
Tom Stellard28d13a42015-05-12 17:13:02 +000048FunctionPass *createSIFixControlFlowLiveIntervalsPass();
Stanislav Mekhanoshin37e7f952017-08-01 23:14:32 +000049FunctionPass *createSIOptimizeExecMaskingPreRAPass();
Matt Arsenault782c03b2015-11-03 22:30:13 +000050FunctionPass *createSIFixSGPRCopiesPass();
Konstantin Zhuravlyove9a5a772017-07-21 21:19:23 +000051FunctionPass *createSIMemoryLegalizerPass();
Konstantin Zhuravlyova7919322016-05-10 18:33:41 +000052FunctionPass *createSIDebuggerInsertNopsPass();
Tom Stellard6e1967e2016-02-05 17:42:38 +000053FunctionPass *createSIInsertWaitsPass();
Kannan Narayananacb089e2017-04-12 03:25:12 +000054FunctionPass *createSIInsertWaitcntsPass();
Connor Abbott92638ab2017-08-04 18:36:52 +000055FunctionPass *createSIFixWWMLivenessPass();
Stanislav Mekhanoshin1d8cf2b2017-09-29 23:40:19 +000056FunctionPass *createAMDGPUSimplifyLibCallsPass(const TargetOptions &);
Stanislav Mekhanoshin7f377942017-08-11 16:42:09 +000057FunctionPass *createAMDGPUUseNativeCallsPass();
Francis Visoiu Mistrih8b617642017-05-18 17:21:13 +000058FunctionPass *createAMDGPUCodeGenPreparePass();
Jan Sjodina06bfe02017-05-15 20:18:37 +000059FunctionPass *createAMDGPUMachineCFGStructurizerPass();
Matt Arsenaultc06574f2017-07-28 18:40:05 +000060FunctionPass *createAMDGPURewriteOutArgumentsPass();
Jan Sjodina06bfe02017-05-15 20:18:37 +000061
Matt Arsenault7016f132017-08-03 22:30:46 +000062void initializeAMDGPUDAGToDAGISelPass(PassRegistry&);
63
Jan Sjodina06bfe02017-05-15 20:18:37 +000064void initializeAMDGPUMachineCFGStructurizerPass(PassRegistry&);
65extern char &AMDGPUMachineCFGStructurizerID;
Tom Stellard75aadc22012-12-11 21:25:42 +000066
Matt Arsenault746e0652017-06-02 18:02:42 +000067void initializeAMDGPUAlwaysInlinePass(PassRegistry&);
68
Matt Arsenault6b930462017-07-13 21:43:42 +000069Pass *createAMDGPUAnnotateKernelFeaturesPass();
Matt Arsenault39319482015-11-06 18:01:57 +000070void initializeAMDGPUAnnotateKernelFeaturesPass(PassRegistry &);
71extern char &AMDGPUAnnotateKernelFeaturesID;
72
Francis Visoiu Mistrih8b617642017-05-18 17:21:13 +000073ModulePass *createAMDGPULowerIntrinsicsPass();
Matt Arsenault0699ef32017-02-09 22:00:42 +000074void initializeAMDGPULowerIntrinsicsPass(PassRegistry &);
75extern char &AMDGPULowerIntrinsicsID;
76
Matt Arsenaultc06574f2017-07-28 18:40:05 +000077void initializeAMDGPURewriteOutArgumentsPass(PassRegistry &);
78extern char &AMDGPURewriteOutArgumentsID;
79
Tom Stellarda2f57be2017-08-02 22:19:45 +000080void initializeR600ClauseMergePassPass(PassRegistry &);
81extern char &R600ClauseMergePassID;
82
83void initializeR600ControlFlowFinalizerPass(PassRegistry &);
84extern char &R600ControlFlowFinalizerID;
85
86void initializeR600ExpandSpecialInstrsPassPass(PassRegistry &);
87extern char &R600ExpandSpecialInstrsPassID;
88
89void initializeR600VectorRegMergerPass(PassRegistry &);
90extern char &R600VectorRegMergerID;
91
92void initializeR600PacketizerPass(PassRegistry &);
93extern char &R600PacketizerID;
94
Tom Stellard6596ba72014-11-21 22:06:37 +000095void initializeSIFoldOperandsPass(PassRegistry &);
96extern char &SIFoldOperandsID;
97
Sam Koltonf60ad582017-03-21 12:51:34 +000098void initializeSIPeepholeSDWAPass(PassRegistry &);
99extern char &SIPeepholeSDWAID;
100
Matt Arsenaultc3a01ec2016-06-09 23:18:47 +0000101void initializeSIShrinkInstructionsPass(PassRegistry&);
102extern char &SIShrinkInstructionsID;
103
Matt Arsenault782c03b2015-11-03 22:30:13 +0000104void initializeSIFixSGPRCopiesPass(PassRegistry &);
105extern char &SIFixSGPRCopiesID;
106
Stanislav Mekhanoshin22a56f22017-01-24 17:46:17 +0000107void initializeSIFixVGPRCopiesPass(PassRegistry &);
108extern char &SIFixVGPRCopiesID;
109
Tom Stellard1bd80722014-04-30 15:31:33 +0000110void initializeSILowerI1CopiesPass(PassRegistry &);
111extern char &SILowerI1CopiesID;
112
Matt Arsenault41033282014-10-10 22:01:59 +0000113void initializeSILoadStoreOptimizerPass(PassRegistry &);
114extern char &SILoadStoreOptimizerID;
115
Nicolai Haehnle213e87f2016-03-21 20:28:33 +0000116void initializeSIWholeQuadModePass(PassRegistry &);
117extern char &SIWholeQuadModeID;
118
Matt Arsenault55d49cf2016-02-12 02:16:10 +0000119void initializeSILowerControlFlowPass(PassRegistry &);
Matt Arsenault78fc9da2016-08-22 19:33:16 +0000120extern char &SILowerControlFlowID;
Matt Arsenault55d49cf2016-02-12 02:16:10 +0000121
Matt Arsenault78fc9da2016-08-22 19:33:16 +0000122void initializeSIInsertSkipsPass(PassRegistry &);
123extern char &SIInsertSkipsPassID;
Matt Arsenault55d49cf2016-02-12 02:16:10 +0000124
Matt Arsenaulte6740752016-09-29 01:44:16 +0000125void initializeSIOptimizeExecMaskingPass(PassRegistry &);
126extern char &SIOptimizeExecMaskingID;
127
Connor Abbott92638ab2017-08-04 18:36:52 +0000128void initializeSIFixWWMLivenessPass(PassRegistry &);
129extern char &SIFixWWMLivenessID;
130
Stanislav Mekhanoshin7f377942017-08-11 16:42:09 +0000131void initializeAMDGPUSimplifyLibCallsPass(PassRegistry &);
132extern char &AMDGPUSimplifyLibCallsID;
133
134void initializeAMDGPUUseNativeCallsPass(PassRegistry &);
135extern char &AMDGPUUseNativeCallsID;
136
Tom Stellard75aadc22012-12-11 21:25:42 +0000137// Passes common to R600 and SI
Francis Visoiu Mistrih8b617642017-05-18 17:21:13 +0000138FunctionPass *createAMDGPUPromoteAlloca();
Matt Arsenaulte0132462016-01-30 05:19:45 +0000139void initializeAMDGPUPromoteAllocaPass(PassRegistry&);
140extern char &AMDGPUPromoteAllocaID;
141
Tom Stellardf8794352012-12-19 22:10:31 +0000142Pass *createAMDGPUStructurizeCFGPass();
Matt Arsenault7016f132017-08-03 22:30:46 +0000143FunctionPass *createAMDGPUISelDag(
144 TargetMachine *TM = nullptr,
145 CodeGenOpt::Level OptLevel = CodeGenOpt::Default);
Stanislav Mekhanoshin89653df2017-03-30 20:16:02 +0000146ModulePass *createAMDGPUAlwaysInlinePass(bool GlobalOpt = true);
Tom Stellardfd253952015-08-07 23:19:30 +0000147ModulePass *createAMDGPUOpenCLImageTypeLoweringPass();
Tom Stellarda6f24c62015-12-15 20:55:55 +0000148FunctionPass *createAMDGPUAnnotateUniformValues();
Tom Stellarda6c6e1b2013-06-07 20:37:48 +0000149
Stanislav Mekhanoshinf6c1feb2017-01-27 16:38:10 +0000150ModulePass* createAMDGPUUnifyMetadataPass();
Stanislav Mekhanoshin50ea93a2016-12-08 19:46:04 +0000151void initializeAMDGPUUnifyMetadataPass(PassRegistry&);
152extern char &AMDGPUUnifyMetadataID;
153
Stanislav Mekhanoshin37e7f952017-08-01 23:14:32 +0000154void initializeSIOptimizeExecMaskingPreRAPass(PassRegistry&);
155extern char &SIOptimizeExecMaskingPreRAID;
156
Tom Stellarda6f24c62015-12-15 20:55:55 +0000157void initializeAMDGPUAnnotateUniformValuesPass(PassRegistry&);
158extern char &AMDGPUAnnotateUniformValuesPassID;
Tom Stellardb2de94e2014-07-02 20:53:48 +0000159
Matt Arsenault86de4862016-06-24 07:07:55 +0000160void initializeAMDGPUCodeGenPreparePass(PassRegistry&);
161extern char &AMDGPUCodeGenPrepareID;
162
Tom Stellard77a17772016-01-20 15:48:27 +0000163void initializeSIAnnotateControlFlowPass(PassRegistry&);
164extern char &SIAnnotateControlFlowPassID;
165
Konstantin Zhuravlyove9a5a772017-07-21 21:19:23 +0000166void initializeSIMemoryLegalizerPass(PassRegistry&);
167extern char &SIMemoryLegalizerID;
168
Konstantin Zhuravlyova7919322016-05-10 18:33:41 +0000169void initializeSIDebuggerInsertNopsPass(PassRegistry&);
170extern char &SIDebuggerInsertNopsID;
Tom Stellardcc7067a62016-03-03 03:53:29 +0000171
Tom Stellard6e1967e2016-02-05 17:42:38 +0000172void initializeSIInsertWaitsPass(PassRegistry&);
173extern char &SIInsertWaitsID;
174
Kannan Narayananacb089e2017-04-12 03:25:12 +0000175void initializeSIInsertWaitcntsPass(PassRegistry&);
176extern char &SIInsertWaitcntsID;
177
Matt Arsenaultb8f8dbc2017-03-24 19:52:05 +0000178void initializeAMDGPUUnifyDivergentExitNodesPass(PassRegistry&);
179extern char &AMDGPUUnifyDivergentExitNodesID;
180
Stanislav Mekhanoshin8e45acf2017-03-17 23:56:58 +0000181ImmutablePass *createAMDGPUAAWrapperPass();
182void initializeAMDGPUAAWrapperPassPass(PassRegistry&);
183
Matt Arsenault7016f132017-08-03 22:30:46 +0000184void initializeAMDGPUArgumentUsageInfoPass(PassRegistry &);
185
Stanislav Mekhanoshin5670e6d2017-09-20 04:25:58 +0000186Pass *createAMDGPUFunctionInliningPass();
187void initializeAMDGPUInlinerPass(PassRegistry&);
188
Yaxun Liude4b88d2017-10-10 19:39:48 +0000189ModulePass *createAMDGPUOpenCLEnqueuedBlockLoweringPass();
190void initializeAMDGPUOpenCLEnqueuedBlockLoweringPass(PassRegistry &);
191extern char &AMDGPUOpenCLEnqueuedBlockLoweringID;
192
Mehdi Aminif42454b2016-10-09 23:00:34 +0000193Target &getTheAMDGPUTarget();
194Target &getTheGCNTarget();
Tom Stellard75aadc22012-12-11 21:25:42 +0000195
Tom Stellard067c8152014-07-21 14:01:14 +0000196namespace AMDGPU {
197enum TargetIndex {
Tom Stellard95292bb2015-01-20 17:49:47 +0000198 TI_CONSTDATA_START,
199 TI_SCRATCH_RSRC_DWORD0,
200 TI_SCRATCH_RSRC_DWORD1,
201 TI_SCRATCH_RSRC_DWORD2,
202 TI_SCRATCH_RSRC_DWORD3
Tom Stellard067c8152014-07-21 14:01:14 +0000203};
204}
205
Tom Stellard75aadc22012-12-11 21:25:42 +0000206} // End namespace llvm
207
Tom Stellarda6c6e1b2013-06-07 20:37:48 +0000208/// OpenCL uses address spaces to differentiate between
209/// various memory regions on the hardware. On the CPU
210/// all of the address spaces point to the same memory,
211/// however on the GPU, each address space points to
Alp Tokercb402912014-01-24 17:20:08 +0000212/// a separate piece of memory that is unique from other
Tom Stellarda6c6e1b2013-06-07 20:37:48 +0000213/// memory locations.
Yaxun Liu1a14bfa2017-03-27 14:04:01 +0000214struct AMDGPUAS {
215 // The following address space values depend on the triple environment.
216 unsigned PRIVATE_ADDRESS; ///< Address space for private memory.
Yaxun Liu1a14bfa2017-03-27 14:04:01 +0000217 unsigned FLAT_ADDRESS; ///< Address space for flat memory.
218 unsigned REGION_ADDRESS; ///< Address space for region memory.
219
220 // The maximum value for flat, generic, local, private, constant and region.
221 const static unsigned MAX_COMMON_ADDRESS = 5;
222
223 const static unsigned GLOBAL_ADDRESS = 1; ///< Address space for global memory (RAT0, VTX0).
Yaxun Liu76ae47c2017-04-06 19:17:32 +0000224 const static unsigned CONSTANT_ADDRESS = 2; ///< Address space for constant memory (VTX2)
Yaxun Liu1a14bfa2017-03-27 14:04:01 +0000225 const static unsigned LOCAL_ADDRESS = 3; ///< Address space for local memory.
226 const static unsigned PARAM_D_ADDRESS = 6; ///< Address space for direct addressible parameter memory (CONST0)
227 const static unsigned PARAM_I_ADDRESS = 7; ///< Address space for indirect addressible parameter memory (VTX1)
Tom Stellard1e803092013-07-23 01:48:18 +0000228
229 // Do not re-order the CONSTANT_BUFFER_* enums. Several places depend on this
230 // order to be able to dynamically index a constant buffer, for example:
231 //
232 // ConstantBufferAS = CONSTANT_BUFFER_0 + CBIdx
233
Yaxun Liu1a14bfa2017-03-27 14:04:01 +0000234 const static unsigned CONSTANT_BUFFER_0 = 8;
235 const static unsigned CONSTANT_BUFFER_1 = 9;
236 const static unsigned CONSTANT_BUFFER_2 = 10;
237 const static unsigned CONSTANT_BUFFER_3 = 11;
238 const static unsigned CONSTANT_BUFFER_4 = 12;
239 const static unsigned CONSTANT_BUFFER_5 = 13;
240 const static unsigned CONSTANT_BUFFER_6 = 14;
241 const static unsigned CONSTANT_BUFFER_7 = 15;
242 const static unsigned CONSTANT_BUFFER_8 = 16;
243 const static unsigned CONSTANT_BUFFER_9 = 17;
244 const static unsigned CONSTANT_BUFFER_10 = 18;
245 const static unsigned CONSTANT_BUFFER_11 = 19;
246 const static unsigned CONSTANT_BUFFER_12 = 20;
247 const static unsigned CONSTANT_BUFFER_13 = 21;
248 const static unsigned CONSTANT_BUFFER_14 = 22;
249 const static unsigned CONSTANT_BUFFER_15 = 23;
Matt Arsenault73e06fa2015-06-04 16:17:42 +0000250
251 // Some places use this if the address space can't be determined.
Yaxun Liu1a14bfa2017-03-27 14:04:01 +0000252 const static unsigned UNKNOWN_ADDRESS_SPACE = ~0u;
Tom Stellarda6c6e1b2013-06-07 20:37:48 +0000253};
254
Yaxun Liu1a14bfa2017-03-27 14:04:01 +0000255namespace llvm {
256namespace AMDGPU {
257AMDGPUAS getAMDGPUAS(const Module &M);
258AMDGPUAS getAMDGPUAS(const TargetMachine &TM);
259AMDGPUAS getAMDGPUAS(Triple T);
260} // namespace AMDGPU
261} // namespace llvm
Tom Stellarda6c6e1b2013-06-07 20:37:48 +0000262
Benjamin Kramera7c40ef2014-08-13 16:26:38 +0000263#endif