blob: 0c4c9d9d982431c562c8d9c7ec3cb3e803d80e1f [file] [log] [blame]
Tom Stellardcb6ba622016-04-30 00:23:06 +00001//===-- GCNHazardRecognizers.h - GCN Hazard Recognizers ---------*- C++ -*-===//
2//
Chandler Carruth2946cd72019-01-19 08:50:56 +00003// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
Tom Stellardcb6ba622016-04-30 00:23:06 +00006//
7//===----------------------------------------------------------------------===//
8//
9// This file defines hazard recognizers for scheduling on GCN processors.
10//
11//===----------------------------------------------------------------------===//
12
13#ifndef LLVM_LIB_TARGET_AMDGPUHAZARDRECOGNIZERS_H
14#define LLVM_LIB_TARGET_AMDGPUHAZARDRECOGNIZERS_H
15
Matt Arsenault03c67d12017-11-17 04:18:24 +000016#include "llvm/ADT/BitVector.h"
Benjamin Kramerd3f4c052016-06-12 16:13:55 +000017#include "llvm/ADT/STLExtras.h"
Tom Stellardcb6ba622016-04-30 00:23:06 +000018#include "llvm/CodeGen/ScheduleHazardRecognizer.h"
Tom Stellardcb6ba622016-04-30 00:23:06 +000019#include <list>
20
21namespace llvm {
22
23class MachineFunction;
24class MachineInstr;
Mark Searlesd29f24a2017-12-07 20:34:25 +000025class MachineOperand;
26class MachineRegisterInfo;
Tom Stellardcb6ba622016-04-30 00:23:06 +000027class ScheduleDAG;
28class SIInstrInfo;
Matt Arsenault03c67d12017-11-17 04:18:24 +000029class SIRegisterInfo;
Tom Stellard5bfbae52018-07-11 20:59:01 +000030class GCNSubtarget;
Tom Stellardcb6ba622016-04-30 00:23:06 +000031
32class GCNHazardRecognizer final : public ScheduleHazardRecognizer {
Stanislav Mekhanoshinf92ed692019-01-21 19:11:26 +000033public:
34 typedef function_ref<bool(MachineInstr *)> IsHazardFn;
35
36private:
37 // Distinguish if we are called from scheduler or hazard recognizer
38 bool IsHazardRecognizerMode;
39
Matt Arsenault43e92fe2016-06-24 06:30:11 +000040 // This variable stores the instruction that has been emitted this cycle. It
41 // will be added to EmittedInstrs, when AdvanceCycle() or RecedeCycle() is
Tom Stellardcb6ba622016-04-30 00:23:06 +000042 // called.
43 MachineInstr *CurrCycleInstr;
44 std::list<MachineInstr*> EmittedInstrs;
45 const MachineFunction &MF;
Tom Stellard5bfbae52018-07-11 20:59:01 +000046 const GCNSubtarget &ST;
Matt Arsenault59ece952017-03-17 21:36:28 +000047 const SIInstrInfo &TII;
Matt Arsenault03c67d12017-11-17 04:18:24 +000048 const SIRegisterInfo &TRI;
49
50 /// RegUnits of uses in the current soft memory clause.
51 BitVector ClauseUses;
52
53 /// RegUnits of defs in the current soft memory clause.
54 BitVector ClauseDefs;
55
56 void resetClause() {
57 ClauseUses.reset();
58 ClauseDefs.reset();
59 }
60
61 void addClauseInst(const MachineInstr &MI);
Tom Stellardcb6ba622016-04-30 00:23:06 +000062
Austin Kerbow8a3d3a92019-05-07 22:12:15 +000063 // Advance over a MachineInstr bundle. Look for hazards in the bundled
64 // instructions.
65 void processBundle();
66
Stanislav Mekhanoshinf92ed692019-01-21 19:11:26 +000067 int getWaitStatesSince(IsHazardFn IsHazard, int Limit);
68 int getWaitStatesSinceDef(unsigned Reg, IsHazardFn IsHazardDef, int Limit);
69 int getWaitStatesSinceSetReg(IsHazardFn IsHazard, int Limit);
Tom Stellardcb6ba622016-04-30 00:23:06 +000070
Matt Arsenaulta41351e2017-11-17 21:35:32 +000071 int checkSoftClauseHazards(MachineInstr *SMEM);
Tom Stellardcb6ba622016-04-30 00:23:06 +000072 int checkSMRDHazards(MachineInstr *SMRD);
73 int checkVMEMHazards(MachineInstr* VMEM);
Tom Stellarda27007e2016-05-02 16:23:09 +000074 int checkDPPHazards(MachineInstr *DPP);
Tom Stellard5ab61542016-10-07 23:42:48 +000075 int checkDivFMasHazards(MachineInstr *DivFMas);
Tom Stellard961811c2016-10-15 00:58:14 +000076 int checkGetRegHazards(MachineInstr *GetRegInstr);
Tom Stellard30d30822016-10-27 20:39:09 +000077 int checkSetRegHazards(MachineInstr *SetRegInstr);
Tom Stellardb133fbb2016-10-27 23:05:31 +000078 int createsVALUHazard(const MachineInstr &MI);
79 int checkVALUHazards(MachineInstr *VALU);
Mark Searlesd29f24a2017-12-07 20:34:25 +000080 int checkVALUHazardsHelper(const MachineOperand &Def, const MachineRegisterInfo &MRI);
Tom Stellard04051b52016-10-27 23:42:29 +000081 int checkRWLaneHazards(MachineInstr *RWLane);
Tom Stellardaea899e2016-10-27 23:50:21 +000082 int checkRFEHazards(MachineInstr *RFE);
Mark Searlesd29f24a2017-12-07 20:34:25 +000083 int checkInlineAsmHazards(MachineInstr *IA);
Matt Arsenaulte823d922017-02-18 18:29:53 +000084 int checkAnyInstHazards(MachineInstr *MI);
85 int checkReadM0Hazards(MachineInstr *SMovRel);
Stanislav Mekhanoshin51d14152019-05-04 04:30:57 +000086 int checkNSAtoVMEMHazard(MachineInstr *MI);
Austin Kerbow8a3d3a92019-05-07 22:12:15 +000087 void fixHazards(MachineInstr *MI);
Stanislav Mekhanoshin5f581c92019-06-12 17:52:51 +000088 bool fixVcmpxPermlaneHazards(MachineInstr *MI);
Stanislav Mekhanoshin51d14152019-05-04 04:30:57 +000089 bool fixVMEMtoScalarWriteHazards(MachineInstr *MI);
90 bool fixSMEMtoVectorWriteHazards(MachineInstr *MI);
91 bool fixVcmpxExecWARHazard(MachineInstr *MI);
92 bool fixLdsBranchVmemWARHazard(MachineInstr *MI);
93
Tom Stellardcb6ba622016-04-30 00:23:06 +000094public:
95 GCNHazardRecognizer(const MachineFunction &MF);
96 // We can only issue one instruction per cycle.
97 bool atIssueLimit() const override { return true; }
98 void EmitInstruction(SUnit *SU) override;
99 void EmitInstruction(MachineInstr *MI) override;
100 HazardType getHazardType(SUnit *SU, int Stalls) override;
101 void EmitNoop() override;
102 unsigned PreEmitNoops(SUnit *SU) override;
103 unsigned PreEmitNoops(MachineInstr *) override;
Stanislav Mekhanoshinf92ed692019-01-21 19:11:26 +0000104 unsigned PreEmitNoopsCommon(MachineInstr *);
Tom Stellardcb6ba622016-04-30 00:23:06 +0000105 void AdvanceCycle() override;
106 void RecedeCycle() override;
107};
108
109} // end namespace llvm
110
111#endif //LLVM_LIB_TARGET_AMDGPUHAZARDRECOGNIZERS_H