AMDGPU: Fix breaking SMEM clauses
This was completely ignoring subregisters,
so was not very useful. Also only break them
if xnack is actually enabled.
llvm-svn: 318505
diff --git a/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp b/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp
index 3f54f8b..2b73108 100644
--- a/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp
@@ -40,7 +40,10 @@
CurrCycleInstr(nullptr),
MF(MF),
ST(MF.getSubtarget<SISubtarget>()),
- TII(*ST.getInstrInfo()) {
+ TII(*ST.getInstrInfo()),
+ TRI(TII.getRegisterInfo()),
+ ClauseUses(TRI.getNumRegUnits()),
+ ClauseDefs(TRI.getNumRegUnits()) {
MaxLookAhead = 5;
}
@@ -258,19 +261,35 @@
// No-op Hazard Detection
//===----------------------------------------------------------------------===//
-static void addRegsToSet(iterator_range<MachineInstr::const_mop_iterator> Ops,
- std::set<unsigned> &Set) {
+static void addRegUnits(const SIRegisterInfo &TRI,
+ BitVector &BV, unsigned Reg) {
+ for (MCRegUnitIterator RUI(Reg, &TRI); RUI.isValid(); ++RUI)
+ BV.set(*RUI);
+}
+
+static void addRegsToSet(const SIRegisterInfo &TRI,
+ iterator_range<MachineInstr::const_mop_iterator> Ops,
+ BitVector &Set) {
for (const MachineOperand &Op : Ops) {
if (Op.isReg())
- Set.insert(Op.getReg());
+ addRegUnits(TRI, Set, Op.getReg());
}
}
+void GCNHazardRecognizer::addClauseInst(const MachineInstr &MI) {
+ // XXX: Do we need to worry about implicit operands
+ addRegsToSet(TRI, MI.defs(), ClauseDefs);
+ addRegsToSet(TRI, MI.uses(), ClauseUses);
+}
+
int GCNHazardRecognizer::checkSMEMSoftClauseHazards(MachineInstr *SMEM) {
- // SMEM soft clause are only present on VI+
- if (ST.getGeneration() < SISubtarget::VOLCANIC_ISLANDS)
+ // SMEM soft clause are only present on VI+, and only matter if xnack is
+ // enabled.
+ if (!ST.isXNACKEnabled())
return 0;
+ resetClause();
+
// A soft-clause is any group of consecutive SMEM instructions. The
// instructions in this group may return out of order and/or may be
// replayed (i.e. the same instruction issued more than once).
@@ -281,21 +300,16 @@
// (including itself). If we encounter this situaion, we need to break the
// clause by inserting a non SMEM instruction.
- std::set<unsigned> ClauseDefs;
- std::set<unsigned> ClauseUses;
-
for (MachineInstr *MI : EmittedInstrs) {
-
// When we hit a non-SMEM instruction then we have passed the start of the
// clause and we can stop.
if (!MI || !SIInstrInfo::isSMRD(*MI))
break;
- addRegsToSet(MI->defs(), ClauseDefs);
- addRegsToSet(MI->uses(), ClauseUses);
+ addClauseInst(*MI);
}
- if (ClauseDefs.empty())
+ if (ClauseDefs.none())
return 0;
// FIXME: When we support stores, we need to make sure not to put loads and
@@ -304,21 +318,11 @@
if (SMEM->mayStore())
return 1;
- addRegsToSet(SMEM->defs(), ClauseDefs);
- addRegsToSet(SMEM->uses(), ClauseUses);
-
- std::vector<unsigned> Result(std::max(ClauseDefs.size(), ClauseUses.size()));
- std::vector<unsigned>::iterator End;
-
- End = std::set_intersection(ClauseDefs.begin(), ClauseDefs.end(),
- ClauseUses.begin(), ClauseUses.end(), Result.begin());
+ addClauseInst(*SMEM);
// If the set of defs and uses intersect then we cannot add this instruction
// to the clause, so we have a hazard.
- if (End != Result.begin())
- return 1;
-
- return 0;
+ return ClauseDefs.anyCommon(ClauseUses) ? 1 : 0;
}
int GCNHazardRecognizer::checkSMRDHazards(MachineInstr *SMRD) {
diff --git a/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.h b/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.h
index 5680c3d..eb382cc 100644
--- a/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.h
+++ b/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.h
@@ -14,6 +14,7 @@
#ifndef LLVM_LIB_TARGET_AMDGPUHAZARDRECOGNIZERS_H
#define LLVM_LIB_TARGET_AMDGPUHAZARDRECOGNIZERS_H
+#include "llvm/ADT/BitVector.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/CodeGen/ScheduleHazardRecognizer.h"
#include <list>
@@ -24,6 +25,7 @@
class MachineInstr;
class ScheduleDAG;
class SIInstrInfo;
+class SIRegisterInfo;
class SISubtarget;
class GCNHazardRecognizer final : public ScheduleHazardRecognizer {
@@ -35,6 +37,20 @@
const MachineFunction &MF;
const SISubtarget &ST;
const SIInstrInfo &TII;
+ const SIRegisterInfo &TRI;
+
+ /// RegUnits of uses in the current soft memory clause.
+ BitVector ClauseUses;
+
+ /// RegUnits of defs in the current soft memory clause.
+ BitVector ClauseDefs;
+
+ void resetClause() {
+ ClauseUses.reset();
+ ClauseDefs.reset();
+ }
+
+ void addClauseInst(const MachineInstr &MI);
int getWaitStatesSince(function_ref<bool(MachineInstr *)> IsHazard);
int getWaitStatesSinceDef(unsigned Reg,