blob: 90f664a7f7963c90a19646e96ae9efa99e9d647f [file] [log] [blame]
Richard Sandifordbdbb8af2013-08-05 10:58:53 +00001//===-- SystemZElimCompare.cpp - Eliminate comparison instructions --------===//
2//
Chandler Carruth2946cd72019-01-19 08:50:56 +00003// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
Richard Sandifordbdbb8af2013-08-05 10:58:53 +00006//
7//===----------------------------------------------------------------------===//
8//
9// This pass:
10// (1) tries to remove compares if CC already contains the required information
11// (2) fuses compares and branches into COMPARE AND BRANCH instructions
12//
13//===----------------------------------------------------------------------===//
14
Eugene Zelenko3943d2b2017-01-24 22:10:43 +000015#include "SystemZ.h"
16#include "SystemZInstrInfo.h"
Richard Sandifordbdbb8af2013-08-05 10:58:53 +000017#include "SystemZTargetMachine.h"
Eugene Zelenko3943d2b2017-01-24 22:10:43 +000018#include "llvm/ADT/SmallVector.h"
Richard Sandifordbdbb8af2013-08-05 10:58:53 +000019#include "llvm/ADT/Statistic.h"
Eugene Zelenko3943d2b2017-01-24 22:10:43 +000020#include "llvm/ADT/StringRef.h"
Jonas Paulssonbf6744d2019-11-04 16:11:12 +010021#include "llvm/CodeGen/LivePhysRegs.h"
Eugene Zelenko3943d2b2017-01-24 22:10:43 +000022#include "llvm/CodeGen/MachineBasicBlock.h"
23#include "llvm/CodeGen/MachineFunction.h"
Richard Sandifordbdbb8af2013-08-05 10:58:53 +000024#include "llvm/CodeGen/MachineFunctionPass.h"
Eugene Zelenko3943d2b2017-01-24 22:10:43 +000025#include "llvm/CodeGen/MachineInstr.h"
Richard Sandifordbdbb8af2013-08-05 10:58:53 +000026#include "llvm/CodeGen/MachineInstrBuilder.h"
Eugene Zelenko3943d2b2017-01-24 22:10:43 +000027#include "llvm/CodeGen/MachineOperand.h"
David Blaikieb3bde2e2017-11-17 01:07:10 +000028#include "llvm/CodeGen/TargetRegisterInfo.h"
29#include "llvm/CodeGen/TargetSubtargetInfo.h"
Eugene Zelenko3943d2b2017-01-24 22:10:43 +000030#include "llvm/MC/MCInstrDesc.h"
Eugene Zelenko3943d2b2017-01-24 22:10:43 +000031#include <cassert>
32#include <cstdint>
Richard Sandifordbdbb8af2013-08-05 10:58:53 +000033
34using namespace llvm;
35
Chandler Carruth84e68b22014-04-22 02:41:26 +000036#define DEBUG_TYPE "systemz-elim-compare"
37
Richard Sandifordc2121252013-08-05 11:23:46 +000038STATISTIC(BranchOnCounts, "Number of branch-on-count instructions");
Ulrich Weigand2d9e3d92016-11-28 13:59:22 +000039STATISTIC(LoadAndTraps, "Number of load-and-trap instructions");
Richard Sandifordbdbb8af2013-08-05 10:58:53 +000040STATISTIC(EliminatedComparisons, "Number of eliminated comparisons");
41STATISTIC(FusedComparisons, "Number of fused compare-and-branch instructions");
42
43namespace {
Eugene Zelenko3943d2b2017-01-24 22:10:43 +000044
Richard Sandifordc2312692014-03-06 10:38:30 +000045// Represents the references to a particular register in one or more
46// instructions.
47struct Reference {
Eugene Zelenko3943d2b2017-01-24 22:10:43 +000048 Reference() = default;
Richard Sandifordc2121252013-08-05 11:23:46 +000049
Richard Sandifordc2312692014-03-06 10:38:30 +000050 Reference &operator|=(const Reference &Other) {
51 Def |= Other.Def;
Richard Sandifordc2312692014-03-06 10:38:30 +000052 Use |= Other.Use;
Richard Sandifordc2312692014-03-06 10:38:30 +000053 return *this;
54 }
Richard Sandifordc2121252013-08-05 11:23:46 +000055
Aaron Ballmanb46962f2015-02-15 22:00:20 +000056 explicit operator bool() const { return Def || Use; }
Richard Sandifordc2121252013-08-05 11:23:46 +000057
Richard Sandifordc2312692014-03-06 10:38:30 +000058 // True if the register is defined or used in some form, either directly or
59 // via a sub- or super-register.
Eugene Zelenko3943d2b2017-01-24 22:10:43 +000060 bool Def = false;
61 bool Use = false;
Richard Sandifordc2312692014-03-06 10:38:30 +000062};
Richard Sandifordc2121252013-08-05 11:23:46 +000063
Richard Sandifordc2312692014-03-06 10:38:30 +000064class SystemZElimCompare : public MachineFunctionPass {
65public:
66 static char ID;
Eugene Zelenko3943d2b2017-01-24 22:10:43 +000067
Richard Sandifordc2312692014-03-06 10:38:30 +000068 SystemZElimCompare(const SystemZTargetMachine &tm)
Eugene Zelenko3943d2b2017-01-24 22:10:43 +000069 : MachineFunctionPass(ID) {}
Richard Sandifordbdbb8af2013-08-05 10:58:53 +000070
Mehdi Amini117296c2016-10-01 02:56:57 +000071 StringRef getPassName() const override {
Richard Sandifordc2312692014-03-06 10:38:30 +000072 return "SystemZ Comparison Elimination";
73 }
Richard Sandifordbdbb8af2013-08-05 10:58:53 +000074
Richard Sandiford28c111e2014-03-06 11:00:15 +000075 bool processBlock(MachineBasicBlock &MBB);
Craig Topper9d74a5a2014-04-29 07:58:41 +000076 bool runOnMachineFunction(MachineFunction &F) override;
Eugene Zelenko3943d2b2017-01-24 22:10:43 +000077
Derek Schuff1dbf7a52016-04-04 17:09:25 +000078 MachineFunctionProperties getRequiredProperties() const override {
79 return MachineFunctionProperties().set(
Matthias Braun1eb47362016-08-25 01:27:13 +000080 MachineFunctionProperties::Property::NoVRegs);
Derek Schuff1dbf7a52016-04-04 17:09:25 +000081 }
Richard Sandifordbdbb8af2013-08-05 10:58:53 +000082
Richard Sandifordc2312692014-03-06 10:38:30 +000083private:
Duncan P. N. Exon Smith4565ec02016-07-12 01:39:01 +000084 Reference getRegReferences(MachineInstr &MI, unsigned Reg);
85 bool convertToBRCT(MachineInstr &MI, MachineInstr &Compare,
Richard Sandifordc2312692014-03-06 10:38:30 +000086 SmallVectorImpl<MachineInstr *> &CCUsers);
Ulrich Weigand2d9e3d92016-11-28 13:59:22 +000087 bool convertToLoadAndTrap(MachineInstr &MI, MachineInstr &Compare,
88 SmallVectorImpl<MachineInstr *> &CCUsers);
Jonas Paulsson776a81a2018-01-15 15:41:26 +000089 bool convertToLoadAndTest(MachineInstr &MI, MachineInstr &Compare,
90 SmallVectorImpl<MachineInstr *> &CCUsers);
Duncan P. N. Exon Smith4565ec02016-07-12 01:39:01 +000091 bool adjustCCMasksForInstr(MachineInstr &MI, MachineInstr &Compare,
Jonas Paulsson776a81a2018-01-15 15:41:26 +000092 SmallVectorImpl<MachineInstr *> &CCUsers,
93 unsigned ConvOpc = 0);
Duncan P. N. Exon Smith4565ec02016-07-12 01:39:01 +000094 bool optimizeCompareZero(MachineInstr &Compare,
Richard Sandifordc2312692014-03-06 10:38:30 +000095 SmallVectorImpl<MachineInstr *> &CCUsers);
Duncan P. N. Exon Smith4565ec02016-07-12 01:39:01 +000096 bool fuseCompareOperations(MachineInstr &Compare,
Zhan Jun Liauab42cbc2016-06-10 19:58:10 +000097 SmallVectorImpl<MachineInstr *> &CCUsers);
Richard Sandifordbdbb8af2013-08-05 10:58:53 +000098
Eugene Zelenko3943d2b2017-01-24 22:10:43 +000099 const SystemZInstrInfo *TII = nullptr;
100 const TargetRegisterInfo *TRI = nullptr;
Richard Sandifordc2312692014-03-06 10:38:30 +0000101};
Richard Sandifordbdbb8af2013-08-05 10:58:53 +0000102
Richard Sandifordc2312692014-03-06 10:38:30 +0000103char SystemZElimCompare::ID = 0;
Richard Sandifordbdbb8af2013-08-05 10:58:53 +0000104
Eugene Zelenko3943d2b2017-01-24 22:10:43 +0000105} // end anonymous namespace
Richard Sandifordbdbb8af2013-08-05 10:58:53 +0000106
Jonas Paulssonb0e8a2e2017-09-21 13:52:24 +0000107// Returns true if MI is an instruction whose output equals the value in Reg.
108static bool preservesValueOf(MachineInstr &MI, unsigned Reg) {
Duncan P. N. Exon Smith4565ec02016-07-12 01:39:01 +0000109 switch (MI.getOpcode()) {
Richard Sandifordb49a3ab2013-08-05 11:03:20 +0000110 case SystemZ::LR:
111 case SystemZ::LGR:
112 case SystemZ::LGFR:
113 case SystemZ::LTR:
114 case SystemZ::LTGR:
115 case SystemZ::LTGFR:
Richard Sandiford0897fce2013-08-07 11:10:06 +0000116 case SystemZ::LER:
117 case SystemZ::LDR:
118 case SystemZ::LXR:
119 case SystemZ::LTEBR:
120 case SystemZ::LTDBR:
121 case SystemZ::LTXBR:
Duncan P. N. Exon Smith4565ec02016-07-12 01:39:01 +0000122 if (MI.getOperand(1).getReg() == Reg)
Richard Sandifordb49a3ab2013-08-05 11:03:20 +0000123 return true;
124 }
125
Richard Sandifordbdbb8af2013-08-05 10:58:53 +0000126 return false;
127}
128
Jonas Paulssonb0e8a2e2017-09-21 13:52:24 +0000129// Return true if any CC result of MI would (perhaps after conversion)
130// reflect the value of Reg.
131static bool resultTests(MachineInstr &MI, unsigned Reg) {
132 if (MI.getNumOperands() > 0 && MI.getOperand(0).isReg() &&
133 MI.getOperand(0).isDef() && MI.getOperand(0).getReg() == Reg)
134 return true;
135
136 return (preservesValueOf(MI, Reg));
137}
138
Jonas Paulssonee3685f2015-10-09 11:27:44 +0000139// Describe the references to Reg or any of its aliases in MI.
Duncan P. N. Exon Smith4565ec02016-07-12 01:39:01 +0000140Reference SystemZElimCompare::getRegReferences(MachineInstr &MI, unsigned Reg) {
Richard Sandifordc2121252013-08-05 11:23:46 +0000141 Reference Ref;
Jonas Paulsson961c47e2019-01-23 07:42:26 +0000142 if (MI.isDebugInstr())
143 return Ref;
144
Duncan P. N. Exon Smith4565ec02016-07-12 01:39:01 +0000145 for (unsigned I = 0, E = MI.getNumOperands(); I != E; ++I) {
146 const MachineOperand &MO = MI.getOperand(I);
Richard Sandifordc2121252013-08-05 11:23:46 +0000147 if (MO.isReg()) {
Daniel Sanders0c476112019-08-15 19:22:08 +0000148 if (Register MOReg = MO.getReg()) {
Jonas Paulssonee3685f2015-10-09 11:27:44 +0000149 if (TRI->regsOverlap(MOReg, Reg)) {
150 if (MO.isUse())
Richard Sandifordc2121252013-08-05 11:23:46 +0000151 Ref.Use = true;
Jonas Paulssonee3685f2015-10-09 11:27:44 +0000152 else if (MO.isDef())
Richard Sandifordc2121252013-08-05 11:23:46 +0000153 Ref.Def = true;
Richard Sandifordc2121252013-08-05 11:23:46 +0000154 }
155 }
156 }
157 }
158 return Ref;
159}
160
Jonas Paulsson5d3fbd32015-10-08 07:40:23 +0000161// Return true if this is a load and test which can be optimized the
162// same way as compare instruction.
Duncan P. N. Exon Smith4565ec02016-07-12 01:39:01 +0000163static bool isLoadAndTestAsCmp(MachineInstr &MI) {
Jonas Paulsson5d3fbd32015-10-08 07:40:23 +0000164 // If we during isel used a load-and-test as a compare with 0, the
165 // def operand is dead.
Duncan P. N. Exon Smith4565ec02016-07-12 01:39:01 +0000166 return (MI.getOpcode() == SystemZ::LTEBR ||
167 MI.getOpcode() == SystemZ::LTDBR ||
168 MI.getOpcode() == SystemZ::LTXBR) &&
169 MI.getOperand(0).isDead();
Jonas Paulsson5d3fbd32015-10-08 07:40:23 +0000170}
171
172// Return the source register of Compare, which is the unknown value
173// being tested.
Duncan P. N. Exon Smith4565ec02016-07-12 01:39:01 +0000174static unsigned getCompareSourceReg(MachineInstr &Compare) {
Jonas Paulsson5d3fbd32015-10-08 07:40:23 +0000175 unsigned reg = 0;
Duncan P. N. Exon Smith4565ec02016-07-12 01:39:01 +0000176 if (Compare.isCompare())
177 reg = Compare.getOperand(0).getReg();
Jonas Paulsson5d3fbd32015-10-08 07:40:23 +0000178 else if (isLoadAndTestAsCmp(Compare))
Duncan P. N. Exon Smith4565ec02016-07-12 01:39:01 +0000179 reg = Compare.getOperand(1).getReg();
Eugene Zelenko3943d2b2017-01-24 22:10:43 +0000180 assert(reg);
Jonas Paulsson5d3fbd32015-10-08 07:40:23 +0000181
182 return reg;
183}
184
Richard Sandifordc2121252013-08-05 11:23:46 +0000185// Compare compares the result of MI against zero. If MI is an addition
186// of -1 and if CCUsers is a single branch on nonzero, eliminate the addition
Ulrich Weigand75839912016-11-28 13:40:08 +0000187// and convert the branch to a BRCT(G) or BRCTH. Return true on success.
Duncan P. N. Exon Smith4565ec02016-07-12 01:39:01 +0000188bool SystemZElimCompare::convertToBRCT(
189 MachineInstr &MI, MachineInstr &Compare,
190 SmallVectorImpl<MachineInstr *> &CCUsers) {
Richard Sandifordc2121252013-08-05 11:23:46 +0000191 // Check whether we have an addition of -1.
Duncan P. N. Exon Smith4565ec02016-07-12 01:39:01 +0000192 unsigned Opcode = MI.getOpcode();
Richard Sandifordc2121252013-08-05 11:23:46 +0000193 unsigned BRCT;
194 if (Opcode == SystemZ::AHI)
195 BRCT = SystemZ::BRCT;
196 else if (Opcode == SystemZ::AGHI)
197 BRCT = SystemZ::BRCTG;
Ulrich Weigand75839912016-11-28 13:40:08 +0000198 else if (Opcode == SystemZ::AIH)
199 BRCT = SystemZ::BRCTH;
Richard Sandifordc2121252013-08-05 11:23:46 +0000200 else
201 return false;
Duncan P. N. Exon Smith4565ec02016-07-12 01:39:01 +0000202 if (MI.getOperand(2).getImm() != -1)
Richard Sandifordc2121252013-08-05 11:23:46 +0000203 return false;
204
205 // Check whether we have a single JLH.
206 if (CCUsers.size() != 1)
207 return false;
208 MachineInstr *Branch = CCUsers[0];
209 if (Branch->getOpcode() != SystemZ::BRC ||
210 Branch->getOperand(0).getImm() != SystemZ::CCMASK_ICMP ||
211 Branch->getOperand(1).getImm() != SystemZ::CCMASK_CMP_NE)
212 return false;
213
214 // We already know that there are no references to the register between
215 // MI and Compare. Make sure that there are also no references between
216 // Compare and Branch.
Jonas Paulsson5d3fbd32015-10-08 07:40:23 +0000217 unsigned SrcReg = getCompareSourceReg(Compare);
Richard Sandifordc2121252013-08-05 11:23:46 +0000218 MachineBasicBlock::iterator MBBI = Compare, MBBE = Branch;
219 for (++MBBI; MBBI != MBBE; ++MBBI)
Duncan P. N. Exon Smith4565ec02016-07-12 01:39:01 +0000220 if (getRegReferences(*MBBI, SrcReg))
Richard Sandifordc2121252013-08-05 11:23:46 +0000221 return false;
222
Ulrich Weigand75839912016-11-28 13:40:08 +0000223 // The transformation is OK. Rebuild Branch as a BRCT(G) or BRCTH.
Richard Sandifordc2121252013-08-05 11:23:46 +0000224 MachineOperand Target(Branch->getOperand(2));
Jonas Paulsson63a2b682015-10-10 07:14:24 +0000225 while (Branch->getNumOperands())
226 Branch->RemoveOperand(0);
Richard Sandifordc2121252013-08-05 11:23:46 +0000227 Branch->setDesc(TII->get(BRCT));
Ulrich Weigand75839912016-11-28 13:40:08 +0000228 MachineInstrBuilder MIB(*Branch->getParent()->getParent(), Branch);
Diana Picus116bbab2017-01-13 09:58:52 +0000229 MIB.add(MI.getOperand(0)).add(MI.getOperand(1)).add(Target);
Ulrich Weigand75839912016-11-28 13:40:08 +0000230 // Add a CC def to BRCT(G), since we may have to split them again if the
231 // branch displacement overflows. BRCTH has a 32-bit displacement, so
232 // this is not necessary there.
233 if (BRCT != SystemZ::BRCTH)
234 MIB.addReg(SystemZ::CC, RegState::ImplicitDefine | RegState::Dead);
Duncan P. N. Exon Smith4565ec02016-07-12 01:39:01 +0000235 MI.eraseFromParent();
Richard Sandifordc2121252013-08-05 11:23:46 +0000236 return true;
237}
238
Ulrich Weigand2d9e3d92016-11-28 13:59:22 +0000239// Compare compares the result of MI against zero. If MI is a suitable load
240// instruction and if CCUsers is a single conditional trap on zero, eliminate
241// the load and convert the branch to a load-and-trap. Return true on success.
242bool SystemZElimCompare::convertToLoadAndTrap(
243 MachineInstr &MI, MachineInstr &Compare,
244 SmallVectorImpl<MachineInstr *> &CCUsers) {
245 unsigned LATOpcode = TII->getLoadAndTrap(MI.getOpcode());
246 if (!LATOpcode)
247 return false;
248
249 // Check whether we have a single CondTrap that traps on zero.
250 if (CCUsers.size() != 1)
251 return false;
252 MachineInstr *Branch = CCUsers[0];
253 if (Branch->getOpcode() != SystemZ::CondTrap ||
254 Branch->getOperand(0).getImm() != SystemZ::CCMASK_ICMP ||
255 Branch->getOperand(1).getImm() != SystemZ::CCMASK_CMP_EQ)
256 return false;
257
258 // We already know that there are no references to the register between
259 // MI and Compare. Make sure that there are also no references between
260 // Compare and Branch.
261 unsigned SrcReg = getCompareSourceReg(Compare);
262 MachineBasicBlock::iterator MBBI = Compare, MBBE = Branch;
263 for (++MBBI; MBBI != MBBE; ++MBBI)
264 if (getRegReferences(*MBBI, SrcReg))
265 return false;
266
267 // The transformation is OK. Rebuild Branch as a load-and-trap.
Ulrich Weigand2d9e3d92016-11-28 13:59:22 +0000268 while (Branch->getNumOperands())
269 Branch->RemoveOperand(0);
270 Branch->setDesc(TII->get(LATOpcode));
271 MachineInstrBuilder(*Branch->getParent()->getParent(), Branch)
Diana Picus116bbab2017-01-13 09:58:52 +0000272 .add(MI.getOperand(0))
273 .add(MI.getOperand(1))
274 .add(MI.getOperand(2))
275 .add(MI.getOperand(3));
Ulrich Weigand2d9e3d92016-11-28 13:59:22 +0000276 MI.eraseFromParent();
277 return true;
278}
279
Richard Sandifordb49a3ab2013-08-05 11:03:20 +0000280// If MI is a load instruction, try to convert it into a LOAD AND TEST.
281// Return true on success.
Jonas Paulsson776a81a2018-01-15 15:41:26 +0000282bool SystemZElimCompare::convertToLoadAndTest(
283 MachineInstr &MI, MachineInstr &Compare,
284 SmallVectorImpl<MachineInstr *> &CCUsers) {
285
286 // Try to adjust CC masks for the LOAD AND TEST opcode that could replace MI.
Duncan P. N. Exon Smith4565ec02016-07-12 01:39:01 +0000287 unsigned Opcode = TII->getLoadAndTest(MI.getOpcode());
Jonas Paulsson776a81a2018-01-15 15:41:26 +0000288 if (!Opcode || !adjustCCMasksForInstr(MI, Compare, CCUsers, Opcode))
Richard Sandifordb49a3ab2013-08-05 11:03:20 +0000289 return false;
290
Jonas Paulssone80d4052018-06-07 05:59:07 +0000291 // Rebuild to get the CC operand in the right place.
Chandler Carruthc73c0302018-08-16 21:30:05 +0000292 auto MIB = BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), TII->get(Opcode));
Jonas Paulssone80d4052018-06-07 05:59:07 +0000293 for (const auto &MO : MI.operands())
Chandler Carruthc73c0302018-08-16 21:30:05 +0000294 MIB.add(MO);
295 MIB.setMemRefs(MI.memoperands());
Jonas Paulssone80d4052018-06-07 05:59:07 +0000296 MI.eraseFromParent();
297
Richard Sandifordb49a3ab2013-08-05 11:03:20 +0000298 return true;
299}
300
Richard Sandifordbdbb8af2013-08-05 10:58:53 +0000301// The CC users in CCUsers are testing the result of a comparison of some
Jonas Paulsson776a81a2018-01-15 15:41:26 +0000302// value X against zero and we know that any CC value produced by MI would
303// also reflect the value of X. ConvOpc may be used to pass the transfomed
304// opcode MI will have if this succeeds. Try to adjust CCUsers so that they
305// test the result of MI directly, returning true on success. Leave
306// everything unchanged on failure.
Duncan P. N. Exon Smith4565ec02016-07-12 01:39:01 +0000307bool SystemZElimCompare::adjustCCMasksForInstr(
308 MachineInstr &MI, MachineInstr &Compare,
Jonas Paulsson776a81a2018-01-15 15:41:26 +0000309 SmallVectorImpl<MachineInstr *> &CCUsers,
310 unsigned ConvOpc) {
311 int Opcode = (ConvOpc ? ConvOpc : MI.getOpcode());
Richard Sandifordbdbb8af2013-08-05 10:58:53 +0000312 const MCInstrDesc &Desc = TII->get(Opcode);
313 unsigned MIFlags = Desc.TSFlags;
314
315 // See which compare-style condition codes are available.
Richard Sandiford0897fce2013-08-07 11:10:06 +0000316 unsigned ReusableCCMask = SystemZII::getCompareZeroCCMask(MIFlags);
Richard Sandifordbdbb8af2013-08-05 10:58:53 +0000317
318 // For unsigned comparisons with zero, only equality makes sense.
Duncan P. N. Exon Smith4565ec02016-07-12 01:39:01 +0000319 unsigned CompareFlags = Compare.getDesc().TSFlags;
Richard Sandiford0897fce2013-08-07 11:10:06 +0000320 if (CompareFlags & SystemZII::IsLogical)
321 ReusableCCMask &= SystemZ::CCMASK_CMP_EQ;
Richard Sandifordbdbb8af2013-08-05 10:58:53 +0000322
323 if (ReusableCCMask == 0)
324 return false;
325
326 unsigned CCValues = SystemZII::getCCValues(MIFlags);
327 assert((ReusableCCMask & ~CCValues) == 0 && "Invalid CCValues");
328
Jonas Paulsson776a81a2018-01-15 15:41:26 +0000329 bool MIEquivalentToCmp =
330 (ReusableCCMask == CCValues &&
331 CCValues == SystemZII::getCCValues(CompareFlags));
Richard Sandifordbdbb8af2013-08-05 10:58:53 +0000332
Jonas Paulsson776a81a2018-01-15 15:41:26 +0000333 if (!MIEquivalentToCmp) {
334 // Now check whether these flags are enough for all users.
335 SmallVector<MachineOperand *, 4> AlterMasks;
336 for (unsigned int I = 0, E = CCUsers.size(); I != E; ++I) {
337 MachineInstr *MI = CCUsers[I];
Richard Sandifordbdbb8af2013-08-05 10:58:53 +0000338
Jonas Paulsson776a81a2018-01-15 15:41:26 +0000339 // Fail if this isn't a use of CC that we understand.
340 unsigned Flags = MI->getDesc().TSFlags;
341 unsigned FirstOpNum;
342 if (Flags & SystemZII::CCMaskFirst)
343 FirstOpNum = 0;
344 else if (Flags & SystemZII::CCMaskLast)
345 FirstOpNum = MI->getNumExplicitOperands() - 2;
346 else
347 return false;
Richard Sandifordbdbb8af2013-08-05 10:58:53 +0000348
Jonas Paulsson776a81a2018-01-15 15:41:26 +0000349 // Check whether the instruction predicate treats all CC values
350 // outside of ReusableCCMask in the same way. In that case it
351 // doesn't matter what those CC values mean.
352 unsigned CCValid = MI->getOperand(FirstOpNum).getImm();
353 unsigned CCMask = MI->getOperand(FirstOpNum + 1).getImm();
354 unsigned OutValid = ~ReusableCCMask & CCValid;
355 unsigned OutMask = ~ReusableCCMask & CCMask;
356 if (OutMask != 0 && OutMask != OutValid)
357 return false;
Richard Sandifordbdbb8af2013-08-05 10:58:53 +0000358
Jonas Paulsson776a81a2018-01-15 15:41:26 +0000359 AlterMasks.push_back(&MI->getOperand(FirstOpNum));
360 AlterMasks.push_back(&MI->getOperand(FirstOpNum + 1));
361 }
362
363 // All users are OK. Adjust the masks for MI.
364 for (unsigned I = 0, E = AlterMasks.size(); I != E; I += 2) {
365 AlterMasks[I]->setImm(CCValues);
366 unsigned CCMask = AlterMasks[I + 1]->getImm();
367 if (CCMask & ~ReusableCCMask)
368 AlterMasks[I + 1]->setImm((CCMask & ReusableCCMask) |
369 (CCValues & ~ReusableCCMask));
370 }
Richard Sandifordbdbb8af2013-08-05 10:58:53 +0000371 }
372
373 // CC is now live after MI.
Jonas Paulssonca6f4522019-09-09 07:58:57 +0000374 if (!ConvOpc)
375 MI.clearRegisterDeads(SystemZ::CC);
Jonas Paulsson776a81a2018-01-15 15:41:26 +0000376
377 // Check if MI lies before Compare.
378 bool BeforeCmp = false;
379 MachineBasicBlock::iterator MBBI = MI, MBBE = MI.getParent()->end();
380 for (++MBBI; MBBI != MBBE; ++MBBI)
381 if (MBBI == Compare) {
382 BeforeCmp = true;
383 break;
384 }
Richard Sandifordbdbb8af2013-08-05 10:58:53 +0000385
386 // Clear any intervening kills of CC.
Jonas Paulsson776a81a2018-01-15 15:41:26 +0000387 if (BeforeCmp) {
388 MachineBasicBlock::iterator MBBI = MI, MBBE = Compare;
389 for (++MBBI; MBBI != MBBE; ++MBBI)
390 MBBI->clearRegisterKills(SystemZ::CC, TRI);
391 }
Richard Sandifordbdbb8af2013-08-05 10:58:53 +0000392
393 return true;
394}
395
Richard Sandiford0897fce2013-08-07 11:10:06 +0000396// Return true if Compare is a comparison against zero.
Duncan P. N. Exon Smith4565ec02016-07-12 01:39:01 +0000397static bool isCompareZero(MachineInstr &Compare) {
398 switch (Compare.getOpcode()) {
Richard Sandiford0897fce2013-08-07 11:10:06 +0000399 case SystemZ::LTEBRCompare:
400 case SystemZ::LTDBRCompare:
401 case SystemZ::LTXBRCompare:
402 return true;
403
404 default:
Jonas Paulsson5d3fbd32015-10-08 07:40:23 +0000405 if (isLoadAndTestAsCmp(Compare))
406 return true;
Duncan P. N. Exon Smith4565ec02016-07-12 01:39:01 +0000407 return Compare.getNumExplicitOperands() == 2 &&
408 Compare.getOperand(1).isImm() && Compare.getOperand(1).getImm() == 0;
Richard Sandiford0897fce2013-08-07 11:10:06 +0000409 }
410}
411
Richard Sandifordbdbb8af2013-08-05 10:58:53 +0000412// Try to optimize cases where comparison instruction Compare is testing
413// a value against zero. Return true on success and if Compare should be
414// deleted as dead. CCUsers is the list of instructions that use the CC
415// value produced by Compare.
Duncan P. N. Exon Smith4565ec02016-07-12 01:39:01 +0000416bool SystemZElimCompare::optimizeCompareZero(
417 MachineInstr &Compare, SmallVectorImpl<MachineInstr *> &CCUsers) {
Richard Sandiford0897fce2013-08-07 11:10:06 +0000418 if (!isCompareZero(Compare))
Richard Sandifordbdbb8af2013-08-05 10:58:53 +0000419 return false;
420
421 // Search back for CC results that are based on the first operand.
Jonas Paulsson5d3fbd32015-10-08 07:40:23 +0000422 unsigned SrcReg = getCompareSourceReg(Compare);
Duncan P. N. Exon Smith4565ec02016-07-12 01:39:01 +0000423 MachineBasicBlock &MBB = *Compare.getParent();
Richard Sandifordc2121252013-08-05 11:23:46 +0000424 Reference CCRefs;
425 Reference SrcRefs;
Jonas Paulssone80d4052018-06-07 05:59:07 +0000426 for (MachineBasicBlock::reverse_iterator MBBI =
427 std::next(MachineBasicBlock::reverse_iterator(&Compare)),
428 MBBE = MBB.rend(); MBBI != MBBE;) {
429 MachineInstr &MI = *MBBI++;
Jonas Paulsson2c96dd62015-10-08 07:40:11 +0000430 if (resultTests(MI, SrcReg)) {
Richard Sandifordc2121252013-08-05 11:23:46 +0000431 // Try to remove both MI and Compare by converting a branch to BRCT(G).
Ulrich Weigand2d9e3d92016-11-28 13:59:22 +0000432 // or a load-and-trap instruction. We don't care in this case whether
433 // CC is modified between MI and Compare.
434 if (!CCRefs.Use && !SrcRefs) {
435 if (convertToBRCT(MI, Compare, CCUsers)) {
436 BranchOnCounts += 1;
437 return true;
438 }
439 if (convertToLoadAndTrap(MI, Compare, CCUsers)) {
440 LoadAndTraps += 1;
441 return true;
442 }
Richard Sandifordc2121252013-08-05 11:23:46 +0000443 }
444 // Try to eliminate Compare by reusing a CC result from MI.
Jonas Paulsson776a81a2018-01-15 15:41:26 +0000445 if ((!CCRefs && convertToLoadAndTest(MI, Compare, CCUsers)) ||
Richard Sandifordc2121252013-08-05 11:23:46 +0000446 (!CCRefs.Def && adjustCCMasksForInstr(MI, Compare, CCUsers))) {
447 EliminatedComparisons += 1;
448 return true;
449 }
Richard Sandifordbdbb8af2013-08-05 10:58:53 +0000450 }
Richard Sandifordc2121252013-08-05 11:23:46 +0000451 SrcRefs |= getRegReferences(MI, SrcReg);
452 if (SrcRefs.Def)
Jonas Paulssonb0e8a2e2017-09-21 13:52:24 +0000453 break;
Richard Sandifordc2121252013-08-05 11:23:46 +0000454 CCRefs |= getRegReferences(MI, SystemZ::CC);
455 if (CCRefs.Use && CCRefs.Def)
Jonas Paulssonb0e8a2e2017-09-21 13:52:24 +0000456 break;
457 }
458
459 // Also do a forward search to handle cases where an instruction after the
Jonas Paulsson22f208f2018-01-08 12:52:40 +0000460 // compare can be converted, like
461 // LTEBRCompare %f0s, %f0s; %f2s = LER %f0s => LTEBRCompare %f2s, %f0s
Jonas Paulssone80d4052018-06-07 05:59:07 +0000462 for (MachineBasicBlock::iterator MBBI =
463 std::next(MachineBasicBlock::iterator(&Compare)), MBBE = MBB.end();
464 MBBI != MBBE;) {
465 MachineInstr &MI = *MBBI++;
Jonas Paulssonb0e8a2e2017-09-21 13:52:24 +0000466 if (preservesValueOf(MI, SrcReg)) {
467 // Try to eliminate Compare by reusing a CC result from MI.
Jonas Paulsson776a81a2018-01-15 15:41:26 +0000468 if (convertToLoadAndTest(MI, Compare, CCUsers)) {
Jonas Paulssonb0e8a2e2017-09-21 13:52:24 +0000469 EliminatedComparisons += 1;
470 return true;
471 }
472 }
473 if (getRegReferences(MI, SrcReg).Def)
474 return false;
475 if (getRegReferences(MI, SystemZ::CC))
Richard Sandifordc2121252013-08-05 11:23:46 +0000476 return false;
Richard Sandifordbdbb8af2013-08-05 10:58:53 +0000477 }
Jonas Paulssonb0e8a2e2017-09-21 13:52:24 +0000478
Richard Sandifordbdbb8af2013-08-05 10:58:53 +0000479 return false;
480}
481
482// Try to fuse comparison instruction Compare into a later branch.
483// Return true on success and if Compare is therefore redundant.
Duncan P. N. Exon Smith4565ec02016-07-12 01:39:01 +0000484bool SystemZElimCompare::fuseCompareOperations(
485 MachineInstr &Compare, SmallVectorImpl<MachineInstr *> &CCUsers) {
Richard Sandifordbdbb8af2013-08-05 10:58:53 +0000486 // See whether we have a single branch with which to fuse.
487 if (CCUsers.size() != 1)
488 return false;
489 MachineInstr *Branch = CCUsers[0];
Zhan Jun Liauab42cbc2016-06-10 19:58:10 +0000490 SystemZII::FusedCompareType Type;
Ulrich Weigand2eb027d2016-04-07 16:11:44 +0000491 switch (Branch->getOpcode()) {
492 case SystemZ::BRC:
493 Type = SystemZII::CompareAndBranch;
494 break;
495 case SystemZ::CondReturn:
496 Type = SystemZII::CompareAndReturn;
497 break;
Ulrich Weigand848a5132016-04-11 12:12:32 +0000498 case SystemZ::CallBCR:
499 Type = SystemZII::CompareAndSibcall;
500 break;
Zhan Jun Liauab42cbc2016-06-10 19:58:10 +0000501 case SystemZ::CondTrap:
502 Type = SystemZII::CompareAndTrap;
503 break;
Ulrich Weigand2eb027d2016-04-07 16:11:44 +0000504 default:
505 return false;
506 }
507
508 // See whether we have a comparison that can be fused.
Duncan P. N. Exon Smith4565ec02016-07-12 01:39:01 +0000509 unsigned FusedOpcode =
510 TII->getFusedCompare(Compare.getOpcode(), Type, &Compare);
Ulrich Weigand2eb027d2016-04-07 16:11:44 +0000511 if (!FusedOpcode)
Richard Sandifordbdbb8af2013-08-05 10:58:53 +0000512 return false;
513
514 // Make sure that the operands are available at the branch.
Ulrich Weiganda0e73252016-11-11 12:48:26 +0000515 // SrcReg2 is the register if the source operand is a register,
516 // 0 if the source operand is immediate, and the base register
517 // if the source operand is memory (index is not supported).
Matt Arsenaulte3a676e2019-06-24 15:50:29 +0000518 Register SrcReg = Compare.getOperand(0).getReg();
519 Register SrcReg2 =
520 Compare.getOperand(1).isReg() ? Compare.getOperand(1).getReg() : Register();
Richard Sandifordbdbb8af2013-08-05 10:58:53 +0000521 MachineBasicBlock::iterator MBBI = Compare, MBBE = Branch;
522 for (++MBBI; MBBI != MBBE; ++MBBI)
523 if (MBBI->modifiesRegister(SrcReg, TRI) ||
524 (SrcReg2 && MBBI->modifiesRegister(SrcReg2, TRI)))
525 return false;
526
Ulrich Weigand848a5132016-04-11 12:12:32 +0000527 // Read the branch mask, target (if applicable), regmask (if applicable).
Richard Sandifordbdbb8af2013-08-05 10:58:53 +0000528 MachineOperand CCMask(MBBI->getOperand(1));
Richard Sandifordbdbb8af2013-08-05 10:58:53 +0000529 assert((CCMask.getImm() & ~SystemZ::CCMASK_ICMP) == 0 &&
530 "Invalid condition-code mask for integer comparison");
Ulrich Weigand2eb027d2016-04-07 16:11:44 +0000531 // This is only valid for CompareAndBranch.
532 MachineOperand Target(MBBI->getOperand(
533 Type == SystemZII::CompareAndBranch ? 2 : 0));
Ulrich Weigand848a5132016-04-11 12:12:32 +0000534 const uint32_t *RegMask;
535 if (Type == SystemZII::CompareAndSibcall)
536 RegMask = MBBI->getOperand(2).getRegMask();
Richard Sandifordbdbb8af2013-08-05 10:58:53 +0000537
538 // Clear out all current operands.
539 int CCUse = MBBI->findRegisterUseOperandIdx(SystemZ::CC, false, TRI);
Ulrich Weigand2eb027d2016-04-07 16:11:44 +0000540 assert(CCUse >= 0 && "BRC/BCR must use CC");
Richard Sandifordbdbb8af2013-08-05 10:58:53 +0000541 Branch->RemoveOperand(CCUse);
Ulrich Weigand848a5132016-04-11 12:12:32 +0000542 // Remove target (branch) or regmask (sibcall).
543 if (Type == SystemZII::CompareAndBranch ||
544 Type == SystemZII::CompareAndSibcall)
Ulrich Weigand2eb027d2016-04-07 16:11:44 +0000545 Branch->RemoveOperand(2);
Richard Sandifordbdbb8af2013-08-05 10:58:53 +0000546 Branch->RemoveOperand(1);
547 Branch->RemoveOperand(0);
548
549 // Rebuild Branch as a fused compare and branch.
Ulrich Weiganda0e73252016-11-11 12:48:26 +0000550 // SrcNOps is the number of MI operands of the compare instruction
551 // that we need to copy over.
552 unsigned SrcNOps = 2;
553 if (FusedOpcode == SystemZ::CLT || FusedOpcode == SystemZ::CLGT)
554 SrcNOps = 3;
Richard Sandifordbdbb8af2013-08-05 10:58:53 +0000555 Branch->setDesc(TII->get(FusedOpcode));
Ulrich Weigand2eb027d2016-04-07 16:11:44 +0000556 MachineInstrBuilder MIB(*Branch->getParent()->getParent(), Branch);
Ulrich Weiganda0e73252016-11-11 12:48:26 +0000557 for (unsigned I = 0; I < SrcNOps; I++)
Diana Picus116bbab2017-01-13 09:58:52 +0000558 MIB.add(Compare.getOperand(I));
559 MIB.add(CCMask);
Ulrich Weigand2eb027d2016-04-07 16:11:44 +0000560
561 if (Type == SystemZII::CompareAndBranch) {
562 // Only conditional branches define CC, as they may be converted back
563 // to a non-fused branch because of a long displacement. Conditional
564 // returns don't have that problem.
Diana Picus116bbab2017-01-13 09:58:52 +0000565 MIB.add(Target).addReg(SystemZ::CC,
566 RegState::ImplicitDefine | RegState::Dead);
Ulrich Weigand2eb027d2016-04-07 16:11:44 +0000567 }
Richard Sandifordbdbb8af2013-08-05 10:58:53 +0000568
Ulrich Weigand848a5132016-04-11 12:12:32 +0000569 if (Type == SystemZII::CompareAndSibcall)
570 MIB.addRegMask(RegMask);
571
Richard Sandifordbdbb8af2013-08-05 10:58:53 +0000572 // Clear any intervening kills of SrcReg and SrcReg2.
573 MBBI = Compare;
574 for (++MBBI; MBBI != MBBE; ++MBBI) {
575 MBBI->clearRegisterKills(SrcReg, TRI);
576 if (SrcReg2)
577 MBBI->clearRegisterKills(SrcReg2, TRI);
578 }
579 FusedComparisons += 1;
580 return true;
581}
582
583// Process all comparison instructions in MBB. Return true if something
584// changed.
Richard Sandiford28c111e2014-03-06 11:00:15 +0000585bool SystemZElimCompare::processBlock(MachineBasicBlock &MBB) {
Richard Sandifordbdbb8af2013-08-05 10:58:53 +0000586 bool Changed = false;
587
588 // Walk backwards through the block looking for comparisons, recording
589 // all CC users as we go. The subroutines can delete Compare and
590 // instructions before it.
Jonas Paulssonbf6744d2019-11-04 16:11:12 +0100591 LivePhysRegs LiveRegs(*TRI);
592 LiveRegs.addLiveOuts(MBB);
593 bool CompleteCCUsers = !LiveRegs.contains(SystemZ::CC);
Richard Sandifordbdbb8af2013-08-05 10:58:53 +0000594 SmallVector<MachineInstr *, 4> CCUsers;
Richard Sandiford28c111e2014-03-06 11:00:15 +0000595 MachineBasicBlock::iterator MBBI = MBB.end();
596 while (MBBI != MBB.begin()) {
Duncan P. N. Exon Smith4565ec02016-07-12 01:39:01 +0000597 MachineInstr &MI = *--MBBI;
598 if (CompleteCCUsers && (MI.isCompare() || isLoadAndTestAsCmp(MI)) &&
Richard Sandifordbdbb8af2013-08-05 10:58:53 +0000599 (optimizeCompareZero(MI, CCUsers) ||
Zhan Jun Liauab42cbc2016-06-10 19:58:10 +0000600 fuseCompareOperations(MI, CCUsers))) {
Richard Sandifordbdbb8af2013-08-05 10:58:53 +0000601 ++MBBI;
Duncan P. N. Exon Smith4565ec02016-07-12 01:39:01 +0000602 MI.eraseFromParent();
Richard Sandifordbdbb8af2013-08-05 10:58:53 +0000603 Changed = true;
604 CCUsers.clear();
Richard Sandifordbdbb8af2013-08-05 10:58:53 +0000605 continue;
606 }
607
Duncan P. N. Exon Smith4565ec02016-07-12 01:39:01 +0000608 if (MI.definesRegister(SystemZ::CC)) {
Richard Sandifordbdbb8af2013-08-05 10:58:53 +0000609 CCUsers.clear();
Jonas Paulsson9e1f3bd2015-10-08 07:39:55 +0000610 CompleteCCUsers = true;
Richard Sandifordc2121252013-08-05 11:23:46 +0000611 }
Duncan P. N. Exon Smith4565ec02016-07-12 01:39:01 +0000612 if (MI.readsRegister(SystemZ::CC) && CompleteCCUsers)
613 CCUsers.push_back(&MI);
Richard Sandifordbdbb8af2013-08-05 10:58:53 +0000614 }
615 return Changed;
616}
617
618bool SystemZElimCompare::runOnMachineFunction(MachineFunction &F) {
Matthias Braunf1caa282017-12-15 22:22:58 +0000619 if (skipFunction(F.getFunction()))
Andrew Kaylord9974cc2016-04-26 23:49:41 +0000620 return false;
621
Eric Christopherfc6de422014-08-05 02:39:49 +0000622 TII = static_cast<const SystemZInstrInfo *>(F.getSubtarget().getInstrInfo());
Richard Sandifordbdbb8af2013-08-05 10:58:53 +0000623 TRI = &TII->getRegisterInfo();
624
625 bool Changed = false;
Richard Sandiford28c111e2014-03-06 11:00:15 +0000626 for (auto &MBB : F)
627 Changed |= processBlock(MBB);
Richard Sandifordbdbb8af2013-08-05 10:58:53 +0000628
629 return Changed;
630}
Eugene Zelenko3943d2b2017-01-24 22:10:43 +0000631
632FunctionPass *llvm::createSystemZElimComparePass(SystemZTargetMachine &TM) {
633 return new SystemZElimCompare(TM);
634}