It's not necessary to do rounding for alloca operations when the requested
alignment is equal to the stack alignment.
git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@40004 91177308-0d34-0410-b5e6-96231b3b80d8
diff --git a/lib/CodeGen/AsmPrinter.cpp b/lib/CodeGen/AsmPrinter.cpp
new file mode 100644
index 0000000..586472c
--- /dev/null
+++ b/lib/CodeGen/AsmPrinter.cpp
@@ -0,0 +1,1210 @@
+//===-- AsmPrinter.cpp - Common AsmPrinter code ---------------------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file was developed by the LLVM research group and is distributed under
+// the University of Illinois Open Source License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements the AsmPrinter class.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/CodeGen/AsmPrinter.h"
+#include "llvm/Assembly/Writer.h"
+#include "llvm/DerivedTypes.h"
+#include "llvm/Constants.h"
+#include "llvm/Module.h"
+#include "llvm/CodeGen/MachineConstantPool.h"
+#include "llvm/CodeGen/MachineJumpTableInfo.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Mangler.h"
+#include "llvm/Support/MathExtras.h"
+#include "llvm/Support/Streams.h"
+#include "llvm/Target/TargetAsmInfo.h"
+#include "llvm/Target/TargetData.h"
+#include "llvm/Target/TargetLowering.h"
+#include "llvm/Target/TargetMachine.h"
+#include <cerrno>
+using namespace llvm;
+
+static cl::opt<bool>
+AsmVerbose("asm-verbose", cl::Hidden, cl::desc("Add comments to directives."));
+
+char AsmPrinter::ID = 0;
+AsmPrinter::AsmPrinter(std::ostream &o, TargetMachine &tm,
+ const TargetAsmInfo *T)
+ : MachineFunctionPass((intptr_t)&ID), FunctionNumber(0), O(o), TM(tm), TAI(T)
+{}
+
+std::string AsmPrinter::getSectionForFunction(const Function &F) const {
+ return TAI->getTextSection();
+}
+
+
+/// SwitchToTextSection - Switch to the specified text section of the executable
+/// if we are not already in it!
+///
+void AsmPrinter::SwitchToTextSection(const char *NewSection,
+ const GlobalValue *GV) {
+ std::string NS;
+ if (GV && GV->hasSection())
+ NS = TAI->getSwitchToSectionDirective() + GV->getSection();
+ else
+ NS = NewSection;
+
+ // If we're already in this section, we're done.
+ if (CurrentSection == NS) return;
+
+ // Close the current section, if applicable.
+ if (TAI->getSectionEndDirectiveSuffix() && !CurrentSection.empty())
+ O << CurrentSection << TAI->getSectionEndDirectiveSuffix() << "\n";
+
+ CurrentSection = NS;
+
+ if (!CurrentSection.empty())
+ O << CurrentSection << TAI->getTextSectionStartSuffix() << '\n';
+}
+
+/// SwitchToDataSection - Switch to the specified data section of the executable
+/// if we are not already in it!
+///
+void AsmPrinter::SwitchToDataSection(const char *NewSection,
+ const GlobalValue *GV) {
+ std::string NS;
+ if (GV && GV->hasSection())
+ NS = TAI->getSwitchToSectionDirective() + GV->getSection();
+ else
+ NS = NewSection;
+
+ // If we're already in this section, we're done.
+ if (CurrentSection == NS) return;
+
+ // Close the current section, if applicable.
+ if (TAI->getSectionEndDirectiveSuffix() && !CurrentSection.empty())
+ O << CurrentSection << TAI->getSectionEndDirectiveSuffix() << "\n";
+
+ CurrentSection = NS;
+
+ if (!CurrentSection.empty())
+ O << CurrentSection << TAI->getDataSectionStartSuffix() << '\n';
+}
+
+
+bool AsmPrinter::doInitialization(Module &M) {
+ Mang = new Mangler(M, TAI->getGlobalPrefix());
+
+ if (!M.getModuleInlineAsm().empty())
+ O << TAI->getCommentString() << " Start of file scope inline assembly\n"
+ << M.getModuleInlineAsm()
+ << "\n" << TAI->getCommentString()
+ << " End of file scope inline assembly\n";
+
+ SwitchToDataSection(""); // Reset back to no section.
+
+ if (MachineModuleInfo *MMI = getAnalysisToUpdate<MachineModuleInfo>()) {
+ MMI->AnalyzeModule(M);
+ }
+
+ return false;
+}
+
+bool AsmPrinter::doFinalization(Module &M) {
+ if (TAI->getWeakRefDirective()) {
+ if (!ExtWeakSymbols.empty())
+ SwitchToDataSection("");
+
+ for (std::set<const GlobalValue*>::iterator i = ExtWeakSymbols.begin(),
+ e = ExtWeakSymbols.end(); i != e; ++i) {
+ const GlobalValue *GV = *i;
+ std::string Name = Mang->getValueName(GV);
+ O << TAI->getWeakRefDirective() << Name << "\n";
+ }
+ }
+
+ if (TAI->getSetDirective()) {
+ if (!M.alias_empty())
+ SwitchToTextSection(TAI->getTextSection());
+
+ O << "\n";
+ for (Module::const_alias_iterator I = M.alias_begin(), E = M.alias_end();
+ I!=E; ++I) {
+ std::string Name = Mang->getValueName(I);
+ std::string Target;
+
+ if (const GlobalValue *GV = I->getAliasedGlobal())
+ Target = Mang->getValueName(GV);
+ else
+ assert(0 && "Unsupported aliasee");
+
+ if (I->hasExternalLinkage())
+ O << "\t.globl\t" << Name << "\n";
+ else if (I->hasWeakLinkage())
+ O << TAI->getWeakRefDirective() << Name << "\n";
+ else if (!I->hasInternalLinkage())
+ assert(0 && "Invalid alias linkage");
+
+ O << TAI->getSetDirective() << Name << ", " << Target << "\n";
+ }
+ }
+
+ delete Mang; Mang = 0;
+ return false;
+}
+
+void AsmPrinter::SetupMachineFunction(MachineFunction &MF) {
+ // What's my mangled name?
+ CurrentFnName = Mang->getValueName(MF.getFunction());
+ IncrementFunctionNumber();
+}
+
+/// EmitConstantPool - Print to the current output stream assembly
+/// representations of the constants in the constant pool MCP. This is
+/// used to print out constants which have been "spilled to memory" by
+/// the code generator.
+///
+void AsmPrinter::EmitConstantPool(MachineConstantPool *MCP) {
+ const std::vector<MachineConstantPoolEntry> &CP = MCP->getConstants();
+ if (CP.empty()) return;
+
+ // Some targets require 4-, 8-, and 16- byte constant literals to be placed
+ // in special sections.
+ std::vector<std::pair<MachineConstantPoolEntry,unsigned> > FourByteCPs;
+ std::vector<std::pair<MachineConstantPoolEntry,unsigned> > EightByteCPs;
+ std::vector<std::pair<MachineConstantPoolEntry,unsigned> > SixteenByteCPs;
+ std::vector<std::pair<MachineConstantPoolEntry,unsigned> > OtherCPs;
+ std::vector<std::pair<MachineConstantPoolEntry,unsigned> > TargetCPs;
+ for (unsigned i = 0, e = CP.size(); i != e; ++i) {
+ MachineConstantPoolEntry CPE = CP[i];
+ const Type *Ty = CPE.getType();
+ if (TAI->getFourByteConstantSection() &&
+ TM.getTargetData()->getTypeSize(Ty) == 4)
+ FourByteCPs.push_back(std::make_pair(CPE, i));
+ else if (TAI->getEightByteConstantSection() &&
+ TM.getTargetData()->getTypeSize(Ty) == 8)
+ EightByteCPs.push_back(std::make_pair(CPE, i));
+ else if (TAI->getSixteenByteConstantSection() &&
+ TM.getTargetData()->getTypeSize(Ty) == 16)
+ SixteenByteCPs.push_back(std::make_pair(CPE, i));
+ else
+ OtherCPs.push_back(std::make_pair(CPE, i));
+ }
+
+ unsigned Alignment = MCP->getConstantPoolAlignment();
+ EmitConstantPool(Alignment, TAI->getFourByteConstantSection(), FourByteCPs);
+ EmitConstantPool(Alignment, TAI->getEightByteConstantSection(), EightByteCPs);
+ EmitConstantPool(Alignment, TAI->getSixteenByteConstantSection(),
+ SixteenByteCPs);
+ EmitConstantPool(Alignment, TAI->getConstantPoolSection(), OtherCPs);
+}
+
+void AsmPrinter::EmitConstantPool(unsigned Alignment, const char *Section,
+ std::vector<std::pair<MachineConstantPoolEntry,unsigned> > &CP) {
+ if (CP.empty()) return;
+
+ SwitchToDataSection(Section);
+ EmitAlignment(Alignment);
+ for (unsigned i = 0, e = CP.size(); i != e; ++i) {
+ O << TAI->getPrivateGlobalPrefix() << "CPI" << getFunctionNumber() << '_'
+ << CP[i].second << ":\t\t\t\t\t" << TAI->getCommentString() << " ";
+ WriteTypeSymbolic(O, CP[i].first.getType(), 0) << '\n';
+ if (CP[i].first.isMachineConstantPoolEntry())
+ EmitMachineConstantPoolValue(CP[i].first.Val.MachineCPVal);
+ else
+ EmitGlobalConstant(CP[i].first.Val.ConstVal);
+ if (i != e-1) {
+ const Type *Ty = CP[i].first.getType();
+ unsigned EntSize =
+ TM.getTargetData()->getTypeSize(Ty);
+ unsigned ValEnd = CP[i].first.getOffset() + EntSize;
+ // Emit inter-object padding for alignment.
+ EmitZeros(CP[i+1].first.getOffset()-ValEnd);
+ }
+ }
+}
+
+/// EmitJumpTableInfo - Print assembly representations of the jump tables used
+/// by the current function to the current output stream.
+///
+void AsmPrinter::EmitJumpTableInfo(MachineJumpTableInfo *MJTI,
+ MachineFunction &MF) {
+ const std::vector<MachineJumpTableEntry> &JT = MJTI->getJumpTables();
+ if (JT.empty()) return;
+ bool IsPic = TM.getRelocationModel() == Reloc::PIC_;
+
+ // Use JumpTableDirective otherwise honor the entry size from the jump table
+ // info.
+ const char *JTEntryDirective = TAI->getJumpTableDirective();
+ bool HadJTEntryDirective = JTEntryDirective != NULL;
+ if (!HadJTEntryDirective) {
+ JTEntryDirective = MJTI->getEntrySize() == 4 ?
+ TAI->getData32bitsDirective() : TAI->getData64bitsDirective();
+ }
+
+ // Pick the directive to use to print the jump table entries, and switch to
+ // the appropriate section.
+ TargetLowering *LoweringInfo = TM.getTargetLowering();
+
+ const char* JumpTableDataSection = TAI->getJumpTableDataSection();
+ if ((IsPic && !(LoweringInfo && LoweringInfo->usesGlobalOffsetTable())) ||
+ !JumpTableDataSection) {
+ // In PIC mode, we need to emit the jump table to the same section as the
+ // function body itself, otherwise the label differences won't make sense.
+ // We should also do if the section name is NULL.
+ const Function *F = MF.getFunction();
+ SwitchToTextSection(getSectionForFunction(*F).c_str(), F);
+ } else {
+ SwitchToDataSection(JumpTableDataSection);
+ }
+
+ EmitAlignment(Log2_32(MJTI->getAlignment()));
+
+ for (unsigned i = 0, e = JT.size(); i != e; ++i) {
+ const std::vector<MachineBasicBlock*> &JTBBs = JT[i].MBBs;
+
+ // If this jump table was deleted, ignore it.
+ if (JTBBs.empty()) continue;
+
+ // For PIC codegen, if possible we want to use the SetDirective to reduce
+ // the number of relocations the assembler will generate for the jump table.
+ // Set directives are all printed before the jump table itself.
+ std::set<MachineBasicBlock*> EmittedSets;
+ if (TAI->getSetDirective() && IsPic)
+ for (unsigned ii = 0, ee = JTBBs.size(); ii != ee; ++ii)
+ if (EmittedSets.insert(JTBBs[ii]).second)
+ printSetLabel(i, JTBBs[ii]);
+
+ // On some targets (e.g. darwin) we want to emit two consequtive labels
+ // before each jump table. The first label is never referenced, but tells
+ // the assembler and linker the extents of the jump table object. The
+ // second label is actually referenced by the code.
+ if (const char *JTLabelPrefix = TAI->getJumpTableSpecialLabelPrefix())
+ O << JTLabelPrefix << "JTI" << getFunctionNumber() << '_' << i << ":\n";
+
+ O << TAI->getPrivateGlobalPrefix() << "JTI" << getFunctionNumber()
+ << '_' << i << ":\n";
+
+ for (unsigned ii = 0, ee = JTBBs.size(); ii != ee; ++ii) {
+ O << JTEntryDirective << ' ';
+ // If we have emitted set directives for the jump table entries, print
+ // them rather than the entries themselves. If we're emitting PIC, then
+ // emit the table entries as differences between two text section labels.
+ // If we're emitting non-PIC code, then emit the entries as direct
+ // references to the target basic blocks.
+ if (!EmittedSets.empty()) {
+ O << TAI->getPrivateGlobalPrefix() << getFunctionNumber()
+ << '_' << i << "_set_" << JTBBs[ii]->getNumber();
+ } else if (IsPic) {
+ printBasicBlockLabel(JTBBs[ii], false, false);
+ // If the arch uses custom Jump Table directives, don't calc relative to
+ // JT
+ if (!HadJTEntryDirective)
+ O << '-' << TAI->getPrivateGlobalPrefix() << "JTI"
+ << getFunctionNumber() << '_' << i;
+ } else {
+ printBasicBlockLabel(JTBBs[ii], false, false);
+ }
+ O << '\n';
+ }
+ }
+}
+
+/// EmitSpecialLLVMGlobal - Check to see if the specified global is a
+/// special global used by LLVM. If so, emit it and return true, otherwise
+/// do nothing and return false.
+bool AsmPrinter::EmitSpecialLLVMGlobal(const GlobalVariable *GV) {
+ // Ignore debug and non-emitted data.
+ if (GV->getSection() == "llvm.metadata") return true;
+
+ if (!GV->hasAppendingLinkage()) return false;
+
+ assert(GV->hasInitializer() && "Not a special LLVM global!");
+
+ if (GV->getName() == "llvm.used") {
+ if (TAI->getUsedDirective() != 0) // No need to emit this at all.
+ EmitLLVMUsedList(GV->getInitializer());
+ return true;
+ }
+
+ const TargetData *TD = TM.getTargetData();
+ unsigned Align = Log2_32(TD->getPointerPrefAlignment());
+ if (GV->getName() == "llvm.global_ctors" && GV->use_empty()) {
+ SwitchToDataSection(TAI->getStaticCtorsSection());
+ EmitAlignment(Align, 0);
+ EmitXXStructorList(GV->getInitializer());
+ return true;
+ }
+
+ if (GV->getName() == "llvm.global_dtors" && GV->use_empty()) {
+ SwitchToDataSection(TAI->getStaticDtorsSection());
+ EmitAlignment(Align, 0);
+ EmitXXStructorList(GV->getInitializer());
+ return true;
+ }
+
+ return false;
+}
+
+/// EmitLLVMUsedList - For targets that define a TAI::UsedDirective, mark each
+/// global in the specified llvm.used list as being used with this directive.
+void AsmPrinter::EmitLLVMUsedList(Constant *List) {
+ const char *Directive = TAI->getUsedDirective();
+
+ // Should be an array of 'sbyte*'.
+ ConstantArray *InitList = dyn_cast<ConstantArray>(List);
+ if (InitList == 0) return;
+
+ for (unsigned i = 0, e = InitList->getNumOperands(); i != e; ++i) {
+ O << Directive;
+ EmitConstantValueOnly(InitList->getOperand(i));
+ O << "\n";
+ }
+}
+
+/// EmitXXStructorList - Emit the ctor or dtor list. This just prints out the
+/// function pointers, ignoring the init priority.
+void AsmPrinter::EmitXXStructorList(Constant *List) {
+ // Should be an array of '{ int, void ()* }' structs. The first value is the
+ // init priority, which we ignore.
+ if (!isa<ConstantArray>(List)) return;
+ ConstantArray *InitList = cast<ConstantArray>(List);
+ for (unsigned i = 0, e = InitList->getNumOperands(); i != e; ++i)
+ if (ConstantStruct *CS = dyn_cast<ConstantStruct>(InitList->getOperand(i))){
+ if (CS->getNumOperands() != 2) return; // Not array of 2-element structs.
+
+ if (CS->getOperand(1)->isNullValue())
+ return; // Found a null terminator, exit printing.
+ // Emit the function pointer.
+ EmitGlobalConstant(CS->getOperand(1));
+ }
+}
+
+/// getGlobalLinkName - Returns the asm/link name of of the specified
+/// global variable. Should be overridden by each target asm printer to
+/// generate the appropriate value.
+const std::string AsmPrinter::getGlobalLinkName(const GlobalVariable *GV) const{
+ std::string LinkName;
+
+ if (isa<Function>(GV)) {
+ LinkName += TAI->getFunctionAddrPrefix();
+ LinkName += Mang->getValueName(GV);
+ LinkName += TAI->getFunctionAddrSuffix();
+ } else {
+ LinkName += TAI->getGlobalVarAddrPrefix();
+ LinkName += Mang->getValueName(GV);
+ LinkName += TAI->getGlobalVarAddrSuffix();
+ }
+
+ return LinkName;
+}
+
+/// EmitExternalGlobal - Emit the external reference to a global variable.
+/// Should be overridden if an indirect reference should be used.
+void AsmPrinter::EmitExternalGlobal(const GlobalVariable *GV) {
+ O << getGlobalLinkName(GV);
+}
+
+
+
+//===----------------------------------------------------------------------===//
+/// LEB 128 number encoding.
+
+/// PrintULEB128 - Print a series of hexidecimal values (separated by commas)
+/// representing an unsigned leb128 value.
+void AsmPrinter::PrintULEB128(unsigned Value) const {
+ do {
+ unsigned Byte = Value & 0x7f;
+ Value >>= 7;
+ if (Value) Byte |= 0x80;
+ O << "0x" << std::hex << Byte << std::dec;
+ if (Value) O << ", ";
+ } while (Value);
+}
+
+/// SizeULEB128 - Compute the number of bytes required for an unsigned leb128
+/// value.
+unsigned AsmPrinter::SizeULEB128(unsigned Value) {
+ unsigned Size = 0;
+ do {
+ Value >>= 7;
+ Size += sizeof(int8_t);
+ } while (Value);
+ return Size;
+}
+
+/// PrintSLEB128 - Print a series of hexidecimal values (separated by commas)
+/// representing a signed leb128 value.
+void AsmPrinter::PrintSLEB128(int Value) const {
+ int Sign = Value >> (8 * sizeof(Value) - 1);
+ bool IsMore;
+
+ do {
+ unsigned Byte = Value & 0x7f;
+ Value >>= 7;
+ IsMore = Value != Sign || ((Byte ^ Sign) & 0x40) != 0;
+ if (IsMore) Byte |= 0x80;
+ O << "0x" << std::hex << Byte << std::dec;
+ if (IsMore) O << ", ";
+ } while (IsMore);
+}
+
+/// SizeSLEB128 - Compute the number of bytes required for a signed leb128
+/// value.
+unsigned AsmPrinter::SizeSLEB128(int Value) {
+ unsigned Size = 0;
+ int Sign = Value >> (8 * sizeof(Value) - 1);
+ bool IsMore;
+
+ do {
+ unsigned Byte = Value & 0x7f;
+ Value >>= 7;
+ IsMore = Value != Sign || ((Byte ^ Sign) & 0x40) != 0;
+ Size += sizeof(int8_t);
+ } while (IsMore);
+ return Size;
+}
+
+//===--------------------------------------------------------------------===//
+// Emission and print routines
+//
+
+/// PrintHex - Print a value as a hexidecimal value.
+///
+void AsmPrinter::PrintHex(int Value) const {
+ O << "0x" << std::hex << Value << std::dec;
+}
+
+/// EOL - Print a newline character to asm stream. If a comment is present
+/// then it will be printed first. Comments should not contain '\n'.
+void AsmPrinter::EOL() const {
+ O << "\n";
+}
+void AsmPrinter::EOL(const std::string &Comment) const {
+ if (AsmVerbose && !Comment.empty()) {
+ O << "\t"
+ << TAI->getCommentString()
+ << " "
+ << Comment;
+ }
+ O << "\n";
+}
+
+/// EmitULEB128Bytes - Emit an assembler byte data directive to compose an
+/// unsigned leb128 value.
+void AsmPrinter::EmitULEB128Bytes(unsigned Value) const {
+ if (TAI->hasLEB128()) {
+ O << "\t.uleb128\t"
+ << Value;
+ } else {
+ O << TAI->getData8bitsDirective();
+ PrintULEB128(Value);
+ }
+}
+
+/// EmitSLEB128Bytes - print an assembler byte data directive to compose a
+/// signed leb128 value.
+void AsmPrinter::EmitSLEB128Bytes(int Value) const {
+ if (TAI->hasLEB128()) {
+ O << "\t.sleb128\t"
+ << Value;
+ } else {
+ O << TAI->getData8bitsDirective();
+ PrintSLEB128(Value);
+ }
+}
+
+/// EmitInt8 - Emit a byte directive and value.
+///
+void AsmPrinter::EmitInt8(int Value) const {
+ O << TAI->getData8bitsDirective();
+ PrintHex(Value & 0xFF);
+}
+
+/// EmitInt16 - Emit a short directive and value.
+///
+void AsmPrinter::EmitInt16(int Value) const {
+ O << TAI->getData16bitsDirective();
+ PrintHex(Value & 0xFFFF);
+}
+
+/// EmitInt32 - Emit a long directive and value.
+///
+void AsmPrinter::EmitInt32(int Value) const {
+ O << TAI->getData32bitsDirective();
+ PrintHex(Value);
+}
+
+/// EmitInt64 - Emit a long long directive and value.
+///
+void AsmPrinter::EmitInt64(uint64_t Value) const {
+ if (TAI->getData64bitsDirective()) {
+ O << TAI->getData64bitsDirective();
+ PrintHex(Value);
+ } else {
+ if (TM.getTargetData()->isBigEndian()) {
+ EmitInt32(unsigned(Value >> 32)); O << "\n";
+ EmitInt32(unsigned(Value));
+ } else {
+ EmitInt32(unsigned(Value)); O << "\n";
+ EmitInt32(unsigned(Value >> 32));
+ }
+ }
+}
+
+/// toOctal - Convert the low order bits of X into an octal digit.
+///
+static inline char toOctal(int X) {
+ return (X&7)+'0';
+}
+
+/// printStringChar - Print a char, escaped if necessary.
+///
+static void printStringChar(std::ostream &O, unsigned char C) {
+ if (C == '"') {
+ O << "\\\"";
+ } else if (C == '\\') {
+ O << "\\\\";
+ } else if (isprint(C)) {
+ O << C;
+ } else {
+ switch(C) {
+ case '\b': O << "\\b"; break;
+ case '\f': O << "\\f"; break;
+ case '\n': O << "\\n"; break;
+ case '\r': O << "\\r"; break;
+ case '\t': O << "\\t"; break;
+ default:
+ O << '\\';
+ O << toOctal(C >> 6);
+ O << toOctal(C >> 3);
+ O << toOctal(C >> 0);
+ break;
+ }
+ }
+}
+
+/// EmitString - Emit a string with quotes and a null terminator.
+/// Special characters are emitted properly.
+/// \literal (Eg. '\t') \endliteral
+void AsmPrinter::EmitString(const std::string &String) const {
+ const char* AscizDirective = TAI->getAscizDirective();
+ if (AscizDirective)
+ O << AscizDirective;
+ else
+ O << TAI->getAsciiDirective();
+ O << "\"";
+ for (unsigned i = 0, N = String.size(); i < N; ++i) {
+ unsigned char C = String[i];
+ printStringChar(O, C);
+ }
+ if (AscizDirective)
+ O << "\"";
+ else
+ O << "\\0\"";
+}
+
+
+//===----------------------------------------------------------------------===//
+
+// EmitAlignment - Emit an alignment directive to the specified power of
+// two boundary. For example, if you pass in 3 here, you will get an 8
+// byte alignment. If a global value is specified, and if that global has
+// an explicit alignment requested, it will unconditionally override the
+// alignment request. However, if ForcedAlignBits is specified, this value
+// has final say: the ultimate alignment will be the max of ForcedAlignBits
+// and the alignment computed with NumBits and the global.
+//
+// The algorithm is:
+// Align = NumBits;
+// if (GV && GV->hasalignment) Align = GV->getalignment();
+// Align = std::max(Align, ForcedAlignBits);
+//
+void AsmPrinter::EmitAlignment(unsigned NumBits, const GlobalValue *GV,
+ unsigned ForcedAlignBits) const {
+ if (GV && GV->getAlignment())
+ NumBits = Log2_32(GV->getAlignment());
+ NumBits = std::max(NumBits, ForcedAlignBits);
+
+ if (NumBits == 0) return; // No need to emit alignment.
+ if (TAI->getAlignmentIsInBytes()) NumBits = 1 << NumBits;
+ O << TAI->getAlignDirective() << NumBits << "\n";
+}
+
+
+/// EmitZeros - Emit a block of zeros.
+///
+void AsmPrinter::EmitZeros(uint64_t NumZeros) const {
+ if (NumZeros) {
+ if (TAI->getZeroDirective()) {
+ O << TAI->getZeroDirective() << NumZeros;
+ if (TAI->getZeroDirectiveSuffix())
+ O << TAI->getZeroDirectiveSuffix();
+ O << "\n";
+ } else {
+ for (; NumZeros; --NumZeros)
+ O << TAI->getData8bitsDirective() << "0\n";
+ }
+ }
+}
+
+// Print out the specified constant, without a storage class. Only the
+// constants valid in constant expressions can occur here.
+void AsmPrinter::EmitConstantValueOnly(const Constant *CV) {
+ if (CV->isNullValue() || isa<UndefValue>(CV))
+ O << "0";
+ else if (const ConstantInt *CI = dyn_cast<ConstantInt>(CV)) {
+ O << CI->getZExtValue();
+ } else if (const GlobalValue *GV = dyn_cast<GlobalValue>(CV)) {
+ // This is a constant address for a global variable or function. Use the
+ // name of the variable or function as the address value, possibly
+ // decorating it with GlobalVarAddrPrefix/Suffix or
+ // FunctionAddrPrefix/Suffix (these all default to "" )
+ if (isa<Function>(GV)) {
+ O << TAI->getFunctionAddrPrefix()
+ << Mang->getValueName(GV)
+ << TAI->getFunctionAddrSuffix();
+ } else {
+ O << TAI->getGlobalVarAddrPrefix()
+ << Mang->getValueName(GV)
+ << TAI->getGlobalVarAddrSuffix();
+ }
+ } else if (const ConstantExpr *CE = dyn_cast<ConstantExpr>(CV)) {
+ const TargetData *TD = TM.getTargetData();
+ unsigned Opcode = CE->getOpcode();
+ switch (Opcode) {
+ case Instruction::GetElementPtr: {
+ // generate a symbolic expression for the byte address
+ const Constant *ptrVal = CE->getOperand(0);
+ SmallVector<Value*, 8> idxVec(CE->op_begin()+1, CE->op_end());
+ if (int64_t Offset = TD->getIndexedOffset(ptrVal->getType(), &idxVec[0],
+ idxVec.size())) {
+ if (Offset)
+ O << "(";
+ EmitConstantValueOnly(ptrVal);
+ if (Offset > 0)
+ O << ") + " << Offset;
+ else if (Offset < 0)
+ O << ") - " << -Offset;
+ } else {
+ EmitConstantValueOnly(ptrVal);
+ }
+ break;
+ }
+ case Instruction::Trunc:
+ case Instruction::ZExt:
+ case Instruction::SExt:
+ case Instruction::FPTrunc:
+ case Instruction::FPExt:
+ case Instruction::UIToFP:
+ case Instruction::SIToFP:
+ case Instruction::FPToUI:
+ case Instruction::FPToSI:
+ assert(0 && "FIXME: Don't yet support this kind of constant cast expr");
+ break;
+ case Instruction::BitCast:
+ return EmitConstantValueOnly(CE->getOperand(0));
+
+ case Instruction::IntToPtr: {
+ // Handle casts to pointers by changing them into casts to the appropriate
+ // integer type. This promotes constant folding and simplifies this code.
+ Constant *Op = CE->getOperand(0);
+ Op = ConstantExpr::getIntegerCast(Op, TD->getIntPtrType(), false/*ZExt*/);
+ return EmitConstantValueOnly(Op);
+ }
+
+
+ case Instruction::PtrToInt: {
+ // Support only foldable casts to/from pointers that can be eliminated by
+ // changing the pointer to the appropriately sized integer type.
+ Constant *Op = CE->getOperand(0);
+ const Type *Ty = CE->getType();
+
+ // We can emit the pointer value into this slot if the slot is an
+ // integer slot greater or equal to the size of the pointer.
+ if (Ty->isInteger() &&
+ TD->getTypeSize(Ty) >= TD->getTypeSize(Op->getType()))
+ return EmitConstantValueOnly(Op);
+
+ assert(0 && "FIXME: Don't yet support this kind of constant cast expr");
+ EmitConstantValueOnly(Op);
+ break;
+ }
+ case Instruction::Add:
+ case Instruction::Sub:
+ O << "(";
+ EmitConstantValueOnly(CE->getOperand(0));
+ O << (Opcode==Instruction::Add ? ") + (" : ") - (");
+ EmitConstantValueOnly(CE->getOperand(1));
+ O << ")";
+ break;
+ default:
+ assert(0 && "Unsupported operator!");
+ }
+ } else {
+ assert(0 && "Unknown constant value!");
+ }
+}
+
+/// printAsCString - Print the specified array as a C compatible string, only if
+/// the predicate isString is true.
+///
+static void printAsCString(std::ostream &O, const ConstantArray *CVA,
+ unsigned LastElt) {
+ assert(CVA->isString() && "Array is not string compatible!");
+
+ O << "\"";
+ for (unsigned i = 0; i != LastElt; ++i) {
+ unsigned char C =
+ (unsigned char)cast<ConstantInt>(CVA->getOperand(i))->getZExtValue();
+ printStringChar(O, C);
+ }
+ O << "\"";
+}
+
+/// EmitString - Emit a zero-byte-terminated string constant.
+///
+void AsmPrinter::EmitString(const ConstantArray *CVA) const {
+ unsigned NumElts = CVA->getNumOperands();
+ if (TAI->getAscizDirective() && NumElts &&
+ cast<ConstantInt>(CVA->getOperand(NumElts-1))->getZExtValue() == 0) {
+ O << TAI->getAscizDirective();
+ printAsCString(O, CVA, NumElts-1);
+ } else {
+ O << TAI->getAsciiDirective();
+ printAsCString(O, CVA, NumElts);
+ }
+ O << "\n";
+}
+
+/// EmitGlobalConstant - Print a general LLVM constant to the .s file.
+///
+void AsmPrinter::EmitGlobalConstant(const Constant *CV) {
+ const TargetData *TD = TM.getTargetData();
+
+ if (CV->isNullValue() || isa<UndefValue>(CV)) {
+ EmitZeros(TD->getTypeSize(CV->getType()));
+ return;
+ } else if (const ConstantArray *CVA = dyn_cast<ConstantArray>(CV)) {
+ if (CVA->isString()) {
+ EmitString(CVA);
+ } else { // Not a string. Print the values in successive locations
+ for (unsigned i = 0, e = CVA->getNumOperands(); i != e; ++i)
+ EmitGlobalConstant(CVA->getOperand(i));
+ }
+ return;
+ } else if (const ConstantStruct *CVS = dyn_cast<ConstantStruct>(CV)) {
+ // Print the fields in successive locations. Pad to align if needed!
+ const StructLayout *cvsLayout = TD->getStructLayout(CVS->getType());
+ uint64_t sizeSoFar = 0;
+ for (unsigned i = 0, e = CVS->getNumOperands(); i != e; ++i) {
+ const Constant* field = CVS->getOperand(i);
+
+ // Check if padding is needed and insert one or more 0s.
+ uint64_t fieldSize = TD->getTypeSize(field->getType());
+ uint64_t padSize = ((i == e-1? cvsLayout->getSizeInBytes()
+ : cvsLayout->getElementOffset(i+1))
+ - cvsLayout->getElementOffset(i)) - fieldSize;
+ sizeSoFar += fieldSize + padSize;
+
+ // Now print the actual field value
+ EmitGlobalConstant(field);
+
+ // Insert the field padding unless it's zero bytes...
+ EmitZeros(padSize);
+ }
+ assert(sizeSoFar == cvsLayout->getSizeInBytes() &&
+ "Layout of constant struct may be incorrect!");
+ return;
+ } else if (const ConstantFP *CFP = dyn_cast<ConstantFP>(CV)) {
+ // FP Constants are printed as integer constants to avoid losing
+ // precision...
+ double Val = CFP->getValue();
+ if (CFP->getType() == Type::DoubleTy) {
+ if (TAI->getData64bitsDirective())
+ O << TAI->getData64bitsDirective() << DoubleToBits(Val) << "\t"
+ << TAI->getCommentString() << " double value: " << Val << "\n";
+ else if (TD->isBigEndian()) {
+ O << TAI->getData32bitsDirective() << unsigned(DoubleToBits(Val) >> 32)
+ << "\t" << TAI->getCommentString()
+ << " double most significant word " << Val << "\n";
+ O << TAI->getData32bitsDirective() << unsigned(DoubleToBits(Val))
+ << "\t" << TAI->getCommentString()
+ << " double least significant word " << Val << "\n";
+ } else {
+ O << TAI->getData32bitsDirective() << unsigned(DoubleToBits(Val))
+ << "\t" << TAI->getCommentString()
+ << " double least significant word " << Val << "\n";
+ O << TAI->getData32bitsDirective() << unsigned(DoubleToBits(Val) >> 32)
+ << "\t" << TAI->getCommentString()
+ << " double most significant word " << Val << "\n";
+ }
+ return;
+ } else {
+ O << TAI->getData32bitsDirective() << FloatToBits(Val)
+ << "\t" << TAI->getCommentString() << " float " << Val << "\n";
+ return;
+ }
+ } else if (CV->getType() == Type::Int64Ty) {
+ if (const ConstantInt *CI = dyn_cast<ConstantInt>(CV)) {
+ uint64_t Val = CI->getZExtValue();
+
+ if (TAI->getData64bitsDirective())
+ O << TAI->getData64bitsDirective() << Val << "\n";
+ else if (TD->isBigEndian()) {
+ O << TAI->getData32bitsDirective() << unsigned(Val >> 32)
+ << "\t" << TAI->getCommentString()
+ << " Double-word most significant word " << Val << "\n";
+ O << TAI->getData32bitsDirective() << unsigned(Val)
+ << "\t" << TAI->getCommentString()
+ << " Double-word least significant word " << Val << "\n";
+ } else {
+ O << TAI->getData32bitsDirective() << unsigned(Val)
+ << "\t" << TAI->getCommentString()
+ << " Double-word least significant word " << Val << "\n";
+ O << TAI->getData32bitsDirective() << unsigned(Val >> 32)
+ << "\t" << TAI->getCommentString()
+ << " Double-word most significant word " << Val << "\n";
+ }
+ return;
+ }
+ } else if (const ConstantVector *CP = dyn_cast<ConstantVector>(CV)) {
+ const VectorType *PTy = CP->getType();
+
+ for (unsigned I = 0, E = PTy->getNumElements(); I < E; ++I)
+ EmitGlobalConstant(CP->getOperand(I));
+
+ return;
+ }
+
+ const Type *type = CV->getType();
+ printDataDirective(type);
+ EmitConstantValueOnly(CV);
+ O << "\n";
+}
+
+void
+AsmPrinter::EmitMachineConstantPoolValue(MachineConstantPoolValue *MCPV) {
+ // Target doesn't support this yet!
+ abort();
+}
+
+/// PrintSpecial - Print information related to the specified machine instr
+/// that is independent of the operand, and may be independent of the instr
+/// itself. This can be useful for portably encoding the comment character
+/// or other bits of target-specific knowledge into the asmstrings. The
+/// syntax used is ${:comment}. Targets can override this to add support
+/// for their own strange codes.
+void AsmPrinter::PrintSpecial(const MachineInstr *MI, const char *Code) {
+ if (!strcmp(Code, "private")) {
+ O << TAI->getPrivateGlobalPrefix();
+ } else if (!strcmp(Code, "comment")) {
+ O << TAI->getCommentString();
+ } else if (!strcmp(Code, "uid")) {
+ // Assign a unique ID to this machine instruction.
+ static const MachineInstr *LastMI = 0;
+ static const Function *F = 0;
+ static unsigned Counter = 0U-1;
+
+ // Comparing the address of MI isn't sufficient, because machineinstrs may
+ // be allocated to the same address across functions.
+ const Function *ThisF = MI->getParent()->getParent()->getFunction();
+
+ // If this is a new machine instruction, bump the counter.
+ if (LastMI != MI || F != ThisF) {
+ ++Counter;
+ LastMI = MI;
+ F = ThisF;
+ }
+ O << Counter;
+ } else {
+ cerr << "Unknown special formatter '" << Code
+ << "' for machine instr: " << *MI;
+ exit(1);
+ }
+}
+
+
+/// printInlineAsm - This method formats and prints the specified machine
+/// instruction that is an inline asm.
+void AsmPrinter::printInlineAsm(const MachineInstr *MI) const {
+ unsigned NumOperands = MI->getNumOperands();
+
+ // Count the number of register definitions.
+ unsigned NumDefs = 0;
+ for (; MI->getOperand(NumDefs).isReg() && MI->getOperand(NumDefs).isDef();
+ ++NumDefs)
+ assert(NumDefs != NumOperands-1 && "No asm string?");
+
+ assert(MI->getOperand(NumDefs).isExternalSymbol() && "No asm string?");
+
+ // Disassemble the AsmStr, printing out the literal pieces, the operands, etc.
+ const char *AsmStr = MI->getOperand(NumDefs).getSymbolName();
+
+ // If this asmstr is empty, don't bother printing the #APP/#NOAPP markers.
+ if (AsmStr[0] == 0) {
+ O << "\n"; // Tab already printed, avoid double indenting next instr.
+ return;
+ }
+
+ O << TAI->getInlineAsmStart() << "\n\t";
+
+ // The variant of the current asmprinter.
+ int AsmPrinterVariant = TAI->getAssemblerDialect();
+
+ int CurVariant = -1; // The number of the {.|.|.} region we are in.
+ const char *LastEmitted = AsmStr; // One past the last character emitted.
+
+ while (*LastEmitted) {
+ switch (*LastEmitted) {
+ default: {
+ // Not a special case, emit the string section literally.
+ const char *LiteralEnd = LastEmitted+1;
+ while (*LiteralEnd && *LiteralEnd != '{' && *LiteralEnd != '|' &&
+ *LiteralEnd != '}' && *LiteralEnd != '$' && *LiteralEnd != '\n')
+ ++LiteralEnd;
+ if (CurVariant == -1 || CurVariant == AsmPrinterVariant)
+ O.write(LastEmitted, LiteralEnd-LastEmitted);
+ LastEmitted = LiteralEnd;
+ break;
+ }
+ case '\n':
+ ++LastEmitted; // Consume newline character.
+ O << "\n"; // Indent code with newline.
+ break;
+ case '$': {
+ ++LastEmitted; // Consume '$' character.
+ bool Done = true;
+
+ // Handle escapes.
+ switch (*LastEmitted) {
+ default: Done = false; break;
+ case '$': // $$ -> $
+ if (CurVariant == -1 || CurVariant == AsmPrinterVariant)
+ O << '$';
+ ++LastEmitted; // Consume second '$' character.
+ break;
+ case '(': // $( -> same as GCC's { character.
+ ++LastEmitted; // Consume '(' character.
+ if (CurVariant != -1) {
+ cerr << "Nested variants found in inline asm string: '"
+ << AsmStr << "'\n";
+ exit(1);
+ }
+ CurVariant = 0; // We're in the first variant now.
+ break;
+ case '|':
+ ++LastEmitted; // consume '|' character.
+ if (CurVariant == -1) {
+ cerr << "Found '|' character outside of variant in inline asm "
+ << "string: '" << AsmStr << "'\n";
+ exit(1);
+ }
+ ++CurVariant; // We're in the next variant.
+ break;
+ case ')': // $) -> same as GCC's } char.
+ ++LastEmitted; // consume ')' character.
+ if (CurVariant == -1) {
+ cerr << "Found '}' character outside of variant in inline asm "
+ << "string: '" << AsmStr << "'\n";
+ exit(1);
+ }
+ CurVariant = -1;
+ break;
+ }
+ if (Done) break;
+
+ bool HasCurlyBraces = false;
+ if (*LastEmitted == '{') { // ${variable}
+ ++LastEmitted; // Consume '{' character.
+ HasCurlyBraces = true;
+ }
+
+ const char *IDStart = LastEmitted;
+ char *IDEnd;
+ errno = 0;
+ long Val = strtol(IDStart, &IDEnd, 10); // We only accept numbers for IDs.
+ if (!isdigit(*IDStart) || (Val == 0 && errno == EINVAL)) {
+ cerr << "Bad $ operand number in inline asm string: '"
+ << AsmStr << "'\n";
+ exit(1);
+ }
+ LastEmitted = IDEnd;
+
+ char Modifier[2] = { 0, 0 };
+
+ if (HasCurlyBraces) {
+ // If we have curly braces, check for a modifier character. This
+ // supports syntax like ${0:u}, which correspond to "%u0" in GCC asm.
+ if (*LastEmitted == ':') {
+ ++LastEmitted; // Consume ':' character.
+ if (*LastEmitted == 0) {
+ cerr << "Bad ${:} expression in inline asm string: '"
+ << AsmStr << "'\n";
+ exit(1);
+ }
+
+ Modifier[0] = *LastEmitted;
+ ++LastEmitted; // Consume modifier character.
+ }
+
+ if (*LastEmitted != '}') {
+ cerr << "Bad ${} expression in inline asm string: '"
+ << AsmStr << "'\n";
+ exit(1);
+ }
+ ++LastEmitted; // Consume '}' character.
+ }
+
+ if ((unsigned)Val >= NumOperands-1) {
+ cerr << "Invalid $ operand number in inline asm string: '"
+ << AsmStr << "'\n";
+ exit(1);
+ }
+
+ // Okay, we finally have a value number. Ask the target to print this
+ // operand!
+ if (CurVariant == -1 || CurVariant == AsmPrinterVariant) {
+ unsigned OpNo = 1;
+
+ bool Error = false;
+
+ // Scan to find the machine operand number for the operand.
+ for (; Val; --Val) {
+ if (OpNo >= MI->getNumOperands()) break;
+ unsigned OpFlags = MI->getOperand(OpNo).getImmedValue();
+ OpNo += (OpFlags >> 3) + 1;
+ }
+
+ if (OpNo >= MI->getNumOperands()) {
+ Error = true;
+ } else {
+ unsigned OpFlags = MI->getOperand(OpNo).getImmedValue();
+ ++OpNo; // Skip over the ID number.
+
+ AsmPrinter *AP = const_cast<AsmPrinter*>(this);
+ if ((OpFlags & 7) == 4 /*ADDR MODE*/) {
+ Error = AP->PrintAsmMemoryOperand(MI, OpNo, AsmPrinterVariant,
+ Modifier[0] ? Modifier : 0);
+ } else {
+ Error = AP->PrintAsmOperand(MI, OpNo, AsmPrinterVariant,
+ Modifier[0] ? Modifier : 0);
+ }
+ }
+ if (Error) {
+ cerr << "Invalid operand found in inline asm: '"
+ << AsmStr << "'\n";
+ MI->dump();
+ exit(1);
+ }
+ }
+ break;
+ }
+ }
+ }
+ O << "\n\t" << TAI->getInlineAsmEnd() << "\n";
+}
+
+/// printLabel - This method prints a local label used by debug and
+/// exception handling tables.
+void AsmPrinter::printLabel(const MachineInstr *MI) const {
+ O << "\n"
+ << TAI->getPrivateGlobalPrefix()
+ << "label"
+ << MI->getOperand(0).getImmedValue()
+ << ":\n";
+}
+
+/// PrintAsmOperand - Print the specified operand of MI, an INLINEASM
+/// instruction, using the specified assembler variant. Targets should
+/// overried this to format as appropriate.
+bool AsmPrinter::PrintAsmOperand(const MachineInstr *MI, unsigned OpNo,
+ unsigned AsmVariant, const char *ExtraCode) {
+ // Target doesn't support this yet!
+ return true;
+}
+
+bool AsmPrinter::PrintAsmMemoryOperand(const MachineInstr *MI, unsigned OpNo,
+ unsigned AsmVariant,
+ const char *ExtraCode) {
+ // Target doesn't support this yet!
+ return true;
+}
+
+/// printBasicBlockLabel - This method prints the label for the specified
+/// MachineBasicBlock
+void AsmPrinter::printBasicBlockLabel(const MachineBasicBlock *MBB,
+ bool printColon,
+ bool printComment) const {
+ O << TAI->getPrivateGlobalPrefix() << "BB" << FunctionNumber << "_"
+ << MBB->getNumber();
+ if (printColon)
+ O << ':';
+ if (printComment && MBB->getBasicBlock())
+ O << '\t' << TAI->getCommentString() << MBB->getBasicBlock()->getName();
+}
+
+/// printSetLabel - This method prints a set label for the specified
+/// MachineBasicBlock
+void AsmPrinter::printSetLabel(unsigned uid,
+ const MachineBasicBlock *MBB) const {
+ if (!TAI->getSetDirective())
+ return;
+
+ O << TAI->getSetDirective() << ' ' << TAI->getPrivateGlobalPrefix()
+ << getFunctionNumber() << '_' << uid << "_set_" << MBB->getNumber() << ',';
+ printBasicBlockLabel(MBB, false, false);
+ O << '-' << TAI->getPrivateGlobalPrefix() << "JTI" << getFunctionNumber()
+ << '_' << uid << '\n';
+}
+
+void AsmPrinter::printSetLabel(unsigned uid, unsigned uid2,
+ const MachineBasicBlock *MBB) const {
+ if (!TAI->getSetDirective())
+ return;
+
+ O << TAI->getSetDirective() << ' ' << TAI->getPrivateGlobalPrefix()
+ << getFunctionNumber() << '_' << uid << '_' << uid2
+ << "_set_" << MBB->getNumber() << ',';
+ printBasicBlockLabel(MBB, false, false);
+ O << '-' << TAI->getPrivateGlobalPrefix() << "JTI" << getFunctionNumber()
+ << '_' << uid << '_' << uid2 << '\n';
+}
+
+/// printDataDirective - This method prints the asm directive for the
+/// specified type.
+void AsmPrinter::printDataDirective(const Type *type) {
+ const TargetData *TD = TM.getTargetData();
+ switch (type->getTypeID()) {
+ case Type::IntegerTyID: {
+ unsigned BitWidth = cast<IntegerType>(type)->getBitWidth();
+ if (BitWidth <= 8)
+ O << TAI->getData8bitsDirective();
+ else if (BitWidth <= 16)
+ O << TAI->getData16bitsDirective();
+ else if (BitWidth <= 32)
+ O << TAI->getData32bitsDirective();
+ else if (BitWidth <= 64) {
+ assert(TAI->getData64bitsDirective() &&
+ "Target cannot handle 64-bit constant exprs!");
+ O << TAI->getData64bitsDirective();
+ }
+ break;
+ }
+ case Type::PointerTyID:
+ if (TD->getPointerSize() == 8) {
+ assert(TAI->getData64bitsDirective() &&
+ "Target cannot handle 64-bit pointer exprs!");
+ O << TAI->getData64bitsDirective();
+ } else {
+ O << TAI->getData32bitsDirective();
+ }
+ break;
+ case Type::FloatTyID: case Type::DoubleTyID:
+ assert (0 && "Should have already output floating point constant.");
+ default:
+ assert (0 && "Can't handle printing this type of thing");
+ break;
+ }
+}
+
diff --git a/lib/CodeGen/BranchFolding.cpp b/lib/CodeGen/BranchFolding.cpp
new file mode 100644
index 0000000..0fca985
--- /dev/null
+++ b/lib/CodeGen/BranchFolding.cpp
@@ -0,0 +1,1093 @@
+//===-- BranchFolding.cpp - Fold machine code branch instructions ---------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file was developed by the LLVM research group and is distributed under
+// the University of Illinois Open Source License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This pass forwards branches to unconditional branches to make them branch
+// directly to the target block. This pass often results in dead MBB's, which
+// it then removes.
+//
+// Note that this pass must be run after register allocation, it cannot handle
+// SSA form.
+//
+//===----------------------------------------------------------------------===//
+
+#define DEBUG_TYPE "branchfolding"
+#include "llvm/CodeGen/Passes.h"
+#include "llvm/CodeGen/MachineModuleInfo.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/MachineJumpTableInfo.h"
+#include "llvm/CodeGen/RegisterScavenging.h"
+#include "llvm/Target/TargetInstrInfo.h"
+#include "llvm/Target/TargetMachine.h"
+#include "llvm/Target/MRegisterInfo.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/ADT/Statistic.h"
+#include "llvm/ADT/STLExtras.h"
+#include <algorithm>
+using namespace llvm;
+
+STATISTIC(NumDeadBlocks, "Number of dead blocks removed");
+STATISTIC(NumBranchOpts, "Number of branches optimized");
+STATISTIC(NumTailMerge , "Number of block tails merged");
+static cl::opt<cl::boolOrDefault> FlagEnableTailMerge("enable-tail-merge",
+ cl::init(cl::BOU_UNSET), cl::Hidden);
+namespace {
+ // Throttle for huge numbers of predecessors (compile speed problems)
+ cl::opt<unsigned>
+ TailMergeThreshold("tail-merge-threshold",
+ cl::desc("Max number of predecessors to consider tail merging"),
+ cl::init(100), cl::Hidden);
+
+ struct BranchFolder : public MachineFunctionPass {
+ static char ID;
+ BranchFolder(bool defaultEnableTailMerge) :
+ MachineFunctionPass((intptr_t)&ID) {
+ switch (FlagEnableTailMerge) {
+ case cl::BOU_UNSET: EnableTailMerge = defaultEnableTailMerge; break;
+ case cl::BOU_TRUE: EnableTailMerge = true; break;
+ case cl::BOU_FALSE: EnableTailMerge = false; break;
+ }
+ }
+
+ virtual bool runOnMachineFunction(MachineFunction &MF);
+ virtual const char *getPassName() const { return "Control Flow Optimizer"; }
+ const TargetInstrInfo *TII;
+ MachineModuleInfo *MMI;
+ bool MadeChange;
+ private:
+ // Tail Merging.
+ bool EnableTailMerge;
+ bool TailMergeBlocks(MachineFunction &MF);
+ bool TryMergeBlocks(MachineBasicBlock* SuccBB,
+ MachineBasicBlock* PredBB);
+ void ReplaceTailWithBranchTo(MachineBasicBlock::iterator OldInst,
+ MachineBasicBlock *NewDest);
+ MachineBasicBlock *SplitMBBAt(MachineBasicBlock &CurMBB,
+ MachineBasicBlock::iterator BBI1);
+
+ std::vector<std::pair<unsigned,MachineBasicBlock*> > MergePotentials;
+ const MRegisterInfo *RegInfo;
+ RegScavenger *RS;
+ // Branch optzn.
+ bool OptimizeBranches(MachineFunction &MF);
+ void OptimizeBlock(MachineBasicBlock *MBB);
+ void RemoveDeadBlock(MachineBasicBlock *MBB);
+
+ bool CanFallThrough(MachineBasicBlock *CurBB);
+ bool CanFallThrough(MachineBasicBlock *CurBB, bool BranchUnAnalyzable,
+ MachineBasicBlock *TBB, MachineBasicBlock *FBB,
+ const std::vector<MachineOperand> &Cond);
+ };
+ char BranchFolder::ID = 0;
+}
+
+FunctionPass *llvm::createBranchFoldingPass(bool DefaultEnableTailMerge) {
+ return new BranchFolder(DefaultEnableTailMerge); }
+
+/// RemoveDeadBlock - Remove the specified dead machine basic block from the
+/// function, updating the CFG.
+void BranchFolder::RemoveDeadBlock(MachineBasicBlock *MBB) {
+ assert(MBB->pred_empty() && "MBB must be dead!");
+ DOUT << "\nRemoving MBB: " << *MBB;
+
+ MachineFunction *MF = MBB->getParent();
+ // drop all successors.
+ while (!MBB->succ_empty())
+ MBB->removeSuccessor(MBB->succ_end()-1);
+
+ // If there is DWARF info to active, check to see if there are any LABEL
+ // records in the basic block. If so, unregister them from MachineModuleInfo.
+ if (MMI && !MBB->empty()) {
+ for (MachineBasicBlock::iterator I = MBB->begin(), E = MBB->end();
+ I != E; ++I) {
+ if ((unsigned)I->getOpcode() == TargetInstrInfo::LABEL) {
+ // The label ID # is always operand #0, an immediate.
+ MMI->InvalidateLabel(I->getOperand(0).getImm());
+ }
+ }
+ }
+
+ // Remove the block.
+ MF->getBasicBlockList().erase(MBB);
+}
+
+bool BranchFolder::runOnMachineFunction(MachineFunction &MF) {
+ TII = MF.getTarget().getInstrInfo();
+ if (!TII) return false;
+
+ // Fix CFG. The later algorithms expect it to be right.
+ bool EverMadeChange = false;
+ for (MachineFunction::iterator I = MF.begin(), E = MF.end(); I != E; I++) {
+ MachineBasicBlock *MBB = I, *TBB = 0, *FBB = 0;
+ std::vector<MachineOperand> Cond;
+ if (!TII->AnalyzeBranch(*MBB, TBB, FBB, Cond))
+ EverMadeChange |= MBB->CorrectExtraCFGEdges(TBB, FBB, !Cond.empty());
+ }
+
+ RegInfo = MF.getTarget().getRegisterInfo();
+ RS = RegInfo->requiresRegisterScavenging(MF) ? new RegScavenger() : NULL;
+
+ MMI = getAnalysisToUpdate<MachineModuleInfo>();
+
+ bool MadeChangeThisIteration = true;
+ while (MadeChangeThisIteration) {
+ MadeChangeThisIteration = false;
+ MadeChangeThisIteration |= TailMergeBlocks(MF);
+ MadeChangeThisIteration |= OptimizeBranches(MF);
+ EverMadeChange |= MadeChangeThisIteration;
+ }
+
+ // See if any jump tables have become mergable or dead as the code generator
+ // did its thing.
+ MachineJumpTableInfo *JTI = MF.getJumpTableInfo();
+ const std::vector<MachineJumpTableEntry> &JTs = JTI->getJumpTables();
+ if (!JTs.empty()) {
+ // Figure out how these jump tables should be merged.
+ std::vector<unsigned> JTMapping;
+ JTMapping.reserve(JTs.size());
+
+ // We always keep the 0th jump table.
+ JTMapping.push_back(0);
+
+ // Scan the jump tables, seeing if there are any duplicates. Note that this
+ // is N^2, which should be fixed someday.
+ for (unsigned i = 1, e = JTs.size(); i != e; ++i)
+ JTMapping.push_back(JTI->getJumpTableIndex(JTs[i].MBBs));
+
+ // If a jump table was merge with another one, walk the function rewriting
+ // references to jump tables to reference the new JT ID's. Keep track of
+ // whether we see a jump table idx, if not, we can delete the JT.
+ std::vector<bool> JTIsLive;
+ JTIsLive.resize(JTs.size());
+ for (MachineFunction::iterator BB = MF.begin(), E = MF.end();
+ BB != E; ++BB) {
+ for (MachineBasicBlock::iterator I = BB->begin(), E = BB->end();
+ I != E; ++I)
+ for (unsigned op = 0, e = I->getNumOperands(); op != e; ++op) {
+ MachineOperand &Op = I->getOperand(op);
+ if (!Op.isJumpTableIndex()) continue;
+ unsigned NewIdx = JTMapping[Op.getJumpTableIndex()];
+ Op.setJumpTableIndex(NewIdx);
+
+ // Remember that this JT is live.
+ JTIsLive[NewIdx] = true;
+ }
+ }
+
+ // Finally, remove dead jump tables. This happens either because the
+ // indirect jump was unreachable (and thus deleted) or because the jump
+ // table was merged with some other one.
+ for (unsigned i = 0, e = JTIsLive.size(); i != e; ++i)
+ if (!JTIsLive[i]) {
+ JTI->RemoveJumpTable(i);
+ EverMadeChange = true;
+ }
+ }
+
+ delete RS;
+ return EverMadeChange;
+}
+
+//===----------------------------------------------------------------------===//
+// Tail Merging of Blocks
+//===----------------------------------------------------------------------===//
+
+/// HashMachineInstr - Compute a hash value for MI and its operands.
+static unsigned HashMachineInstr(const MachineInstr *MI) {
+ unsigned Hash = MI->getOpcode();
+ for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
+ const MachineOperand &Op = MI->getOperand(i);
+
+ // Merge in bits from the operand if easy.
+ unsigned OperandHash = 0;
+ switch (Op.getType()) {
+ case MachineOperand::MO_Register: OperandHash = Op.getReg(); break;
+ case MachineOperand::MO_Immediate: OperandHash = Op.getImm(); break;
+ case MachineOperand::MO_MachineBasicBlock:
+ OperandHash = Op.getMachineBasicBlock()->getNumber();
+ break;
+ case MachineOperand::MO_FrameIndex: OperandHash = Op.getFrameIndex(); break;
+ case MachineOperand::MO_ConstantPoolIndex:
+ OperandHash = Op.getConstantPoolIndex();
+ break;
+ case MachineOperand::MO_JumpTableIndex:
+ OperandHash = Op.getJumpTableIndex();
+ break;
+ case MachineOperand::MO_GlobalAddress:
+ case MachineOperand::MO_ExternalSymbol:
+ // Global address / external symbol are too hard, don't bother, but do
+ // pull in the offset.
+ OperandHash = Op.getOffset();
+ break;
+ default: break;
+ }
+
+ Hash += ((OperandHash << 3) | Op.getType()) << (i&31);
+ }
+ return Hash;
+}
+
+/// HashEndOfMBB - Hash the last few instructions in the MBB. For blocks
+/// with no successors, we hash two instructions, because cross-jumping
+/// only saves code when at least two instructions are removed (since a
+/// branch must be inserted). For blocks with a successor, one of the
+/// two blocks to be tail-merged will end with a branch already, so
+/// it gains to cross-jump even for one instruction.
+
+static unsigned HashEndOfMBB(const MachineBasicBlock *MBB,
+ unsigned minCommonTailLength) {
+ MachineBasicBlock::const_iterator I = MBB->end();
+ if (I == MBB->begin())
+ return 0; // Empty MBB.
+
+ --I;
+ unsigned Hash = HashMachineInstr(I);
+
+ if (I == MBB->begin() || minCommonTailLength == 1)
+ return Hash; // Single instr MBB.
+
+ --I;
+ // Hash in the second-to-last instruction.
+ Hash ^= HashMachineInstr(I) << 2;
+ return Hash;
+}
+
+/// ComputeCommonTailLength - Given two machine basic blocks, compute the number
+/// of instructions they actually have in common together at their end. Return
+/// iterators for the first shared instruction in each block.
+static unsigned ComputeCommonTailLength(MachineBasicBlock *MBB1,
+ MachineBasicBlock *MBB2,
+ MachineBasicBlock::iterator &I1,
+ MachineBasicBlock::iterator &I2) {
+ I1 = MBB1->end();
+ I2 = MBB2->end();
+
+ unsigned TailLen = 0;
+ while (I1 != MBB1->begin() && I2 != MBB2->begin()) {
+ --I1; --I2;
+ if (!I1->isIdenticalTo(I2)) {
+ ++I1; ++I2;
+ break;
+ }
+ ++TailLen;
+ }
+ return TailLen;
+}
+
+/// ReplaceTailWithBranchTo - Delete the instruction OldInst and everything
+/// after it, replacing it with an unconditional branch to NewDest. This
+/// returns true if OldInst's block is modified, false if NewDest is modified.
+void BranchFolder::ReplaceTailWithBranchTo(MachineBasicBlock::iterator OldInst,
+ MachineBasicBlock *NewDest) {
+ MachineBasicBlock *OldBB = OldInst->getParent();
+
+ // Remove all the old successors of OldBB from the CFG.
+ while (!OldBB->succ_empty())
+ OldBB->removeSuccessor(OldBB->succ_begin());
+
+ // Remove all the dead instructions from the end of OldBB.
+ OldBB->erase(OldInst, OldBB->end());
+
+ // If OldBB isn't immediately before OldBB, insert a branch to it.
+ if (++MachineFunction::iterator(OldBB) != MachineFunction::iterator(NewDest))
+ TII->InsertBranch(*OldBB, NewDest, 0, std::vector<MachineOperand>());
+ OldBB->addSuccessor(NewDest);
+ ++NumTailMerge;
+}
+
+/// SplitMBBAt - Given a machine basic block and an iterator into it, split the
+/// MBB so that the part before the iterator falls into the part starting at the
+/// iterator. This returns the new MBB.
+MachineBasicBlock *BranchFolder::SplitMBBAt(MachineBasicBlock &CurMBB,
+ MachineBasicBlock::iterator BBI1) {
+ // Create the fall-through block.
+ MachineFunction::iterator MBBI = &CurMBB;
+ MachineBasicBlock *NewMBB = new MachineBasicBlock(CurMBB.getBasicBlock());
+ CurMBB.getParent()->getBasicBlockList().insert(++MBBI, NewMBB);
+
+ // Move all the successors of this block to the specified block.
+ while (!CurMBB.succ_empty()) {
+ MachineBasicBlock *S = *(CurMBB.succ_end()-1);
+ NewMBB->addSuccessor(S);
+ CurMBB.removeSuccessor(S);
+ }
+
+ // Add an edge from CurMBB to NewMBB for the fall-through.
+ CurMBB.addSuccessor(NewMBB);
+
+ // Splice the code over.
+ NewMBB->splice(NewMBB->end(), &CurMBB, BBI1, CurMBB.end());
+
+ // For targets that use the register scavenger, we must maintain LiveIns.
+ if (RS) {
+ RS->enterBasicBlock(&CurMBB);
+ if (!CurMBB.empty())
+ RS->forward(prior(CurMBB.end()));
+ BitVector RegsLiveAtExit(RegInfo->getNumRegs());
+ RS->getRegsUsed(RegsLiveAtExit, false);
+ for (unsigned int i=0, e=RegInfo->getNumRegs(); i!=e; i++)
+ if (RegsLiveAtExit[i])
+ NewMBB->addLiveIn(i);
+ }
+
+ return NewMBB;
+}
+
+/// EstimateRuntime - Make a rough estimate for how long it will take to run
+/// the specified code.
+static unsigned EstimateRuntime(MachineBasicBlock::iterator I,
+ MachineBasicBlock::iterator E,
+ const TargetInstrInfo *TII) {
+ unsigned Time = 0;
+ for (; I != E; ++I) {
+ const TargetInstrDescriptor &TID = TII->get(I->getOpcode());
+ if (TID.Flags & M_CALL_FLAG)
+ Time += 10;
+ else if (TID.Flags & (M_LOAD_FLAG|M_STORE_FLAG))
+ Time += 2;
+ else
+ ++Time;
+ }
+ return Time;
+}
+
+/// ShouldSplitFirstBlock - We need to either split MBB1 at MBB1I or MBB2 at
+/// MBB2I and then insert an unconditional branch in the other block. Determine
+/// which is the best to split
+static bool ShouldSplitFirstBlock(MachineBasicBlock *MBB1,
+ MachineBasicBlock::iterator MBB1I,
+ MachineBasicBlock *MBB2,
+ MachineBasicBlock::iterator MBB2I,
+ const TargetInstrInfo *TII,
+ MachineBasicBlock *PredBB) {
+ // If one block is the entry block, split the other one; we can't generate
+ // a branch to the entry block, as its label is not emitted.
+ MachineBasicBlock *Entry = MBB1->getParent()->begin();
+ if (MBB1 == Entry)
+ return false;
+ if (MBB2 == Entry)
+ return true;
+
+ // If one block falls through into the common successor, choose that
+ // one to split; it is one instruction less to do that.
+ if (PredBB) {
+ if (MBB1 == PredBB)
+ return true;
+ else if (MBB2 == PredBB)
+ return false;
+ }
+ // TODO: if we had some notion of which block was hotter, we could split
+ // the hot block, so it is the fall-through. Since we don't have profile info
+ // make a decision based on which will hurt most to split.
+ unsigned MBB1Time = EstimateRuntime(MBB1->begin(), MBB1I, TII);
+ unsigned MBB2Time = EstimateRuntime(MBB2->begin(), MBB2I, TII);
+
+ // If the MBB1 prefix takes "less time" to run than the MBB2 prefix, split the
+ // MBB1 block so it falls through. This will penalize the MBB2 path, but will
+ // have a lower overall impact on the program execution.
+ return MBB1Time < MBB2Time;
+}
+
+// CurMBB needs to add an unconditional branch to SuccMBB (we removed these
+// branches temporarily for tail merging). In the case where CurMBB ends
+// with a conditional branch to the next block, optimize by reversing the
+// test and conditionally branching to SuccMBB instead.
+
+static void FixTail(MachineBasicBlock* CurMBB, MachineBasicBlock *SuccBB,
+ const TargetInstrInfo *TII) {
+ MachineFunction *MF = CurMBB->getParent();
+ MachineFunction::iterator I = next(MachineFunction::iterator(CurMBB));
+ MachineBasicBlock *TBB = 0, *FBB = 0;
+ std::vector<MachineOperand> Cond;
+ if (I != MF->end() &&
+ !TII->AnalyzeBranch(*CurMBB, TBB, FBB, Cond)) {
+ MachineBasicBlock *NextBB = I;
+ if (TBB == NextBB && Cond.size() && !FBB) {
+ if (!TII->ReverseBranchCondition(Cond)) {
+ TII->RemoveBranch(*CurMBB);
+ TII->InsertBranch(*CurMBB, SuccBB, NULL, Cond);
+ return;
+ }
+ }
+ }
+ TII->InsertBranch(*CurMBB, SuccBB, NULL, std::vector<MachineOperand>());
+}
+
+static bool MergeCompare(const std::pair<unsigned,MachineBasicBlock*> &p,
+ const std::pair<unsigned,MachineBasicBlock*> &q) {
+ if (p.first < q.first)
+ return true;
+ else if (p.first > q.first)
+ return false;
+ else if (p.second->getNumber() < q.second->getNumber())
+ return true;
+ else if (p.second->getNumber() > q.second->getNumber())
+ return false;
+ else {
+ // _GLIBCXX_DEBUG checks strict weak ordering, which involves comparing
+ // an object with itself.
+#ifndef _GLIBCXX_DEBUG
+ assert(0 && "Predecessor appears twice");
+#endif
+ return(false);
+ }
+}
+
+// See if any of the blocks in MergePotentials (which all have a common single
+// successor, or all have no successor) can be tail-merged. If there is a
+// successor, any blocks in MergePotentials that are not tail-merged and
+// are not immediately before Succ must have an unconditional branch to
+// Succ added (but the predecessor/successor lists need no adjustment).
+// The lone predecessor of Succ that falls through into Succ,
+// if any, is given in PredBB.
+
+bool BranchFolder::TryMergeBlocks(MachineBasicBlock *SuccBB,
+ MachineBasicBlock* PredBB) {
+ unsigned minCommonTailLength = (SuccBB ? 1 : 2);
+ MadeChange = false;
+
+ // Sort by hash value so that blocks with identical end sequences sort
+ // together.
+ std::stable_sort(MergePotentials.begin(), MergePotentials.end(), MergeCompare);
+
+ // Walk through equivalence sets looking for actual exact matches.
+ while (MergePotentials.size() > 1) {
+ unsigned CurHash = (MergePotentials.end()-1)->first;
+ unsigned PrevHash = (MergePotentials.end()-2)->first;
+ MachineBasicBlock *CurMBB = (MergePotentials.end()-1)->second;
+
+ // If there is nothing that matches the hash of the current basic block,
+ // give up.
+ if (CurHash != PrevHash) {
+ if (SuccBB && CurMBB != PredBB)
+ FixTail(CurMBB, SuccBB, TII);
+ MergePotentials.pop_back();
+ continue;
+ }
+
+ // Look through all the pairs of blocks that have the same hash as this
+ // one, and find the pair that has the largest number of instructions in
+ // common.
+ // Since instructions may get combined later (e.g. single stores into
+ // store multiple) this measure is not particularly accurate.
+ MachineBasicBlock::iterator BBI1, BBI2;
+
+ unsigned FoundI = ~0U, FoundJ = ~0U;
+ unsigned maxCommonTailLength = 0U;
+ for (int i = MergePotentials.size()-1;
+ i != -1 && MergePotentials[i].first == CurHash; --i) {
+ for (int j = i-1;
+ j != -1 && MergePotentials[j].first == CurHash; --j) {
+ MachineBasicBlock::iterator TrialBBI1, TrialBBI2;
+ unsigned CommonTailLen = ComputeCommonTailLength(
+ MergePotentials[i].second,
+ MergePotentials[j].second,
+ TrialBBI1, TrialBBI2);
+ if (CommonTailLen >= minCommonTailLength &&
+ CommonTailLen > maxCommonTailLength) {
+ FoundI = i;
+ FoundJ = j;
+ maxCommonTailLength = CommonTailLen;
+ BBI1 = TrialBBI1;
+ BBI2 = TrialBBI2;
+ }
+ }
+ }
+
+ // If we didn't find any pair that has at least minCommonTailLength
+ // instructions in common, bail out. All entries with this
+ // hash code can go away now.
+ if (FoundI == ~0U) {
+ for (int i = MergePotentials.size()-1;
+ i != -1 && MergePotentials[i].first == CurHash; --i) {
+ // Put the unconditional branch back, if we need one.
+ CurMBB = MergePotentials[i].second;
+ if (SuccBB && CurMBB != PredBB)
+ FixTail(CurMBB, SuccBB, TII);
+ MergePotentials.pop_back();
+ }
+ continue;
+ }
+
+ // Otherwise, move the block(s) to the right position(s). So that
+ // BBI1/2 will be valid, the last must be I and the next-to-last J.
+ if (FoundI != MergePotentials.size()-1)
+ std::swap(MergePotentials[FoundI], *(MergePotentials.end()-1));
+ if (FoundJ != MergePotentials.size()-2)
+ std::swap(MergePotentials[FoundJ], *(MergePotentials.end()-2));
+
+ CurMBB = (MergePotentials.end()-1)->second;
+ MachineBasicBlock *MBB2 = (MergePotentials.end()-2)->second;
+
+ // If neither block is the entire common tail, split the tail of one block
+ // to make it redundant with the other tail. Also, we cannot jump to the
+ // entry block, so if one block is the entry block, split the other one.
+ MachineBasicBlock *Entry = CurMBB->getParent()->begin();
+ if (CurMBB->begin() == BBI1 && CurMBB != Entry)
+ ; // CurMBB is common tail
+ else if (MBB2->begin() == BBI2 && MBB2 != Entry)
+ ; // MBB2 is common tail
+ else {
+ if (0) { // Enable this to disable partial tail merges.
+ MergePotentials.pop_back();
+ continue;
+ }
+
+ // Decide whether we want to split CurMBB or MBB2.
+ if (ShouldSplitFirstBlock(CurMBB, BBI1, MBB2, BBI2, TII, PredBB)) {
+ CurMBB = SplitMBBAt(*CurMBB, BBI1);
+ BBI1 = CurMBB->begin();
+ MergePotentials.back().second = CurMBB;
+ } else {
+ MBB2 = SplitMBBAt(*MBB2, BBI2);
+ BBI2 = MBB2->begin();
+ (MergePotentials.end()-2)->second = MBB2;
+ }
+ }
+
+ if (MBB2->begin() == BBI2 && MBB2 != Entry) {
+ // Hack the end off CurMBB, making it jump to MBBI@ instead.
+ ReplaceTailWithBranchTo(BBI1, MBB2);
+ // This modifies CurMBB, so remove it from the worklist.
+ MergePotentials.pop_back();
+ } else {
+ assert(CurMBB->begin() == BBI1 && CurMBB != Entry &&
+ "Didn't split block correctly?");
+ // Hack the end off MBB2, making it jump to CurMBB instead.
+ ReplaceTailWithBranchTo(BBI2, CurMBB);
+ // This modifies MBB2, so remove it from the worklist.
+ MergePotentials.erase(MergePotentials.end()-2);
+ }
+ MadeChange = true;
+ }
+ return MadeChange;
+}
+
+bool BranchFolder::TailMergeBlocks(MachineFunction &MF) {
+
+ if (!EnableTailMerge) return false;
+
+ MadeChange = false;
+
+ // First find blocks with no successors.
+ MergePotentials.clear();
+ for (MachineFunction::iterator I = MF.begin(), E = MF.end(); I != E; ++I) {
+ if (I->succ_empty())
+ MergePotentials.push_back(std::make_pair(HashEndOfMBB(I, 2U), I));
+ }
+ // See if we can do any tail merging on those.
+ if (MergePotentials.size() < TailMergeThreshold)
+ MadeChange |= TryMergeBlocks(NULL, NULL);
+
+ // Look at blocks (IBB) with multiple predecessors (PBB).
+ // We change each predecessor to a canonical form, by
+ // (1) temporarily removing any unconditional branch from the predecessor
+ // to IBB, and
+ // (2) alter conditional branches so they branch to the other block
+ // not IBB; this may require adding back an unconditional branch to IBB
+ // later, where there wasn't one coming in. E.g.
+ // Bcc IBB
+ // fallthrough to QBB
+ // here becomes
+ // Bncc QBB
+ // with a conceptual B to IBB after that, which never actually exists.
+ // With those changes, we see whether the predecessors' tails match,
+ // and merge them if so. We change things out of canonical form and
+ // back to the way they were later in the process. (OptimizeBranches
+ // would undo some of this, but we can't use it, because we'd get into
+ // a compile-time infinite loop repeatedly doing and undoing the same
+ // transformations.)
+
+ for (MachineFunction::iterator I = MF.begin(), E = MF.end(); I != E; ++I) {
+ if (!I->succ_empty() && I->pred_size() >= 2 &&
+ I->pred_size() < TailMergeThreshold) {
+ MachineBasicBlock *IBB = I;
+ MachineBasicBlock *PredBB = prior(I);
+ MergePotentials.clear();
+ for (MachineBasicBlock::pred_iterator P = I->pred_begin(),
+ E2 = I->pred_end();
+ P != E2; ++P) {
+ MachineBasicBlock* PBB = *P;
+ // Skip blocks that loop to themselves, can't tail merge these.
+ if (PBB==IBB)
+ continue;
+ MachineBasicBlock *TBB = 0, *FBB = 0;
+ std::vector<MachineOperand> Cond;
+ if (!TII->AnalyzeBranch(*PBB, TBB, FBB, Cond)) {
+ // Failing case: IBB is the target of a cbr, and
+ // we cannot reverse the branch.
+ std::vector<MachineOperand> NewCond(Cond);
+ if (Cond.size() && TBB==IBB) {
+ if (TII->ReverseBranchCondition(NewCond))
+ continue;
+ // This is the QBB case described above
+ if (!FBB)
+ FBB = next(MachineFunction::iterator(PBB));
+ }
+ // Failing case: the only way IBB can be reached from PBB is via
+ // exception handling. Happens for landing pads. Would be nice
+ // to have a bit in the edge so we didn't have to do all this.
+ if (IBB->isLandingPad()) {
+ MachineFunction::iterator IP = PBB; IP++;
+ MachineBasicBlock* PredNextBB = NULL;
+ if (IP!=MF.end())
+ PredNextBB = IP;
+ if (TBB==NULL) {
+ if (IBB!=PredNextBB) // fallthrough
+ continue;
+ } else if (FBB) {
+ if (TBB!=IBB && FBB!=IBB) // cbr then ubr
+ continue;
+ } else if (Cond.size() == 0) {
+ if (TBB!=IBB) // ubr
+ continue;
+ } else {
+ if (TBB!=IBB && IBB!=PredNextBB) // cbr
+ continue;
+ }
+ }
+ // Remove the unconditional branch at the end, if any.
+ if (TBB && (Cond.size()==0 || FBB)) {
+ TII->RemoveBranch(*PBB);
+ if (Cond.size())
+ // reinsert conditional branch only, for now
+ TII->InsertBranch(*PBB, (TBB==IBB) ? FBB : TBB, 0, NewCond);
+ }
+ MergePotentials.push_back(std::make_pair(HashEndOfMBB(PBB, 1U), *P));
+ }
+ }
+ if (MergePotentials.size() >= 2)
+ MadeChange |= TryMergeBlocks(I, PredBB);
+ // Reinsert an unconditional branch if needed.
+ // The 1 below can be either an original single predecessor, or a result
+ // of removing blocks in TryMergeBlocks.
+ PredBB = prior(I); // this may have been changed in TryMergeBlocks
+ if (MergePotentials.size()==1 &&
+ (MergePotentials.begin())->second != PredBB)
+ FixTail((MergePotentials.begin())->second, I, TII);
+ }
+ }
+ return MadeChange;
+}
+
+//===----------------------------------------------------------------------===//
+// Branch Optimization
+//===----------------------------------------------------------------------===//
+
+bool BranchFolder::OptimizeBranches(MachineFunction &MF) {
+ MadeChange = false;
+
+ // Make sure blocks are numbered in order
+ MF.RenumberBlocks();
+
+ for (MachineFunction::iterator I = ++MF.begin(), E = MF.end(); I != E; ) {
+ MachineBasicBlock *MBB = I++;
+ OptimizeBlock(MBB);
+
+ // If it is dead, remove it.
+ if (MBB->pred_empty()) {
+ RemoveDeadBlock(MBB);
+ MadeChange = true;
+ ++NumDeadBlocks;
+ }
+ }
+ return MadeChange;
+}
+
+
+/// CanFallThrough - Return true if the specified block (with the specified
+/// branch condition) can implicitly transfer control to the block after it by
+/// falling off the end of it. This should return false if it can reach the
+/// block after it, but it uses an explicit branch to do so (e.g. a table jump).
+///
+/// True is a conservative answer.
+///
+bool BranchFolder::CanFallThrough(MachineBasicBlock *CurBB,
+ bool BranchUnAnalyzable,
+ MachineBasicBlock *TBB, MachineBasicBlock *FBB,
+ const std::vector<MachineOperand> &Cond) {
+ MachineFunction::iterator Fallthrough = CurBB;
+ ++Fallthrough;
+ // If FallthroughBlock is off the end of the function, it can't fall through.
+ if (Fallthrough == CurBB->getParent()->end())
+ return false;
+
+ // If FallthroughBlock isn't a successor of CurBB, no fallthrough is possible.
+ if (!CurBB->isSuccessor(Fallthrough))
+ return false;
+
+ // If we couldn't analyze the branch, assume it could fall through.
+ if (BranchUnAnalyzable) return true;
+
+ // If there is no branch, control always falls through.
+ if (TBB == 0) return true;
+
+ // If there is some explicit branch to the fallthrough block, it can obviously
+ // reach, even though the branch should get folded to fall through implicitly.
+ if (MachineFunction::iterator(TBB) == Fallthrough ||
+ MachineFunction::iterator(FBB) == Fallthrough)
+ return true;
+
+ // If it's an unconditional branch to some block not the fall through, it
+ // doesn't fall through.
+ if (Cond.empty()) return false;
+
+ // Otherwise, if it is conditional and has no explicit false block, it falls
+ // through.
+ return FBB == 0;
+}
+
+/// CanFallThrough - Return true if the specified can implicitly transfer
+/// control to the block after it by falling off the end of it. This should
+/// return false if it can reach the block after it, but it uses an explicit
+/// branch to do so (e.g. a table jump).
+///
+/// True is a conservative answer.
+///
+bool BranchFolder::CanFallThrough(MachineBasicBlock *CurBB) {
+ MachineBasicBlock *TBB = 0, *FBB = 0;
+ std::vector<MachineOperand> Cond;
+ bool CurUnAnalyzable = TII->AnalyzeBranch(*CurBB, TBB, FBB, Cond);
+ return CanFallThrough(CurBB, CurUnAnalyzable, TBB, FBB, Cond);
+}
+
+/// IsBetterFallthrough - Return true if it would be clearly better to
+/// fall-through to MBB1 than to fall through into MBB2. This has to return
+/// a strict ordering, returning true for both (MBB1,MBB2) and (MBB2,MBB1) will
+/// result in infinite loops.
+static bool IsBetterFallthrough(MachineBasicBlock *MBB1,
+ MachineBasicBlock *MBB2,
+ const TargetInstrInfo &TII) {
+ // Right now, we use a simple heuristic. If MBB2 ends with a call, and
+ // MBB1 doesn't, we prefer to fall through into MBB1. This allows us to
+ // optimize branches that branch to either a return block or an assert block
+ // into a fallthrough to the return.
+ if (MBB1->empty() || MBB2->empty()) return false;
+
+ MachineInstr *MBB1I = --MBB1->end();
+ MachineInstr *MBB2I = --MBB2->end();
+ return TII.isCall(MBB2I->getOpcode()) && !TII.isCall(MBB1I->getOpcode());
+}
+
+/// OptimizeBlock - Analyze and optimize control flow related to the specified
+/// block. This is never called on the entry block.
+void BranchFolder::OptimizeBlock(MachineBasicBlock *MBB) {
+ MachineFunction::iterator FallThrough = MBB;
+ ++FallThrough;
+
+ // If this block is empty, make everyone use its fall-through, not the block
+ // explicitly. Landing pads should not do this since the landing-pad table
+ // points to this block.
+ if (MBB->empty() && !MBB->isLandingPad()) {
+ // Dead block? Leave for cleanup later.
+ if (MBB->pred_empty()) return;
+
+ if (FallThrough == MBB->getParent()->end()) {
+ // TODO: Simplify preds to not branch here if possible!
+ } else {
+ // Rewrite all predecessors of the old block to go to the fallthrough
+ // instead.
+ while (!MBB->pred_empty()) {
+ MachineBasicBlock *Pred = *(MBB->pred_end()-1);
+ Pred->ReplaceUsesOfBlockWith(MBB, FallThrough);
+ }
+
+ // If MBB was the target of a jump table, update jump tables to go to the
+ // fallthrough instead.
+ MBB->getParent()->getJumpTableInfo()->
+ ReplaceMBBInJumpTables(MBB, FallThrough);
+ MadeChange = true;
+ }
+ return;
+ }
+
+ // Check to see if we can simplify the terminator of the block before this
+ // one.
+ MachineBasicBlock &PrevBB = *prior(MachineFunction::iterator(MBB));
+
+ MachineBasicBlock *PriorTBB = 0, *PriorFBB = 0;
+ std::vector<MachineOperand> PriorCond;
+ bool PriorUnAnalyzable =
+ TII->AnalyzeBranch(PrevBB, PriorTBB, PriorFBB, PriorCond);
+ if (!PriorUnAnalyzable) {
+ // If the CFG for the prior block has extra edges, remove them.
+ MadeChange |= PrevBB.CorrectExtraCFGEdges(PriorTBB, PriorFBB,
+ !PriorCond.empty());
+
+ // If the previous branch is conditional and both conditions go to the same
+ // destination, remove the branch, replacing it with an unconditional one or
+ // a fall-through.
+ if (PriorTBB && PriorTBB == PriorFBB) {
+ TII->RemoveBranch(PrevBB);
+ PriorCond.clear();
+ if (PriorTBB != MBB)
+ TII->InsertBranch(PrevBB, PriorTBB, 0, PriorCond);
+ MadeChange = true;
+ ++NumBranchOpts;
+ return OptimizeBlock(MBB);
+ }
+
+ // If the previous branch *only* branches to *this* block (conditional or
+ // not) remove the branch.
+ if (PriorTBB == MBB && PriorFBB == 0) {
+ TII->RemoveBranch(PrevBB);
+ MadeChange = true;
+ ++NumBranchOpts;
+ return OptimizeBlock(MBB);
+ }
+
+ // If the prior block branches somewhere else on the condition and here if
+ // the condition is false, remove the uncond second branch.
+ if (PriorFBB == MBB) {
+ TII->RemoveBranch(PrevBB);
+ TII->InsertBranch(PrevBB, PriorTBB, 0, PriorCond);
+ MadeChange = true;
+ ++NumBranchOpts;
+ return OptimizeBlock(MBB);
+ }
+
+ // If the prior block branches here on true and somewhere else on false, and
+ // if the branch condition is reversible, reverse the branch to create a
+ // fall-through.
+ if (PriorTBB == MBB) {
+ std::vector<MachineOperand> NewPriorCond(PriorCond);
+ if (!TII->ReverseBranchCondition(NewPriorCond)) {
+ TII->RemoveBranch(PrevBB);
+ TII->InsertBranch(PrevBB, PriorFBB, 0, NewPriorCond);
+ MadeChange = true;
+ ++NumBranchOpts;
+ return OptimizeBlock(MBB);
+ }
+ }
+
+ // If this block doesn't fall through (e.g. it ends with an uncond branch or
+ // has no successors) and if the pred falls through into this block, and if
+ // it would otherwise fall through into the block after this, move this
+ // block to the end of the function.
+ //
+ // We consider it more likely that execution will stay in the function (e.g.
+ // due to loops) than it is to exit it. This asserts in loops etc, moving
+ // the assert condition out of the loop body.
+ if (!PriorCond.empty() && PriorFBB == 0 &&
+ MachineFunction::iterator(PriorTBB) == FallThrough &&
+ !CanFallThrough(MBB)) {
+ bool DoTransform = true;
+
+ // We have to be careful that the succs of PredBB aren't both no-successor
+ // blocks. If neither have successors and if PredBB is the second from
+ // last block in the function, we'd just keep swapping the two blocks for
+ // last. Only do the swap if one is clearly better to fall through than
+ // the other.
+ if (FallThrough == --MBB->getParent()->end() &&
+ !IsBetterFallthrough(PriorTBB, MBB, *TII))
+ DoTransform = false;
+
+ // We don't want to do this transformation if we have control flow like:
+ // br cond BB2
+ // BB1:
+ // ..
+ // jmp BBX
+ // BB2:
+ // ..
+ // ret
+ //
+ // In this case, we could actually be moving the return block *into* a
+ // loop!
+ if (DoTransform && !MBB->succ_empty() &&
+ (!CanFallThrough(PriorTBB) || PriorTBB->empty()))
+ DoTransform = false;
+
+
+ if (DoTransform) {
+ // Reverse the branch so we will fall through on the previous true cond.
+ std::vector<MachineOperand> NewPriorCond(PriorCond);
+ if (!TII->ReverseBranchCondition(NewPriorCond)) {
+ DOUT << "\nMoving MBB: " << *MBB;
+ DOUT << "To make fallthrough to: " << *PriorTBB << "\n";
+
+ TII->RemoveBranch(PrevBB);
+ TII->InsertBranch(PrevBB, MBB, 0, NewPriorCond);
+
+ // Move this block to the end of the function.
+ MBB->moveAfter(--MBB->getParent()->end());
+ MadeChange = true;
+ ++NumBranchOpts;
+ return;
+ }
+ }
+ }
+ }
+
+ // Analyze the branch in the current block.
+ MachineBasicBlock *CurTBB = 0, *CurFBB = 0;
+ std::vector<MachineOperand> CurCond;
+ bool CurUnAnalyzable = TII->AnalyzeBranch(*MBB, CurTBB, CurFBB, CurCond);
+ if (!CurUnAnalyzable) {
+ // If the CFG for the prior block has extra edges, remove them.
+ MadeChange |= MBB->CorrectExtraCFGEdges(CurTBB, CurFBB, !CurCond.empty());
+
+ // If this is a two-way branch, and the FBB branches to this block, reverse
+ // the condition so the single-basic-block loop is faster. Instead of:
+ // Loop: xxx; jcc Out; jmp Loop
+ // we want:
+ // Loop: xxx; jncc Loop; jmp Out
+ if (CurTBB && CurFBB && CurFBB == MBB && CurTBB != MBB) {
+ std::vector<MachineOperand> NewCond(CurCond);
+ if (!TII->ReverseBranchCondition(NewCond)) {
+ TII->RemoveBranch(*MBB);
+ TII->InsertBranch(*MBB, CurFBB, CurTBB, NewCond);
+ MadeChange = true;
+ ++NumBranchOpts;
+ return OptimizeBlock(MBB);
+ }
+ }
+
+
+ // If this branch is the only thing in its block, see if we can forward
+ // other blocks across it.
+ if (CurTBB && CurCond.empty() && CurFBB == 0 &&
+ TII->isBranch(MBB->begin()->getOpcode()) && CurTBB != MBB) {
+ // This block may contain just an unconditional branch. Because there can
+ // be 'non-branch terminators' in the block, try removing the branch and
+ // then seeing if the block is empty.
+ TII->RemoveBranch(*MBB);
+
+ // If this block is just an unconditional branch to CurTBB, we can
+ // usually completely eliminate the block. The only case we cannot
+ // completely eliminate the block is when the block before this one
+ // falls through into MBB and we can't understand the prior block's branch
+ // condition.
+ if (MBB->empty()) {
+ bool PredHasNoFallThrough = TII->BlockHasNoFallThrough(PrevBB);
+ if (PredHasNoFallThrough || !PriorUnAnalyzable ||
+ !PrevBB.isSuccessor(MBB)) {
+ // If the prior block falls through into us, turn it into an
+ // explicit branch to us to make updates simpler.
+ if (!PredHasNoFallThrough && PrevBB.isSuccessor(MBB) &&
+ PriorTBB != MBB && PriorFBB != MBB) {
+ if (PriorTBB == 0) {
+ assert(PriorCond.empty() && PriorFBB == 0 &&
+ "Bad branch analysis");
+ PriorTBB = MBB;
+ } else {
+ assert(PriorFBB == 0 && "Machine CFG out of date!");
+ PriorFBB = MBB;
+ }
+ TII->RemoveBranch(PrevBB);
+ TII->InsertBranch(PrevBB, PriorTBB, PriorFBB, PriorCond);
+ }
+
+ // Iterate through all the predecessors, revectoring each in-turn.
+ size_t PI = 0;
+ bool DidChange = false;
+ bool HasBranchToSelf = false;
+ while(PI != MBB->pred_size()) {
+ MachineBasicBlock *PMBB = *(MBB->pred_begin() + PI);
+ if (PMBB == MBB) {
+ // If this block has an uncond branch to itself, leave it.
+ ++PI;
+ HasBranchToSelf = true;
+ } else {
+ DidChange = true;
+ PMBB->ReplaceUsesOfBlockWith(MBB, CurTBB);
+ }
+ }
+
+ // Change any jumptables to go to the new MBB.
+ MBB->getParent()->getJumpTableInfo()->
+ ReplaceMBBInJumpTables(MBB, CurTBB);
+ if (DidChange) {
+ ++NumBranchOpts;
+ MadeChange = true;
+ if (!HasBranchToSelf) return;
+ }
+ }
+ }
+
+ // Add the branch back if the block is more than just an uncond branch.
+ TII->InsertBranch(*MBB, CurTBB, 0, CurCond);
+ }
+ }
+
+ // If the prior block doesn't fall through into this block, and if this
+ // block doesn't fall through into some other block, see if we can find a
+ // place to move this block where a fall-through will happen.
+ if (!CanFallThrough(&PrevBB, PriorUnAnalyzable,
+ PriorTBB, PriorFBB, PriorCond)) {
+ // Now we know that there was no fall-through into this block, check to
+ // see if it has a fall-through into its successor.
+ bool CurFallsThru = CanFallThrough(MBB, CurUnAnalyzable, CurTBB, CurFBB,
+ CurCond);
+
+ if (!MBB->isLandingPad()) {
+ // Check all the predecessors of this block. If one of them has no fall
+ // throughs, move this block right after it.
+ for (MachineBasicBlock::pred_iterator PI = MBB->pred_begin(),
+ E = MBB->pred_end(); PI != E; ++PI) {
+ // Analyze the branch at the end of the pred.
+ MachineBasicBlock *PredBB = *PI;
+ MachineFunction::iterator PredFallthrough = PredBB; ++PredFallthrough;
+ if (PredBB != MBB && !CanFallThrough(PredBB)
+ && (!CurFallsThru || !CurTBB || !CurFBB)
+ && (!CurFallsThru || MBB->getNumber() >= PredBB->getNumber())) {
+ // If the current block doesn't fall through, just move it.
+ // If the current block can fall through and does not end with a
+ // conditional branch, we need to append an unconditional jump to
+ // the (current) next block. To avoid a possible compile-time
+ // infinite loop, move blocks only backward in this case.
+ // Also, if there are already 2 branches here, we cannot add a third;
+ // this means we have the case
+ // Bcc next
+ // B elsewhere
+ // next:
+ if (CurFallsThru) {
+ MachineBasicBlock *NextBB = next(MachineFunction::iterator(MBB));
+ CurCond.clear();
+ TII->InsertBranch(*MBB, NextBB, 0, CurCond);
+ }
+ MBB->moveAfter(PredBB);
+ MadeChange = true;
+ return OptimizeBlock(MBB);
+ }
+ }
+ }
+
+ if (!CurFallsThru) {
+ // Check all successors to see if we can move this block before it.
+ for (MachineBasicBlock::succ_iterator SI = MBB->succ_begin(),
+ E = MBB->succ_end(); SI != E; ++SI) {
+ // Analyze the branch at the end of the block before the succ.
+ MachineBasicBlock *SuccBB = *SI;
+ MachineFunction::iterator SuccPrev = SuccBB; --SuccPrev;
+ std::vector<MachineOperand> SuccPrevCond;
+
+ // If this block doesn't already fall-through to that successor, and if
+ // the succ doesn't already have a block that can fall through into it,
+ // and if the successor isn't an EH destination, we can arrange for the
+ // fallthrough to happen.
+ if (SuccBB != MBB && !CanFallThrough(SuccPrev) &&
+ !SuccBB->isLandingPad()) {
+ MBB->moveBefore(SuccBB);
+ MadeChange = true;
+ return OptimizeBlock(MBB);
+ }
+ }
+
+ // Okay, there is no really great place to put this block. If, however,
+ // the block before this one would be a fall-through if this block were
+ // removed, move this block to the end of the function.
+ if (FallThrough != MBB->getParent()->end() &&
+ PrevBB.isSuccessor(FallThrough)) {
+ MBB->moveAfter(--MBB->getParent()->end());
+ MadeChange = true;
+ return;
+ }
+ }
+ }
+}
diff --git a/lib/CodeGen/DwarfWriter.cpp b/lib/CodeGen/DwarfWriter.cpp
new file mode 100644
index 0000000..257a776
--- /dev/null
+++ b/lib/CodeGen/DwarfWriter.cpp
@@ -0,0 +1,3616 @@
+//===-- llvm/CodeGen/DwarfWriter.cpp - Dwarf Framework ----------*- C++ -*-===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file was developed by James M. Laskey and is distributed under the
+// University of Illinois Open Source License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains support for writing dwarf info into asm files.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/CodeGen/DwarfWriter.h"
+
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/FoldingSet.h"
+#include "llvm/ADT/StringExtras.h"
+#include "llvm/ADT/UniqueVector.h"
+#include "llvm/Module.h"
+#include "llvm/Type.h"
+#include "llvm/CodeGen/AsmPrinter.h"
+#include "llvm/CodeGen/MachineModuleInfo.h"
+#include "llvm/CodeGen/MachineFrameInfo.h"
+#include "llvm/CodeGen/MachineLocation.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/Dwarf.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/DataTypes.h"
+#include "llvm/Support/Mangler.h"
+#include "llvm/Target/TargetAsmInfo.h"
+#include "llvm/Target/MRegisterInfo.h"
+#include "llvm/Target/TargetData.h"
+#include "llvm/Target/TargetFrameInfo.h"
+#include "llvm/Target/TargetInstrInfo.h"
+#include "llvm/Target/TargetMachine.h"
+#include "llvm/Target/TargetOptions.h"
+#include <ostream>
+#include <string>
+using namespace llvm;
+using namespace llvm::dwarf;
+
+namespace llvm {
+
+//===----------------------------------------------------------------------===//
+
+/// Configuration values for initial hash set sizes (log2).
+///
+static const unsigned InitDiesSetSize = 9; // 512
+static const unsigned InitAbbreviationsSetSize = 9; // 512
+static const unsigned InitValuesSetSize = 9; // 512
+
+//===----------------------------------------------------------------------===//
+/// Forward declarations.
+///
+class DIE;
+class DIEValue;
+
+//===----------------------------------------------------------------------===//
+/// DWLabel - Labels are used to track locations in the assembler file.
+/// Labels appear in the form <prefix><Tag><Number>, where the tag is a
+/// category of label (Ex. location) and number is a value unique in that
+/// category.
+class DWLabel {
+public:
+ /// Tag - Label category tag. Should always be a staticly declared C string.
+ ///
+ const char *Tag;
+
+ /// Number - Value to make label unique.
+ ///
+ unsigned Number;
+
+ DWLabel(const char *T, unsigned N) : Tag(T), Number(N) {}
+
+ void Profile(FoldingSetNodeID &ID) const {
+ ID.AddString(std::string(Tag));
+ ID.AddInteger(Number);
+ }
+
+#ifndef NDEBUG
+ void print(std::ostream *O) const {
+ if (O) print(*O);
+ }
+ void print(std::ostream &O) const {
+ O << "." << Tag;
+ if (Number) O << Number;
+ }
+#endif
+};
+
+//===----------------------------------------------------------------------===//
+/// DIEAbbrevData - Dwarf abbreviation data, describes the one attribute of a
+/// Dwarf abbreviation.
+class DIEAbbrevData {
+private:
+ /// Attribute - Dwarf attribute code.
+ ///
+ unsigned Attribute;
+
+ /// Form - Dwarf form code.
+ ///
+ unsigned Form;
+
+public:
+ DIEAbbrevData(unsigned A, unsigned F)
+ : Attribute(A)
+ , Form(F)
+ {}
+
+ // Accessors.
+ unsigned getAttribute() const { return Attribute; }
+ unsigned getForm() const { return Form; }
+
+ /// Profile - Used to gather unique data for the abbreviation folding set.
+ ///
+ void Profile(FoldingSetNodeID &ID)const {
+ ID.AddInteger(Attribute);
+ ID.AddInteger(Form);
+ }
+};
+
+//===----------------------------------------------------------------------===//
+/// DIEAbbrev - Dwarf abbreviation, describes the organization of a debug
+/// information object.
+class DIEAbbrev : public FoldingSetNode {
+private:
+ /// Tag - Dwarf tag code.
+ ///
+ unsigned Tag;
+
+ /// Unique number for node.
+ ///
+ unsigned Number;
+
+ /// ChildrenFlag - Dwarf children flag.
+ ///
+ unsigned ChildrenFlag;
+
+ /// Data - Raw data bytes for abbreviation.
+ ///
+ std::vector<DIEAbbrevData> Data;
+
+public:
+
+ DIEAbbrev(unsigned T, unsigned C)
+ : Tag(T)
+ , ChildrenFlag(C)
+ , Data()
+ {}
+ ~DIEAbbrev() {}
+
+ // Accessors.
+ unsigned getTag() const { return Tag; }
+ unsigned getNumber() const { return Number; }
+ unsigned getChildrenFlag() const { return ChildrenFlag; }
+ const std::vector<DIEAbbrevData> &getData() const { return Data; }
+ void setTag(unsigned T) { Tag = T; }
+ void setChildrenFlag(unsigned CF) { ChildrenFlag = CF; }
+ void setNumber(unsigned N) { Number = N; }
+
+ /// AddAttribute - Adds another set of attribute information to the
+ /// abbreviation.
+ void AddAttribute(unsigned Attribute, unsigned Form) {
+ Data.push_back(DIEAbbrevData(Attribute, Form));
+ }
+
+ /// AddFirstAttribute - Adds a set of attribute information to the front
+ /// of the abbreviation.
+ void AddFirstAttribute(unsigned Attribute, unsigned Form) {
+ Data.insert(Data.begin(), DIEAbbrevData(Attribute, Form));
+ }
+
+ /// Profile - Used to gather unique data for the abbreviation folding set.
+ ///
+ void Profile(FoldingSetNodeID &ID) {
+ ID.AddInteger(Tag);
+ ID.AddInteger(ChildrenFlag);
+
+ // For each attribute description.
+ for (unsigned i = 0, N = Data.size(); i < N; ++i)
+ Data[i].Profile(ID);
+ }
+
+ /// Emit - Print the abbreviation using the specified Dwarf writer.
+ ///
+ void Emit(const DwarfDebug &DD) const;
+
+#ifndef NDEBUG
+ void print(std::ostream *O) {
+ if (O) print(*O);
+ }
+ void print(std::ostream &O);
+ void dump();
+#endif
+};
+
+//===----------------------------------------------------------------------===//
+/// DIE - A structured debug information entry. Has an abbreviation which
+/// describes it's organization.
+class DIE : public FoldingSetNode {
+protected:
+ /// Abbrev - Buffer for constructing abbreviation.
+ ///
+ DIEAbbrev Abbrev;
+
+ /// Offset - Offset in debug info section.
+ ///
+ unsigned Offset;
+
+ /// Size - Size of instance + children.
+ ///
+ unsigned Size;
+
+ /// Children DIEs.
+ ///
+ std::vector<DIE *> Children;
+
+ /// Attributes values.
+ ///
+ std::vector<DIEValue *> Values;
+
+public:
+ DIE(unsigned Tag)
+ : Abbrev(Tag, DW_CHILDREN_no)
+ , Offset(0)
+ , Size(0)
+ , Children()
+ , Values()
+ {}
+ virtual ~DIE();
+
+ // Accessors.
+ DIEAbbrev &getAbbrev() { return Abbrev; }
+ unsigned getAbbrevNumber() const {
+ return Abbrev.getNumber();
+ }
+ unsigned getTag() const { return Abbrev.getTag(); }
+ unsigned getOffset() const { return Offset; }
+ unsigned getSize() const { return Size; }
+ const std::vector<DIE *> &getChildren() const { return Children; }
+ std::vector<DIEValue *> &getValues() { return Values; }
+ void setTag(unsigned Tag) { Abbrev.setTag(Tag); }
+ void setOffset(unsigned O) { Offset = O; }
+ void setSize(unsigned S) { Size = S; }
+
+ /// AddValue - Add a value and attributes to a DIE.
+ ///
+ void AddValue(unsigned Attribute, unsigned Form, DIEValue *Value) {
+ Abbrev.AddAttribute(Attribute, Form);
+ Values.push_back(Value);
+ }
+
+ /// SiblingOffset - Return the offset of the debug information entry's
+ /// sibling.
+ unsigned SiblingOffset() const { return Offset + Size; }
+
+ /// AddSiblingOffset - Add a sibling offset field to the front of the DIE.
+ ///
+ void AddSiblingOffset();
+
+ /// AddChild - Add a child to the DIE.
+ ///
+ void AddChild(DIE *Child) {
+ Abbrev.setChildrenFlag(DW_CHILDREN_yes);
+ Children.push_back(Child);
+ }
+
+ /// Detach - Detaches objects connected to it after copying.
+ ///
+ void Detach() {
+ Children.clear();
+ }
+
+ /// Profile - Used to gather unique data for the value folding set.
+ ///
+ void Profile(FoldingSetNodeID &ID) ;
+
+#ifndef NDEBUG
+ void print(std::ostream *O, unsigned IncIndent = 0) {
+ if (O) print(*O, IncIndent);
+ }
+ void print(std::ostream &O, unsigned IncIndent = 0);
+ void dump();
+#endif
+};
+
+//===----------------------------------------------------------------------===//
+/// DIEValue - A debug information entry value.
+///
+class DIEValue : public FoldingSetNode {
+public:
+ enum {
+ isInteger,
+ isString,
+ isLabel,
+ isAsIsLabel,
+ isDelta,
+ isEntry,
+ isBlock
+ };
+
+ /// Type - Type of data stored in the value.
+ ///
+ unsigned Type;
+
+ DIEValue(unsigned T)
+ : Type(T)
+ {}
+ virtual ~DIEValue() {}
+
+ // Accessors
+ unsigned getType() const { return Type; }
+
+ // Implement isa/cast/dyncast.
+ static bool classof(const DIEValue *) { return true; }
+
+ /// EmitValue - Emit value via the Dwarf writer.
+ ///
+ virtual void EmitValue(DwarfDebug &DD, unsigned Form) = 0;
+
+ /// SizeOf - Return the size of a value in bytes.
+ ///
+ virtual unsigned SizeOf(const DwarfDebug &DD, unsigned Form) const = 0;
+
+ /// Profile - Used to gather unique data for the value folding set.
+ ///
+ virtual void Profile(FoldingSetNodeID &ID) = 0;
+
+#ifndef NDEBUG
+ void print(std::ostream *O) {
+ if (O) print(*O);
+ }
+ virtual void print(std::ostream &O) = 0;
+ void dump();
+#endif
+};
+
+//===----------------------------------------------------------------------===//
+/// DWInteger - An integer value DIE.
+///
+class DIEInteger : public DIEValue {
+private:
+ uint64_t Integer;
+
+public:
+ DIEInteger(uint64_t I) : DIEValue(isInteger), Integer(I) {}
+
+ // Implement isa/cast/dyncast.
+ static bool classof(const DIEInteger *) { return true; }
+ static bool classof(const DIEValue *I) { return I->Type == isInteger; }
+
+ /// BestForm - Choose the best form for integer.
+ ///
+ static unsigned BestForm(bool IsSigned, uint64_t Integer) {
+ if (IsSigned) {
+ if ((char)Integer == (signed)Integer) return DW_FORM_data1;
+ if ((short)Integer == (signed)Integer) return DW_FORM_data2;
+ if ((int)Integer == (signed)Integer) return DW_FORM_data4;
+ } else {
+ if ((unsigned char)Integer == Integer) return DW_FORM_data1;
+ if ((unsigned short)Integer == Integer) return DW_FORM_data2;
+ if ((unsigned int)Integer == Integer) return DW_FORM_data4;
+ }
+ return DW_FORM_data8;
+ }
+
+ /// EmitValue - Emit integer of appropriate size.
+ ///
+ virtual void EmitValue(DwarfDebug &DD, unsigned Form);
+
+ /// SizeOf - Determine size of integer value in bytes.
+ ///
+ virtual unsigned SizeOf(const DwarfDebug &DD, unsigned Form) const;
+
+ /// Profile - Used to gather unique data for the value folding set.
+ ///
+ static void Profile(FoldingSetNodeID &ID, unsigned Integer) {
+ ID.AddInteger(isInteger);
+ ID.AddInteger(Integer);
+ }
+ virtual void Profile(FoldingSetNodeID &ID) { Profile(ID, Integer); }
+
+#ifndef NDEBUG
+ virtual void print(std::ostream &O) {
+ O << "Int: " << (int64_t)Integer
+ << " 0x" << std::hex << Integer << std::dec;
+ }
+#endif
+};
+
+//===----------------------------------------------------------------------===//
+/// DIEString - A string value DIE.
+///
+class DIEString : public DIEValue {
+public:
+ const std::string String;
+
+ DIEString(const std::string &S) : DIEValue(isString), String(S) {}
+
+ // Implement isa/cast/dyncast.
+ static bool classof(const DIEString *) { return true; }
+ static bool classof(const DIEValue *S) { return S->Type == isString; }
+
+ /// EmitValue - Emit string value.
+ ///
+ virtual void EmitValue(DwarfDebug &DD, unsigned Form);
+
+ /// SizeOf - Determine size of string value in bytes.
+ ///
+ virtual unsigned SizeOf(const DwarfDebug &DD, unsigned Form) const {
+ return String.size() + sizeof(char); // sizeof('\0');
+ }
+
+ /// Profile - Used to gather unique data for the value folding set.
+ ///
+ static void Profile(FoldingSetNodeID &ID, const std::string &String) {
+ ID.AddInteger(isString);
+ ID.AddString(String);
+ }
+ virtual void Profile(FoldingSetNodeID &ID) { Profile(ID, String); }
+
+#ifndef NDEBUG
+ virtual void print(std::ostream &O) {
+ O << "Str: \"" << String << "\"";
+ }
+#endif
+};
+
+//===----------------------------------------------------------------------===//
+/// DIEDwarfLabel - A Dwarf internal label expression DIE.
+//
+class DIEDwarfLabel : public DIEValue {
+public:
+
+ const DWLabel Label;
+
+ DIEDwarfLabel(const DWLabel &L) : DIEValue(isLabel), Label(L) {}
+
+ // Implement isa/cast/dyncast.
+ static bool classof(const DIEDwarfLabel *) { return true; }
+ static bool classof(const DIEValue *L) { return L->Type == isLabel; }
+
+ /// EmitValue - Emit label value.
+ ///
+ virtual void EmitValue(DwarfDebug &DD, unsigned Form);
+
+ /// SizeOf - Determine size of label value in bytes.
+ ///
+ virtual unsigned SizeOf(const DwarfDebug &DD, unsigned Form) const;
+
+ /// Profile - Used to gather unique data for the value folding set.
+ ///
+ static void Profile(FoldingSetNodeID &ID, const DWLabel &Label) {
+ ID.AddInteger(isLabel);
+ Label.Profile(ID);
+ }
+ virtual void Profile(FoldingSetNodeID &ID) { Profile(ID, Label); }
+
+#ifndef NDEBUG
+ virtual void print(std::ostream &O) {
+ O << "Lbl: ";
+ Label.print(O);
+ }
+#endif
+};
+
+
+//===----------------------------------------------------------------------===//
+/// DIEObjectLabel - A label to an object in code or data.
+//
+class DIEObjectLabel : public DIEValue {
+public:
+ const std::string Label;
+
+ DIEObjectLabel(const std::string &L) : DIEValue(isAsIsLabel), Label(L) {}
+
+ // Implement isa/cast/dyncast.
+ static bool classof(const DIEObjectLabel *) { return true; }
+ static bool classof(const DIEValue *L) { return L->Type == isAsIsLabel; }
+
+ /// EmitValue - Emit label value.
+ ///
+ virtual void EmitValue(DwarfDebug &DD, unsigned Form);
+
+ /// SizeOf - Determine size of label value in bytes.
+ ///
+ virtual unsigned SizeOf(const DwarfDebug &DD, unsigned Form) const;
+
+ /// Profile - Used to gather unique data for the value folding set.
+ ///
+ static void Profile(FoldingSetNodeID &ID, const std::string &Label) {
+ ID.AddInteger(isAsIsLabel);
+ ID.AddString(Label);
+ }
+ virtual void Profile(FoldingSetNodeID &ID) { Profile(ID, Label); }
+
+#ifndef NDEBUG
+ virtual void print(std::ostream &O) {
+ O << "Obj: " << Label;
+ }
+#endif
+};
+
+//===----------------------------------------------------------------------===//
+/// DIEDelta - A simple label difference DIE.
+///
+class DIEDelta : public DIEValue {
+public:
+ const DWLabel LabelHi;
+ const DWLabel LabelLo;
+
+ DIEDelta(const DWLabel &Hi, const DWLabel &Lo)
+ : DIEValue(isDelta), LabelHi(Hi), LabelLo(Lo) {}
+
+ // Implement isa/cast/dyncast.
+ static bool classof(const DIEDelta *) { return true; }
+ static bool classof(const DIEValue *D) { return D->Type == isDelta; }
+
+ /// EmitValue - Emit delta value.
+ ///
+ virtual void EmitValue(DwarfDebug &DD, unsigned Form);
+
+ /// SizeOf - Determine size of delta value in bytes.
+ ///
+ virtual unsigned SizeOf(const DwarfDebug &DD, unsigned Form) const;
+
+ /// Profile - Used to gather unique data for the value folding set.
+ ///
+ static void Profile(FoldingSetNodeID &ID, const DWLabel &LabelHi,
+ const DWLabel &LabelLo) {
+ ID.AddInteger(isDelta);
+ LabelHi.Profile(ID);
+ LabelLo.Profile(ID);
+ }
+ virtual void Profile(FoldingSetNodeID &ID) { Profile(ID, LabelHi, LabelLo); }
+
+#ifndef NDEBUG
+ virtual void print(std::ostream &O) {
+ O << "Del: ";
+ LabelHi.print(O);
+ O << "-";
+ LabelLo.print(O);
+ }
+#endif
+};
+
+//===----------------------------------------------------------------------===//
+/// DIEntry - A pointer to another debug information entry. An instance of this
+/// class can also be used as a proxy for a debug information entry not yet
+/// defined (ie. types.)
+class DIEntry : public DIEValue {
+public:
+ DIE *Entry;
+
+ DIEntry(DIE *E) : DIEValue(isEntry), Entry(E) {}
+
+ // Implement isa/cast/dyncast.
+ static bool classof(const DIEntry *) { return true; }
+ static bool classof(const DIEValue *E) { return E->Type == isEntry; }
+
+ /// EmitValue - Emit debug information entry offset.
+ ///
+ virtual void EmitValue(DwarfDebug &DD, unsigned Form);
+
+ /// SizeOf - Determine size of debug information entry in bytes.
+ ///
+ virtual unsigned SizeOf(const DwarfDebug &DD, unsigned Form) const {
+ return sizeof(int32_t);
+ }
+
+ /// Profile - Used to gather unique data for the value folding set.
+ ///
+ static void Profile(FoldingSetNodeID &ID, DIE *Entry) {
+ ID.AddInteger(isEntry);
+ ID.AddPointer(Entry);
+ }
+ virtual void Profile(FoldingSetNodeID &ID) {
+ ID.AddInteger(isEntry);
+
+ if (Entry) {
+ ID.AddPointer(Entry);
+ } else {
+ ID.AddPointer(this);
+ }
+ }
+
+#ifndef NDEBUG
+ virtual void print(std::ostream &O) {
+ O << "Die: 0x" << std::hex << (intptr_t)Entry << std::dec;
+ }
+#endif
+};
+
+//===----------------------------------------------------------------------===//
+/// DIEBlock - A block of values. Primarily used for location expressions.
+//
+class DIEBlock : public DIEValue, public DIE {
+public:
+ unsigned Size; // Size in bytes excluding size header.
+
+ DIEBlock()
+ : DIEValue(isBlock)
+ , DIE(0)
+ , Size(0)
+ {}
+ ~DIEBlock() {
+ }
+
+ // Implement isa/cast/dyncast.
+ static bool classof(const DIEBlock *) { return true; }
+ static bool classof(const DIEValue *E) { return E->Type == isBlock; }
+
+ /// ComputeSize - calculate the size of the block.
+ ///
+ unsigned ComputeSize(DwarfDebug &DD);
+
+ /// BestForm - Choose the best form for data.
+ ///
+ unsigned BestForm() const {
+ if ((unsigned char)Size == Size) return DW_FORM_block1;
+ if ((unsigned short)Size == Size) return DW_FORM_block2;
+ if ((unsigned int)Size == Size) return DW_FORM_block4;
+ return DW_FORM_block;
+ }
+
+ /// EmitValue - Emit block data.
+ ///
+ virtual void EmitValue(DwarfDebug &DD, unsigned Form);
+
+ /// SizeOf - Determine size of block data in bytes.
+ ///
+ virtual unsigned SizeOf(const DwarfDebug &DD, unsigned Form) const;
+
+
+ /// Profile - Used to gather unique data for the value folding set.
+ ///
+ virtual void Profile(FoldingSetNodeID &ID) {
+ ID.AddInteger(isBlock);
+ DIE::Profile(ID);
+ }
+
+#ifndef NDEBUG
+ virtual void print(std::ostream &O) {
+ O << "Blk: ";
+ DIE::print(O, 5);
+ }
+#endif
+};
+
+//===----------------------------------------------------------------------===//
+/// CompileUnit - This dwarf writer support class manages information associate
+/// with a source file.
+class CompileUnit {
+private:
+ /// Desc - Compile unit debug descriptor.
+ ///
+ CompileUnitDesc *Desc;
+
+ /// ID - File identifier for source.
+ ///
+ unsigned ID;
+
+ /// Die - Compile unit debug information entry.
+ ///
+ DIE *Die;
+
+ /// DescToDieMap - Tracks the mapping of unit level debug informaton
+ /// descriptors to debug information entries.
+ std::map<DebugInfoDesc *, DIE *> DescToDieMap;
+
+ /// DescToDIEntryMap - Tracks the mapping of unit level debug informaton
+ /// descriptors to debug information entries using a DIEntry proxy.
+ std::map<DebugInfoDesc *, DIEntry *> DescToDIEntryMap;
+
+ /// Globals - A map of globally visible named entities for this unit.
+ ///
+ std::map<std::string, DIE *> Globals;
+
+ /// DiesSet - Used to uniquely define dies within the compile unit.
+ ///
+ FoldingSet<DIE> DiesSet;
+
+ /// Dies - List of all dies in the compile unit.
+ ///
+ std::vector<DIE *> Dies;
+
+public:
+ CompileUnit(CompileUnitDesc *CUD, unsigned I, DIE *D)
+ : Desc(CUD)
+ , ID(I)
+ , Die(D)
+ , DescToDieMap()
+ , DescToDIEntryMap()
+ , Globals()
+ , DiesSet(InitDiesSetSize)
+ , Dies()
+ {}
+
+ ~CompileUnit() {
+ delete Die;
+
+ for (unsigned i = 0, N = Dies.size(); i < N; ++i)
+ delete Dies[i];
+ }
+
+ // Accessors.
+ CompileUnitDesc *getDesc() const { return Desc; }
+ unsigned getID() const { return ID; }
+ DIE* getDie() const { return Die; }
+ std::map<std::string, DIE *> &getGlobals() { return Globals; }
+
+ /// hasContent - Return true if this compile unit has something to write out.
+ ///
+ bool hasContent() const {
+ return !Die->getChildren().empty();
+ }
+
+ /// AddGlobal - Add a new global entity to the compile unit.
+ ///
+ void AddGlobal(const std::string &Name, DIE *Die) {
+ Globals[Name] = Die;
+ }
+
+ /// getDieMapSlotFor - Returns the debug information entry map slot for the
+ /// specified debug descriptor.
+ DIE *&getDieMapSlotFor(DebugInfoDesc *DID) {
+ return DescToDieMap[DID];
+ }
+
+ /// getDIEntrySlotFor - Returns the debug information entry proxy slot for the
+ /// specified debug descriptor.
+ DIEntry *&getDIEntrySlotFor(DebugInfoDesc *DID) {
+ return DescToDIEntryMap[DID];
+ }
+
+ /// AddDie - Adds or interns the DIE to the compile unit.
+ ///
+ DIE *AddDie(DIE &Buffer) {
+ FoldingSetNodeID ID;
+ Buffer.Profile(ID);
+ void *Where;
+ DIE *Die = DiesSet.FindNodeOrInsertPos(ID, Where);
+
+ if (!Die) {
+ Die = new DIE(Buffer);
+ DiesSet.InsertNode(Die, Where);
+ this->Die->AddChild(Die);
+ Buffer.Detach();
+ }
+
+ return Die;
+ }
+};
+
+//===----------------------------------------------------------------------===//
+/// Dwarf - Emits general Dwarf directives.
+///
+class Dwarf {
+
+protected:
+
+ //===--------------------------------------------------------------------===//
+ // Core attributes used by the Dwarf writer.
+ //
+
+ //
+ /// O - Stream to .s file.
+ ///
+ std::ostream &O;
+
+ /// Asm - Target of Dwarf emission.
+ ///
+ AsmPrinter *Asm;
+
+ /// TAI - Target Asm Printer.
+ const TargetAsmInfo *TAI;
+
+ /// TD - Target data.
+ const TargetData *TD;
+
+ /// RI - Register Information.
+ const MRegisterInfo *RI;
+
+ /// M - Current module.
+ ///
+ Module *M;
+
+ /// MF - Current machine function.
+ ///
+ MachineFunction *MF;
+
+ /// MMI - Collected machine module information.
+ ///
+ MachineModuleInfo *MMI;
+
+ /// SubprogramCount - The running count of functions being compiled.
+ ///
+ unsigned SubprogramCount;
+
+ unsigned SetCounter;
+ Dwarf(std::ostream &OS, AsmPrinter *A, const TargetAsmInfo *T)
+ : O(OS)
+ , Asm(A)
+ , TAI(T)
+ , TD(Asm->TM.getTargetData())
+ , RI(Asm->TM.getRegisterInfo())
+ , M(NULL)
+ , MF(NULL)
+ , MMI(NULL)
+ , SubprogramCount(0)
+ , SetCounter(1)
+ {
+ }
+
+public:
+
+ //===--------------------------------------------------------------------===//
+ // Accessors.
+ //
+ AsmPrinter *getAsm() const { return Asm; }
+ MachineModuleInfo *getMMI() const { return MMI; }
+ const TargetAsmInfo *getTargetAsmInfo() const { return TAI; }
+
+ /// PrintLabelName - Print label name in form used by Dwarf writer.
+ ///
+ void PrintLabelName(DWLabel Label) const {
+ PrintLabelName(Label.Tag, Label.Number);
+ }
+ void PrintLabelName(const char *Tag, unsigned Number,
+ bool isInSection = false) const {
+ if (isInSection && TAI->getDwarfSectionOffsetDirective())
+ O << TAI->getDwarfSectionOffsetDirective() << Tag;
+ else
+ O << TAI->getPrivateGlobalPrefix() << Tag;
+ if (Number) O << Number;
+ }
+
+ /// EmitLabel - Emit location label for internal use by Dwarf.
+ ///
+ void EmitLabel(DWLabel Label) const {
+ EmitLabel(Label.Tag, Label.Number);
+ }
+ void EmitLabel(const char *Tag, unsigned Number) const {
+ PrintLabelName(Tag, Number);
+ O << ":\n";
+ }
+
+ /// EmitReference - Emit a reference to a label.
+ ///
+ void EmitReference(DWLabel Label, bool IsPCRelative = false) const {
+ EmitReference(Label.Tag, Label.Number, IsPCRelative);
+ }
+ void EmitReference(const char *Tag, unsigned Number,
+ bool IsPCRelative = false) const {
+ if (TAI->getAddressSize() == sizeof(int32_t))
+ O << TAI->getData32bitsDirective();
+ else
+ O << TAI->getData64bitsDirective();
+
+ PrintLabelName(Tag, Number);
+
+ if (IsPCRelative) O << "-" << TAI->getPCSymbol();
+ }
+ void EmitReference(const std::string &Name, bool IsPCRelative = false) const {
+ if (TAI->getAddressSize() == sizeof(int32_t))
+ O << TAI->getData32bitsDirective();
+ else
+ O << TAI->getData64bitsDirective();
+
+ O << Name;
+
+ if (IsPCRelative) O << "-" << TAI->getPCSymbol();
+ }
+
+ /// EmitDifference - Emit the difference between two labels. Some
+ /// assemblers do not behave with absolute expressions with data directives,
+ /// so there is an option (needsSet) to use an intermediary set expression.
+ void EmitDifference(DWLabel LabelHi, DWLabel LabelLo,
+ bool IsSmall = false) {
+ EmitDifference(LabelHi.Tag, LabelHi.Number,
+ LabelLo.Tag, LabelLo.Number,
+ IsSmall);
+ }
+ void EmitDifference(const char *TagHi, unsigned NumberHi,
+ const char *TagLo, unsigned NumberLo,
+ bool IsSmall = false) {
+ if (TAI->needsSet()) {
+ O << "\t.set\t";
+ PrintLabelName("set", SetCounter);
+ O << ",";
+ PrintLabelName(TagHi, NumberHi);
+ O << "-";
+ PrintLabelName(TagLo, NumberLo);
+ O << "\n";
+
+ if (IsSmall || TAI->getAddressSize() == sizeof(int32_t))
+ O << TAI->getData32bitsDirective();
+ else
+ O << TAI->getData64bitsDirective();
+
+ PrintLabelName("set", SetCounter);
+
+ ++SetCounter;
+ } else {
+ if (IsSmall || TAI->getAddressSize() == sizeof(int32_t))
+ O << TAI->getData32bitsDirective();
+ else
+ O << TAI->getData64bitsDirective();
+
+ PrintLabelName(TagHi, NumberHi);
+ O << "-";
+ PrintLabelName(TagLo, NumberLo);
+ }
+ }
+
+ void EmitSectionOffset(const char* Label, const char* Section,
+ unsigned LabelNumber, unsigned SectionNumber,
+ bool IsSmall = false, bool isEH = false) {
+ bool printAbsolute = false;
+ if (TAI->needsSet()) {
+ O << "\t.set\t";
+ PrintLabelName("set", SetCounter);
+ O << ",";
+ PrintLabelName(Label, LabelNumber, true);
+
+ if (isEH)
+ printAbsolute = TAI->isAbsoluteEHSectionOffsets();
+ else
+ printAbsolute = TAI->isAbsoluteDebugSectionOffsets();
+
+ if (!printAbsolute) {
+ O << "-";
+ PrintLabelName(Section, SectionNumber);
+ }
+ O << "\n";
+
+ if (IsSmall || TAI->getAddressSize() == sizeof(int32_t))
+ O << TAI->getData32bitsDirective();
+ else
+ O << TAI->getData64bitsDirective();
+
+ PrintLabelName("set", SetCounter);
+ ++SetCounter;
+ } else {
+ if (IsSmall || TAI->getAddressSize() == sizeof(int32_t))
+ O << TAI->getData32bitsDirective();
+ else
+ O << TAI->getData64bitsDirective();
+
+ PrintLabelName(Label, LabelNumber, true);
+
+ if (isEH)
+ printAbsolute = TAI->isAbsoluteEHSectionOffsets();
+ else
+ printAbsolute = TAI->isAbsoluteDebugSectionOffsets();
+
+ if (!printAbsolute) {
+ O << "-";
+ PrintLabelName(Section, SectionNumber);
+ }
+ }
+ }
+
+ /// EmitFrameMoves - Emit frame instructions to describe the layout of the
+ /// frame.
+ void EmitFrameMoves(const char *BaseLabel, unsigned BaseLabelID,
+ const std::vector<MachineMove> &Moves) {
+ int stackGrowth =
+ Asm->TM.getFrameInfo()->getStackGrowthDirection() ==
+ TargetFrameInfo::StackGrowsUp ?
+ TAI->getAddressSize() : -TAI->getAddressSize();
+ bool IsLocal = BaseLabel && strcmp(BaseLabel, "label") == 0;
+
+ for (unsigned i = 0, N = Moves.size(); i < N; ++i) {
+ const MachineMove &Move = Moves[i];
+ unsigned LabelID = Move.getLabelID();
+
+ if (LabelID) {
+ LabelID = MMI->MappedLabel(LabelID);
+
+ // Throw out move if the label is invalid.
+ if (!LabelID) continue;
+ }
+
+ const MachineLocation &Dst = Move.getDestination();
+ const MachineLocation &Src = Move.getSource();
+
+ // Advance row if new location.
+ if (BaseLabel && LabelID && (BaseLabelID != LabelID || !IsLocal)) {
+ Asm->EmitInt8(DW_CFA_advance_loc4);
+ Asm->EOL("DW_CFA_advance_loc4");
+ EmitDifference("label", LabelID, BaseLabel, BaseLabelID, true);
+ Asm->EOL();
+
+ BaseLabelID = LabelID;
+ BaseLabel = "label";
+ IsLocal = true;
+ }
+
+ // If advancing cfa.
+ if (Dst.isRegister() && Dst.getRegister() == MachineLocation::VirtualFP) {
+ if (!Src.isRegister()) {
+ if (Src.getRegister() == MachineLocation::VirtualFP) {
+ Asm->EmitInt8(DW_CFA_def_cfa_offset);
+ Asm->EOL("DW_CFA_def_cfa_offset");
+ } else {
+ Asm->EmitInt8(DW_CFA_def_cfa);
+ Asm->EOL("DW_CFA_def_cfa");
+ Asm->EmitULEB128Bytes(RI->getDwarfRegNum(Src.getRegister()));
+ Asm->EOL("Register");
+ }
+
+ int Offset = -Src.getOffset();
+
+ Asm->EmitULEB128Bytes(Offset);
+ Asm->EOL("Offset");
+ } else {
+ assert(0 && "Machine move no supported yet.");
+ }
+ } else if (Src.isRegister() &&
+ Src.getRegister() == MachineLocation::VirtualFP) {
+ if (Dst.isRegister()) {
+ Asm->EmitInt8(DW_CFA_def_cfa_register);
+ Asm->EOL("DW_CFA_def_cfa_register");
+ Asm->EmitULEB128Bytes(RI->getDwarfRegNum(Dst.getRegister()));
+ Asm->EOL("Register");
+ } else {
+ assert(0 && "Machine move no supported yet.");
+ }
+ } else {
+ unsigned Reg = RI->getDwarfRegNum(Src.getRegister());
+ int Offset = Dst.getOffset() / stackGrowth;
+
+ if (Offset < 0) {
+ Asm->EmitInt8(DW_CFA_offset_extended_sf);
+ Asm->EOL("DW_CFA_offset_extended_sf");
+ Asm->EmitULEB128Bytes(Reg);
+ Asm->EOL("Reg");
+ Asm->EmitSLEB128Bytes(Offset);
+ Asm->EOL("Offset");
+ } else if (Reg < 64) {
+ Asm->EmitInt8(DW_CFA_offset + Reg);
+ Asm->EOL("DW_CFA_offset + Reg");
+ Asm->EmitULEB128Bytes(Offset);
+ Asm->EOL("Offset");
+ } else {
+ Asm->EmitInt8(DW_CFA_offset_extended);
+ Asm->EOL("DW_CFA_offset_extended");
+ Asm->EmitULEB128Bytes(Reg);
+ Asm->EOL("Reg");
+ Asm->EmitULEB128Bytes(Offset);
+ Asm->EOL("Offset");
+ }
+ }
+ }
+ }
+
+};
+
+//===----------------------------------------------------------------------===//
+/// DwarfDebug - Emits Dwarf debug directives.
+///
+class DwarfDebug : public Dwarf {
+
+private:
+ //===--------------------------------------------------------------------===//
+ // Attributes used to construct specific Dwarf sections.
+ //
+
+ /// CompileUnits - All the compile units involved in this build. The index
+ /// of each entry in this vector corresponds to the sources in MMI.
+ std::vector<CompileUnit *> CompileUnits;
+
+ /// AbbreviationsSet - Used to uniquely define abbreviations.
+ ///
+ FoldingSet<DIEAbbrev> AbbreviationsSet;
+
+ /// Abbreviations - A list of all the unique abbreviations in use.
+ ///
+ std::vector<DIEAbbrev *> Abbreviations;
+
+ /// ValuesSet - Used to uniquely define values.
+ ///
+ FoldingSet<DIEValue> ValuesSet;
+
+ /// Values - A list of all the unique values in use.
+ ///
+ std::vector<DIEValue *> Values;
+
+ /// StringPool - A UniqueVector of strings used by indirect references.
+ ///
+ UniqueVector<std::string> StringPool;
+
+ /// UnitMap - Map debug information descriptor to compile unit.
+ ///
+ std::map<DebugInfoDesc *, CompileUnit *> DescToUnitMap;
+
+ /// SectionMap - Provides a unique id per text section.
+ ///
+ UniqueVector<std::string> SectionMap;
+
+ /// SectionSourceLines - Tracks line numbers per text section.
+ ///
+ std::vector<std::vector<SourceLineInfo> > SectionSourceLines;
+
+ /// didInitial - Flag to indicate if initial emission has been done.
+ ///
+ bool didInitial;
+
+ /// shouldEmit - Flag to indicate if debug information should be emitted.
+ ///
+ bool shouldEmit;
+
+ struct FunctionDebugFrameInfo {
+ unsigned Number;
+ std::vector<MachineMove> Moves;
+
+ FunctionDebugFrameInfo(unsigned Num, const std::vector<MachineMove> &M):
+ Number(Num), Moves(M) { };
+ };
+
+ std::vector<FunctionDebugFrameInfo> DebugFrames;
+
+public:
+
+ /// ShouldEmitDwarf - Returns true if Dwarf declarations should be made.
+ ///
+ bool ShouldEmitDwarf() const { return shouldEmit; }
+
+ /// AssignAbbrevNumber - Define a unique number for the abbreviation.
+ ///
+ void AssignAbbrevNumber(DIEAbbrev &Abbrev) {
+ // Profile the node so that we can make it unique.
+ FoldingSetNodeID ID;
+ Abbrev.Profile(ID);
+
+ // Check the set for priors.
+ DIEAbbrev *InSet = AbbreviationsSet.GetOrInsertNode(&Abbrev);
+
+ // If it's newly added.
+ if (InSet == &Abbrev) {
+ // Add to abbreviation list.
+ Abbreviations.push_back(&Abbrev);
+ // Assign the vector position + 1 as its number.
+ Abbrev.setNumber(Abbreviations.size());
+ } else {
+ // Assign existing abbreviation number.
+ Abbrev.setNumber(InSet->getNumber());
+ }
+ }
+
+ /// NewString - Add a string to the constant pool and returns a label.
+ ///
+ DWLabel NewString(const std::string &String) {
+ unsigned StringID = StringPool.insert(String);
+ return DWLabel("string", StringID);
+ }
+
+ /// NewDIEntry - Creates a new DIEntry to be a proxy for a debug information
+ /// entry.
+ DIEntry *NewDIEntry(DIE *Entry = NULL) {
+ DIEntry *Value;
+
+ if (Entry) {
+ FoldingSetNodeID ID;
+ DIEntry::Profile(ID, Entry);
+ void *Where;
+ Value = static_cast<DIEntry *>(ValuesSet.FindNodeOrInsertPos(ID, Where));
+
+ if (Value) return Value;
+
+ Value = new DIEntry(Entry);
+ ValuesSet.InsertNode(Value, Where);
+ } else {
+ Value = new DIEntry(Entry);
+ }
+
+ Values.push_back(Value);
+ return Value;
+ }
+
+ /// SetDIEntry - Set a DIEntry once the debug information entry is defined.
+ ///
+ void SetDIEntry(DIEntry *Value, DIE *Entry) {
+ Value->Entry = Entry;
+ // Add to values set if not already there. If it is, we merely have a
+ // duplicate in the values list (no harm.)
+ ValuesSet.GetOrInsertNode(Value);
+ }
+
+ /// AddUInt - Add an unsigned integer attribute data and value.
+ ///
+ void AddUInt(DIE *Die, unsigned Attribute, unsigned Form, uint64_t Integer) {
+ if (!Form) Form = DIEInteger::BestForm(false, Integer);
+
+ FoldingSetNodeID ID;
+ DIEInteger::Profile(ID, Integer);
+ void *Where;
+ DIEValue *Value = ValuesSet.FindNodeOrInsertPos(ID, Where);
+ if (!Value) {
+ Value = new DIEInteger(Integer);
+ ValuesSet.InsertNode(Value, Where);
+ Values.push_back(Value);
+ }
+
+ Die->AddValue(Attribute, Form, Value);
+ }
+
+ /// AddSInt - Add an signed integer attribute data and value.
+ ///
+ void AddSInt(DIE *Die, unsigned Attribute, unsigned Form, int64_t Integer) {
+ if (!Form) Form = DIEInteger::BestForm(true, Integer);
+
+ FoldingSetNodeID ID;
+ DIEInteger::Profile(ID, (uint64_t)Integer);
+ void *Where;
+ DIEValue *Value = ValuesSet.FindNodeOrInsertPos(ID, Where);
+ if (!Value) {
+ Value = new DIEInteger(Integer);
+ ValuesSet.InsertNode(Value, Where);
+ Values.push_back(Value);
+ }
+
+ Die->AddValue(Attribute, Form, Value);
+ }
+
+ /// AddString - Add a std::string attribute data and value.
+ ///
+ void AddString(DIE *Die, unsigned Attribute, unsigned Form,
+ const std::string &String) {
+ FoldingSetNodeID ID;
+ DIEString::Profile(ID, String);
+ void *Where;
+ DIEValue *Value = ValuesSet.FindNodeOrInsertPos(ID, Where);
+ if (!Value) {
+ Value = new DIEString(String);
+ ValuesSet.InsertNode(Value, Where);
+ Values.push_back(Value);
+ }
+
+ Die->AddValue(Attribute, Form, Value);
+ }
+
+ /// AddLabel - Add a Dwarf label attribute data and value.
+ ///
+ void AddLabel(DIE *Die, unsigned Attribute, unsigned Form,
+ const DWLabel &Label) {
+ FoldingSetNodeID ID;
+ DIEDwarfLabel::Profile(ID, Label);
+ void *Where;
+ DIEValue *Value = ValuesSet.FindNodeOrInsertPos(ID, Where);
+ if (!Value) {
+ Value = new DIEDwarfLabel(Label);
+ ValuesSet.InsertNode(Value, Where);
+ Values.push_back(Value);
+ }
+
+ Die->AddValue(Attribute, Form, Value);
+ }
+
+ /// AddObjectLabel - Add an non-Dwarf label attribute data and value.
+ ///
+ void AddObjectLabel(DIE *Die, unsigned Attribute, unsigned Form,
+ const std::string &Label) {
+ FoldingSetNodeID ID;
+ DIEObjectLabel::Profile(ID, Label);
+ void *Where;
+ DIEValue *Value = ValuesSet.FindNodeOrInsertPos(ID, Where);
+ if (!Value) {
+ Value = new DIEObjectLabel(Label);
+ ValuesSet.InsertNode(Value, Where);
+ Values.push_back(Value);
+ }
+
+ Die->AddValue(Attribute, Form, Value);
+ }
+
+ /// AddDelta - Add a label delta attribute data and value.
+ ///
+ void AddDelta(DIE *Die, unsigned Attribute, unsigned Form,
+ const DWLabel &Hi, const DWLabel &Lo) {
+ FoldingSetNodeID ID;
+ DIEDelta::Profile(ID, Hi, Lo);
+ void *Where;
+ DIEValue *Value = ValuesSet.FindNodeOrInsertPos(ID, Where);
+ if (!Value) {
+ Value = new DIEDelta(Hi, Lo);
+ ValuesSet.InsertNode(Value, Where);
+ Values.push_back(Value);
+ }
+
+ Die->AddValue(Attribute, Form, Value);
+ }
+
+ /// AddDIEntry - Add a DIE attribute data and value.
+ ///
+ void AddDIEntry(DIE *Die, unsigned Attribute, unsigned Form, DIE *Entry) {
+ Die->AddValue(Attribute, Form, NewDIEntry(Entry));
+ }
+
+ /// AddBlock - Add block data.
+ ///
+ void AddBlock(DIE *Die, unsigned Attribute, unsigned Form, DIEBlock *Block) {
+ Block->ComputeSize(*this);
+ FoldingSetNodeID ID;
+ Block->Profile(ID);
+ void *Where;
+ DIEValue *Value = ValuesSet.FindNodeOrInsertPos(ID, Where);
+ if (!Value) {
+ Value = Block;
+ ValuesSet.InsertNode(Value, Where);
+ Values.push_back(Value);
+ } else {
+ delete Block;
+ }
+
+ Die->AddValue(Attribute, Block->BestForm(), Value);
+ }
+
+private:
+
+ /// AddSourceLine - Add location information to specified debug information
+ /// entry.
+ void AddSourceLine(DIE *Die, CompileUnitDesc *File, unsigned Line) {
+ if (File && Line) {
+ CompileUnit *FileUnit = FindCompileUnit(File);
+ unsigned FileID = FileUnit->getID();
+ AddUInt(Die, DW_AT_decl_file, 0, FileID);
+ AddUInt(Die, DW_AT_decl_line, 0, Line);
+ }
+ }
+
+ /// AddAddress - Add an address attribute to a die based on the location
+ /// provided.
+ void AddAddress(DIE *Die, unsigned Attribute,
+ const MachineLocation &Location) {
+ unsigned Reg = RI->getDwarfRegNum(Location.getRegister());
+ DIEBlock *Block = new DIEBlock();
+
+ if (Location.isRegister()) {
+ if (Reg < 32) {
+ AddUInt(Block, 0, DW_FORM_data1, DW_OP_reg0 + Reg);
+ } else {
+ AddUInt(Block, 0, DW_FORM_data1, DW_OP_regx);
+ AddUInt(Block, 0, DW_FORM_udata, Reg);
+ }
+ } else {
+ if (Reg < 32) {
+ AddUInt(Block, 0, DW_FORM_data1, DW_OP_breg0 + Reg);
+ } else {
+ AddUInt(Block, 0, DW_FORM_data1, DW_OP_bregx);
+ AddUInt(Block, 0, DW_FORM_udata, Reg);
+ }
+ AddUInt(Block, 0, DW_FORM_sdata, Location.getOffset());
+ }
+
+ AddBlock(Die, Attribute, 0, Block);
+ }
+
+ /// AddBasicType - Add a new basic type attribute to the specified entity.
+ ///
+ void AddBasicType(DIE *Entity, CompileUnit *Unit,
+ const std::string &Name,
+ unsigned Encoding, unsigned Size) {
+ DIE *Die = ConstructBasicType(Unit, Name, Encoding, Size);
+ AddDIEntry(Entity, DW_AT_type, DW_FORM_ref4, Die);
+ }
+
+ /// ConstructBasicType - Construct a new basic type.
+ ///
+ DIE *ConstructBasicType(CompileUnit *Unit,
+ const std::string &Name,
+ unsigned Encoding, unsigned Size) {
+ DIE Buffer(DW_TAG_base_type);
+ AddUInt(&Buffer, DW_AT_byte_size, 0, Size);
+ AddUInt(&Buffer, DW_AT_encoding, DW_FORM_data1, Encoding);
+ if (!Name.empty()) AddString(&Buffer, DW_AT_name, DW_FORM_string, Name);
+ return Unit->AddDie(Buffer);
+ }
+
+ /// AddPointerType - Add a new pointer type attribute to the specified entity.
+ ///
+ void AddPointerType(DIE *Entity, CompileUnit *Unit, const std::string &Name) {
+ DIE *Die = ConstructPointerType(Unit, Name);
+ AddDIEntry(Entity, DW_AT_type, DW_FORM_ref4, Die);
+ }
+
+ /// ConstructPointerType - Construct a new pointer type.
+ ///
+ DIE *ConstructPointerType(CompileUnit *Unit, const std::string &Name) {
+ DIE Buffer(DW_TAG_pointer_type);
+ AddUInt(&Buffer, DW_AT_byte_size, 0, TAI->getAddressSize());
+ if (!Name.empty()) AddString(&Buffer, DW_AT_name, DW_FORM_string, Name);
+ return Unit->AddDie(Buffer);
+ }
+
+ /// AddType - Add a new type attribute to the specified entity.
+ ///
+ void AddType(DIE *Entity, TypeDesc *TyDesc, CompileUnit *Unit) {
+ if (!TyDesc) {
+ AddBasicType(Entity, Unit, "", DW_ATE_signed, sizeof(int32_t));
+ } else {
+ // Check for pre-existence.
+ DIEntry *&Slot = Unit->getDIEntrySlotFor(TyDesc);
+
+ // If it exists then use the existing value.
+ if (Slot) {
+ Entity->AddValue(DW_AT_type, DW_FORM_ref4, Slot);
+ return;
+ }
+
+ if (SubprogramDesc *SubprogramTy = dyn_cast<SubprogramDesc>(TyDesc)) {
+ // FIXME - Not sure why programs and variables are coming through here.
+ // Short cut for handling subprogram types (not really a TyDesc.)
+ AddPointerType(Entity, Unit, SubprogramTy->getName());
+ } else if (GlobalVariableDesc *GlobalTy =
+ dyn_cast<GlobalVariableDesc>(TyDesc)) {
+ // FIXME - Not sure why programs and variables are coming through here.
+ // Short cut for handling global variable types (not really a TyDesc.)
+ AddPointerType(Entity, Unit, GlobalTy->getName());
+ } else {
+ // Set up proxy.
+ Slot = NewDIEntry();
+
+ // Construct type.
+ DIE Buffer(DW_TAG_base_type);
+ ConstructType(Buffer, TyDesc, Unit);
+
+ // Add debug information entry to entity and unit.
+ DIE *Die = Unit->AddDie(Buffer);
+ SetDIEntry(Slot, Die);
+ Entity->AddValue(DW_AT_type, DW_FORM_ref4, Slot);
+ }
+ }
+ }
+
+ /// ConstructType - Adds all the required attributes to the type.
+ ///
+ void ConstructType(DIE &Buffer, TypeDesc *TyDesc, CompileUnit *Unit) {
+ // Get core information.
+ const std::string &Name = TyDesc->getName();
+ uint64_t Size = TyDesc->getSize() >> 3;
+
+ if (BasicTypeDesc *BasicTy = dyn_cast<BasicTypeDesc>(TyDesc)) {
+ // Fundamental types like int, float, bool
+ Buffer.setTag(DW_TAG_base_type);
+ AddUInt(&Buffer, DW_AT_encoding, DW_FORM_data1, BasicTy->getEncoding());
+ } else if (DerivedTypeDesc *DerivedTy = dyn_cast<DerivedTypeDesc>(TyDesc)) {
+ // Fetch tag.
+ unsigned Tag = DerivedTy->getTag();
+ // FIXME - Workaround for templates.
+ if (Tag == DW_TAG_inheritance) Tag = DW_TAG_reference_type;
+ // Pointers, typedefs et al.
+ Buffer.setTag(Tag);
+ // Map to main type, void will not have a type.
+ if (TypeDesc *FromTy = DerivedTy->getFromType())
+ AddType(&Buffer, FromTy, Unit);
+ } else if (CompositeTypeDesc *CompTy = dyn_cast<CompositeTypeDesc>(TyDesc)){
+ // Fetch tag.
+ unsigned Tag = CompTy->getTag();
+
+ // Set tag accordingly.
+ if (Tag == DW_TAG_vector_type)
+ Buffer.setTag(DW_TAG_array_type);
+ else
+ Buffer.setTag(Tag);
+
+ std::vector<DebugInfoDesc *> &Elements = CompTy->getElements();
+
+ switch (Tag) {
+ case DW_TAG_vector_type:
+ AddUInt(&Buffer, DW_AT_GNU_vector, DW_FORM_flag, 1);
+ // Fall thru
+ case DW_TAG_array_type: {
+ // Add element type.
+ if (TypeDesc *FromTy = CompTy->getFromType())
+ AddType(&Buffer, FromTy, Unit);
+
+ // Don't emit size attribute.
+ Size = 0;
+
+ // Construct an anonymous type for index type.
+ DIE *IndexTy = ConstructBasicType(Unit, "", DW_ATE_signed,
+ sizeof(int32_t));
+
+ // Add subranges to array type.
+ for(unsigned i = 0, N = Elements.size(); i < N; ++i) {
+ SubrangeDesc *SRD = cast<SubrangeDesc>(Elements[i]);
+ int64_t Lo = SRD->getLo();
+ int64_t Hi = SRD->getHi();
+ DIE *Subrange = new DIE(DW_TAG_subrange_type);
+
+ // If a range is available.
+ if (Lo != Hi) {
+ AddDIEntry(Subrange, DW_AT_type, DW_FORM_ref4, IndexTy);
+ // Only add low if non-zero.
+ if (Lo) AddSInt(Subrange, DW_AT_lower_bound, 0, Lo);
+ AddSInt(Subrange, DW_AT_upper_bound, 0, Hi);
+ }
+
+ Buffer.AddChild(Subrange);
+ }
+ break;
+ }
+ case DW_TAG_structure_type:
+ case DW_TAG_union_type: {
+ // Add elements to structure type.
+ for(unsigned i = 0, N = Elements.size(); i < N; ++i) {
+ DebugInfoDesc *Element = Elements[i];
+
+ if (DerivedTypeDesc *MemberDesc = dyn_cast<DerivedTypeDesc>(Element)){
+ // Add field or base class.
+
+ unsigned Tag = MemberDesc->getTag();
+
+ // Extract the basic information.
+ const std::string &Name = MemberDesc->getName();
+ uint64_t Size = MemberDesc->getSize();
+ uint64_t Align = MemberDesc->getAlign();
+ uint64_t Offset = MemberDesc->getOffset();
+
+ // Construct member debug information entry.
+ DIE *Member = new DIE(Tag);
+
+ // Add name if not "".
+ if (!Name.empty())
+ AddString(Member, DW_AT_name, DW_FORM_string, Name);
+ // Add location if available.
+ AddSourceLine(Member, MemberDesc->getFile(), MemberDesc->getLine());
+
+ // Most of the time the field info is the same as the members.
+ uint64_t FieldSize = Size;
+ uint64_t FieldAlign = Align;
+ uint64_t FieldOffset = Offset;
+
+ // Set the member type.
+ TypeDesc *FromTy = MemberDesc->getFromType();
+ AddType(Member, FromTy, Unit);
+
+ // Walk up typedefs until a real size is found.
+ while (FromTy) {
+ if (FromTy->getTag() != DW_TAG_typedef) {
+ FieldSize = FromTy->getSize();
+ FieldAlign = FromTy->getSize();
+ break;
+ }
+
+ FromTy = dyn_cast<DerivedTypeDesc>(FromTy)->getFromType();
+ }
+
+ // Unless we have a bit field.
+ if (Tag == DW_TAG_member && FieldSize != Size) {
+ // Construct the alignment mask.
+ uint64_t AlignMask = ~(FieldAlign - 1);
+ // Determine the high bit + 1 of the declared size.
+ uint64_t HiMark = (Offset + FieldSize) & AlignMask;
+ // Work backwards to determine the base offset of the field.
+ FieldOffset = HiMark - FieldSize;
+ // Now normalize offset to the field.
+ Offset -= FieldOffset;
+
+ // Maybe we need to work from the other end.
+ if (TD->isLittleEndian()) Offset = FieldSize - (Offset + Size);
+
+ // Add size and offset.
+ AddUInt(Member, DW_AT_byte_size, 0, FieldSize >> 3);
+ AddUInt(Member, DW_AT_bit_size, 0, Size);
+ AddUInt(Member, DW_AT_bit_offset, 0, Offset);
+ }
+
+ // Add computation for offset.
+ DIEBlock *Block = new DIEBlock();
+ AddUInt(Block, 0, DW_FORM_data1, DW_OP_plus_uconst);
+ AddUInt(Block, 0, DW_FORM_udata, FieldOffset >> 3);
+ AddBlock(Member, DW_AT_data_member_location, 0, Block);
+
+ // Add accessibility (public default unless is base class.
+ if (MemberDesc->isProtected()) {
+ AddUInt(Member, DW_AT_accessibility, 0, DW_ACCESS_protected);
+ } else if (MemberDesc->isPrivate()) {
+ AddUInt(Member, DW_AT_accessibility, 0, DW_ACCESS_private);
+ } else if (Tag == DW_TAG_inheritance) {
+ AddUInt(Member, DW_AT_accessibility, 0, DW_ACCESS_public);
+ }
+
+ Buffer.AddChild(Member);
+ } else if (GlobalVariableDesc *StaticDesc =
+ dyn_cast<GlobalVariableDesc>(Element)) {
+ // Add static member.
+
+ // Construct member debug information entry.
+ DIE *Static = new DIE(DW_TAG_variable);
+
+ // Add name and mangled name.
+ const std::string &Name = StaticDesc->getName();
+ const std::string &LinkageName = StaticDesc->getLinkageName();
+ AddString(Static, DW_AT_name, DW_FORM_string, Name);
+ if (!LinkageName.empty()) {
+ AddString(Static, DW_AT_MIPS_linkage_name, DW_FORM_string,
+ LinkageName);
+ }
+
+ // Add location.
+ AddSourceLine(Static, StaticDesc->getFile(), StaticDesc->getLine());
+
+ // Add type.
+ if (TypeDesc *StaticTy = StaticDesc->getType())
+ AddType(Static, StaticTy, Unit);
+
+ // Add flags.
+ if (!StaticDesc->isStatic())
+ AddUInt(Static, DW_AT_external, DW_FORM_flag, 1);
+ AddUInt(Static, DW_AT_declaration, DW_FORM_flag, 1);
+
+ Buffer.AddChild(Static);
+ } else if (SubprogramDesc *MethodDesc =
+ dyn_cast<SubprogramDesc>(Element)) {
+ // Add member function.
+
+ // Construct member debug information entry.
+ DIE *Method = new DIE(DW_TAG_subprogram);
+
+ // Add name and mangled name.
+ const std::string &Name = MethodDesc->getName();
+ const std::string &LinkageName = MethodDesc->getLinkageName();
+
+ AddString(Method, DW_AT_name, DW_FORM_string, Name);
+ bool IsCTor = TyDesc->getName() == Name;
+
+ if (!LinkageName.empty()) {
+ AddString(Method, DW_AT_MIPS_linkage_name, DW_FORM_string,
+ LinkageName);
+ }
+
+ // Add location.
+ AddSourceLine(Method, MethodDesc->getFile(), MethodDesc->getLine());
+
+ // Add type.
+ if (CompositeTypeDesc *MethodTy =
+ dyn_cast_or_null<CompositeTypeDesc>(MethodDesc->getType())) {
+ // Get argument information.
+ std::vector<DebugInfoDesc *> &Args = MethodTy->getElements();
+
+ // If not a ctor.
+ if (!IsCTor) {
+ // Add return type.
+ AddType(Method, dyn_cast<TypeDesc>(Args[0]), Unit);
+ }
+
+ // Add arguments.
+ for(unsigned i = 1, N = Args.size(); i < N; ++i) {
+ DIE *Arg = new DIE(DW_TAG_formal_parameter);
+ AddType(Arg, cast<TypeDesc>(Args[i]), Unit);
+ AddUInt(Arg, DW_AT_artificial, DW_FORM_flag, 1);
+ Method->AddChild(Arg);
+ }
+ }
+
+ // Add flags.
+ if (!MethodDesc->isStatic())
+ AddUInt(Method, DW_AT_external, DW_FORM_flag, 1);
+ AddUInt(Method, DW_AT_declaration, DW_FORM_flag, 1);
+
+ Buffer.AddChild(Method);
+ }
+ }
+ break;
+ }
+ case DW_TAG_enumeration_type: {
+ // Add enumerators to enumeration type.
+ for(unsigned i = 0, N = Elements.size(); i < N; ++i) {
+ EnumeratorDesc *ED = cast<EnumeratorDesc>(Elements[i]);
+ const std::string &Name = ED->getName();
+ int64_t Value = ED->getValue();
+ DIE *Enumerator = new DIE(DW_TAG_enumerator);
+ AddString(Enumerator, DW_AT_name, DW_FORM_string, Name);
+ AddSInt(Enumerator, DW_AT_const_value, DW_FORM_sdata, Value);
+ Buffer.AddChild(Enumerator);
+ }
+
+ break;
+ }
+ case DW_TAG_subroutine_type: {
+ // Add prototype flag.
+ AddUInt(&Buffer, DW_AT_prototyped, DW_FORM_flag, 1);
+ // Add return type.
+ AddType(&Buffer, dyn_cast<TypeDesc>(Elements[0]), Unit);
+
+ // Add arguments.
+ for(unsigned i = 1, N = Elements.size(); i < N; ++i) {
+ DIE *Arg = new DIE(DW_TAG_formal_parameter);
+ AddType(Arg, cast<TypeDesc>(Elements[i]), Unit);
+ Buffer.AddChild(Arg);
+ }
+
+ break;
+ }
+ default: break;
+ }
+ }
+
+ // Add size if non-zero (derived types don't have a size.)
+ if (Size) AddUInt(&Buffer, DW_AT_byte_size, 0, Size);
+ // Add name if not anonymous or intermediate type.
+ if (!Name.empty()) AddString(&Buffer, DW_AT_name, DW_FORM_string, Name);
+ // Add source line info if available.
+ AddSourceLine(&Buffer, TyDesc->getFile(), TyDesc->getLine());
+ }
+
+ /// NewCompileUnit - Create new compile unit and it's debug information entry.
+ ///
+ CompileUnit *NewCompileUnit(CompileUnitDesc *UnitDesc, unsigned ID) {
+ // Construct debug information entry.
+ DIE *Die = new DIE(DW_TAG_compile_unit);
+ if (TAI->isAbsoluteDebugSectionOffsets())
+ AddLabel(Die, DW_AT_stmt_list, DW_FORM_data4, DWLabel("section_line", 0));
+ else
+ AddDelta(Die, DW_AT_stmt_list, DW_FORM_data4, DWLabel("section_line", 0),
+ DWLabel("section_line", 0));
+ AddString(Die, DW_AT_producer, DW_FORM_string, UnitDesc->getProducer());
+ AddUInt (Die, DW_AT_language, DW_FORM_data1, UnitDesc->getLanguage());
+ AddString(Die, DW_AT_name, DW_FORM_string, UnitDesc->getFileName());
+ AddString(Die, DW_AT_comp_dir, DW_FORM_string, UnitDesc->getDirectory());
+
+ // Construct compile unit.
+ CompileUnit *Unit = new CompileUnit(UnitDesc, ID, Die);
+
+ // Add Unit to compile unit map.
+ DescToUnitMap[UnitDesc] = Unit;
+
+ return Unit;
+ }
+
+ /// GetBaseCompileUnit - Get the main compile unit.
+ ///
+ CompileUnit *GetBaseCompileUnit() const {
+ CompileUnit *Unit = CompileUnits[0];
+ assert(Unit && "Missing compile unit.");
+ return Unit;
+ }
+
+ /// FindCompileUnit - Get the compile unit for the given descriptor.
+ ///
+ CompileUnit *FindCompileUnit(CompileUnitDesc *UnitDesc) {
+ CompileUnit *Unit = DescToUnitMap[UnitDesc];
+ assert(Unit && "Missing compile unit.");
+ return Unit;
+ }
+
+ /// NewGlobalVariable - Add a new global variable DIE.
+ ///
+ DIE *NewGlobalVariable(GlobalVariableDesc *GVD) {
+ // Get the compile unit context.
+ CompileUnitDesc *UnitDesc =
+ static_cast<CompileUnitDesc *>(GVD->getContext());
+ CompileUnit *Unit = GetBaseCompileUnit();
+
+ // Check for pre-existence.
+ DIE *&Slot = Unit->getDieMapSlotFor(GVD);
+ if (Slot) return Slot;
+
+ // Get the global variable itself.
+ GlobalVariable *GV = GVD->getGlobalVariable();
+
+ const std::string &Name = GVD->getName();
+ const std::string &FullName = GVD->getFullName();
+ const std::string &LinkageName = GVD->getLinkageName();
+ // Create the global's variable DIE.
+ DIE *VariableDie = new DIE(DW_TAG_variable);
+ AddString(VariableDie, DW_AT_name, DW_FORM_string, Name);
+ if (!LinkageName.empty()) {
+ AddString(VariableDie, DW_AT_MIPS_linkage_name, DW_FORM_string,
+ LinkageName);
+ }
+ AddType(VariableDie, GVD->getType(), Unit);
+ if (!GVD->isStatic())
+ AddUInt(VariableDie, DW_AT_external, DW_FORM_flag, 1);
+
+ // Add source line info if available.
+ AddSourceLine(VariableDie, UnitDesc, GVD->getLine());
+
+ // Add address.
+ DIEBlock *Block = new DIEBlock();
+ AddUInt(Block, 0, DW_FORM_data1, DW_OP_addr);
+ AddObjectLabel(Block, 0, DW_FORM_udata, Asm->getGlobalLinkName(GV));
+ AddBlock(VariableDie, DW_AT_location, 0, Block);
+
+ // Add to map.
+ Slot = VariableDie;
+
+ // Add to context owner.
+ Unit->getDie()->AddChild(VariableDie);
+
+ // Expose as global.
+ // FIXME - need to check external flag.
+ Unit->AddGlobal(FullName, VariableDie);
+
+ return VariableDie;
+ }
+
+ /// NewSubprogram - Add a new subprogram DIE.
+ ///
+ DIE *NewSubprogram(SubprogramDesc *SPD) {
+ // Get the compile unit context.
+ CompileUnitDesc *UnitDesc =
+ static_cast<CompileUnitDesc *>(SPD->getContext());
+ CompileUnit *Unit = GetBaseCompileUnit();
+
+ // Check for pre-existence.
+ DIE *&Slot = Unit->getDieMapSlotFor(SPD);
+ if (Slot) return Slot;
+
+ // Gather the details (simplify add attribute code.)
+ const std::string &Name = SPD->getName();
+ const std::string &FullName = SPD->getFullName();
+ const std::string &LinkageName = SPD->getLinkageName();
+
+ DIE *SubprogramDie = new DIE(DW_TAG_subprogram);
+ AddString(SubprogramDie, DW_AT_name, DW_FORM_string, Name);
+ if (!LinkageName.empty()) {
+ AddString(SubprogramDie, DW_AT_MIPS_linkage_name, DW_FORM_string,
+ LinkageName);
+ }
+ if (SPD->getType()) AddType(SubprogramDie, SPD->getType(), Unit);
+ if (!SPD->isStatic())
+ AddUInt(SubprogramDie, DW_AT_external, DW_FORM_flag, 1);
+ AddUInt(SubprogramDie, DW_AT_prototyped, DW_FORM_flag, 1);
+
+ // Add source line info if available.
+ AddSourceLine(SubprogramDie, UnitDesc, SPD->getLine());
+
+ // Add to map.
+ Slot = SubprogramDie;
+
+ // Add to context owner.
+ Unit->getDie()->AddChild(SubprogramDie);
+
+ // Expose as global.
+ Unit->AddGlobal(FullName, SubprogramDie);
+
+ return SubprogramDie;
+ }
+
+ /// NewScopeVariable - Create a new scope variable.
+ ///
+ DIE *NewScopeVariable(DebugVariable *DV, CompileUnit *Unit) {
+ // Get the descriptor.
+ VariableDesc *VD = DV->getDesc();
+
+ // Translate tag to proper Dwarf tag. The result variable is dropped for
+ // now.
+ unsigned Tag;
+ switch (VD->getTag()) {
+ case DW_TAG_return_variable: return NULL;
+ case DW_TAG_arg_variable: Tag = DW_TAG_formal_parameter; break;
+ case DW_TAG_auto_variable: // fall thru
+ default: Tag = DW_TAG_variable; break;
+ }
+
+ // Define variable debug information entry.
+ DIE *VariableDie = new DIE(Tag);
+ AddString(VariableDie, DW_AT_name, DW_FORM_string, VD->getName());
+
+ // Add source line info if available.
+ AddSourceLine(VariableDie, VD->getFile(), VD->getLine());
+
+ // Add variable type.
+ AddType(VariableDie, VD->getType(), Unit);
+
+ // Add variable address.
+ MachineLocation Location;
+ RI->getLocation(*MF, DV->getFrameIndex(), Location);
+ AddAddress(VariableDie, DW_AT_location, Location);
+
+ return VariableDie;
+ }
+
+ /// ConstructScope - Construct the components of a scope.
+ ///
+ void ConstructScope(DebugScope *ParentScope,
+ unsigned ParentStartID, unsigned ParentEndID,
+ DIE *ParentDie, CompileUnit *Unit) {
+ // Add variables to scope.
+ std::vector<DebugVariable *> &Variables = ParentScope->getVariables();
+ for (unsigned i = 0, N = Variables.size(); i < N; ++i) {
+ DIE *VariableDie = NewScopeVariable(Variables[i], Unit);
+ if (VariableDie) ParentDie->AddChild(VariableDie);
+ }
+
+ // Add nested scopes.
+ std::vector<DebugScope *> &Scopes = ParentScope->getScopes();
+ for (unsigned j = 0, M = Scopes.size(); j < M; ++j) {
+ // Define the Scope debug information entry.
+ DebugScope *Scope = Scopes[j];
+ // FIXME - Ignore inlined functions for the time being.
+ if (!Scope->getParent()) continue;
+
+ unsigned StartID = MMI->MappedLabel(Scope->getStartLabelID());
+ unsigned EndID = MMI->MappedLabel(Scope->getEndLabelID());
+
+ // Ignore empty scopes.
+ if (StartID == EndID && StartID != 0) continue;
+ if (Scope->getScopes().empty() && Scope->getVariables().empty()) continue;
+
+ if (StartID == ParentStartID && EndID == ParentEndID) {
+ // Just add stuff to the parent scope.
+ ConstructScope(Scope, ParentStartID, ParentEndID, ParentDie, Unit);
+ } else {
+ DIE *ScopeDie = new DIE(DW_TAG_lexical_block);
+
+ // Add the scope bounds.
+ if (StartID) {
+ AddLabel(ScopeDie, DW_AT_low_pc, DW_FORM_addr,
+ DWLabel("label", StartID));
+ } else {
+ AddLabel(ScopeDie, DW_AT_low_pc, DW_FORM_addr,
+ DWLabel("func_begin", SubprogramCount));
+ }
+ if (EndID) {
+ AddLabel(ScopeDie, DW_AT_high_pc, DW_FORM_addr,
+ DWLabel("label", EndID));
+ } else {
+ AddLabel(ScopeDie, DW_AT_high_pc, DW_FORM_addr,
+ DWLabel("func_end", SubprogramCount));
+ }
+
+ // Add the scope contents.
+ ConstructScope(Scope, StartID, EndID, ScopeDie, Unit);
+ ParentDie->AddChild(ScopeDie);
+ }
+ }
+ }
+
+ /// ConstructRootScope - Construct the scope for the subprogram.
+ ///
+ void ConstructRootScope(DebugScope *RootScope) {
+ // Exit if there is no root scope.
+ if (!RootScope) return;
+
+ // Get the subprogram debug information entry.
+ SubprogramDesc *SPD = cast<SubprogramDesc>(RootScope->getDesc());
+
+ // Get the compile unit context.
+ CompileUnit *Unit = GetBaseCompileUnit();
+
+ // Get the subprogram die.
+ DIE *SPDie = Unit->getDieMapSlotFor(SPD);
+ assert(SPDie && "Missing subprogram descriptor");
+
+ // Add the function bounds.
+ AddLabel(SPDie, DW_AT_low_pc, DW_FORM_addr,
+ DWLabel("func_begin", SubprogramCount));
+ AddLabel(SPDie, DW_AT_high_pc, DW_FORM_addr,
+ DWLabel("func_end", SubprogramCount));
+ MachineLocation Location(RI->getFrameRegister(*MF));
+ AddAddress(SPDie, DW_AT_frame_base, Location);
+
+ ConstructScope(RootScope, 0, 0, SPDie, Unit);
+ }
+
+ /// EmitInitial - Emit initial Dwarf declarations. This is necessary for cc
+ /// tools to recognize the object file contains Dwarf information.
+ void EmitInitial() {
+ // Check to see if we already emitted intial headers.
+ if (didInitial) return;
+ didInitial = true;
+
+ // Dwarf sections base addresses.
+ if (TAI->doesDwarfRequireFrameSection()) {
+ Asm->SwitchToDataSection(TAI->getDwarfFrameSection());
+ EmitLabel("section_debug_frame", 0);
+ }
+ Asm->SwitchToDataSection(TAI->getDwarfInfoSection());
+ EmitLabel("section_info", 0);
+ Asm->SwitchToDataSection(TAI->getDwarfAbbrevSection());
+ EmitLabel("section_abbrev", 0);
+ Asm->SwitchToDataSection(TAI->getDwarfARangesSection());
+ EmitLabel("section_aranges", 0);
+ Asm->SwitchToDataSection(TAI->getDwarfMacInfoSection());
+ EmitLabel("section_macinfo", 0);
+ Asm->SwitchToDataSection(TAI->getDwarfLineSection());
+ EmitLabel("section_line", 0);
+ Asm->SwitchToDataSection(TAI->getDwarfLocSection());
+ EmitLabel("section_loc", 0);
+ Asm->SwitchToDataSection(TAI->getDwarfPubNamesSection());
+ EmitLabel("section_pubnames", 0);
+ Asm->SwitchToDataSection(TAI->getDwarfStrSection());
+ EmitLabel("section_str", 0);
+ Asm->SwitchToDataSection(TAI->getDwarfRangesSection());
+ EmitLabel("section_ranges", 0);
+
+ Asm->SwitchToTextSection(TAI->getTextSection());
+ EmitLabel("text_begin", 0);
+ Asm->SwitchToDataSection(TAI->getDataSection());
+ EmitLabel("data_begin", 0);
+ }
+
+ /// EmitDIE - Recusively Emits a debug information entry.
+ ///
+ void EmitDIE(DIE *Die) {
+ // Get the abbreviation for this DIE.
+ unsigned AbbrevNumber = Die->getAbbrevNumber();
+ const DIEAbbrev *Abbrev = Abbreviations[AbbrevNumber - 1];
+
+ Asm->EOL();
+
+ // Emit the code (index) for the abbreviation.
+ Asm->EmitULEB128Bytes(AbbrevNumber);
+ Asm->EOL(std::string("Abbrev [" +
+ utostr(AbbrevNumber) +
+ "] 0x" + utohexstr(Die->getOffset()) +
+ ":0x" + utohexstr(Die->getSize()) + " " +
+ TagString(Abbrev->getTag())));
+
+ std::vector<DIEValue *> &Values = Die->getValues();
+ const std::vector<DIEAbbrevData> &AbbrevData = Abbrev->getData();
+
+ // Emit the DIE attribute values.
+ for (unsigned i = 0, N = Values.size(); i < N; ++i) {
+ unsigned Attr = AbbrevData[i].getAttribute();
+ unsigned Form = AbbrevData[i].getForm();
+ assert(Form && "Too many attributes for DIE (check abbreviation)");
+
+ switch (Attr) {
+ case DW_AT_sibling: {
+ Asm->EmitInt32(Die->SiblingOffset());
+ break;
+ }
+ default: {
+ // Emit an attribute using the defined form.
+ Values[i]->EmitValue(*this, Form);
+ break;
+ }
+ }
+
+ Asm->EOL(AttributeString(Attr));
+ }
+
+ // Emit the DIE children if any.
+ if (Abbrev->getChildrenFlag() == DW_CHILDREN_yes) {
+ const std::vector<DIE *> &Children = Die->getChildren();
+
+ for (unsigned j = 0, M = Children.size(); j < M; ++j) {
+ EmitDIE(Children[j]);
+ }
+
+ Asm->EmitInt8(0); Asm->EOL("End Of Children Mark");
+ }
+ }
+
+ /// SizeAndOffsetDie - Compute the size and offset of a DIE.
+ ///
+ unsigned SizeAndOffsetDie(DIE *Die, unsigned Offset, bool Last) {
+ // Get the children.
+ const std::vector<DIE *> &Children = Die->getChildren();
+
+ // If not last sibling and has children then add sibling offset attribute.
+ if (!Last && !Children.empty()) Die->AddSiblingOffset();
+
+ // Record the abbreviation.
+ AssignAbbrevNumber(Die->getAbbrev());
+
+ // Get the abbreviation for this DIE.
+ unsigned AbbrevNumber = Die->getAbbrevNumber();
+ const DIEAbbrev *Abbrev = Abbreviations[AbbrevNumber - 1];
+
+ // Set DIE offset
+ Die->setOffset(Offset);
+
+ // Start the size with the size of abbreviation code.
+ Offset += Asm->SizeULEB128(AbbrevNumber);
+
+ const std::vector<DIEValue *> &Values = Die->getValues();
+ const std::vector<DIEAbbrevData> &AbbrevData = Abbrev->getData();
+
+ // Size the DIE attribute values.
+ for (unsigned i = 0, N = Values.size(); i < N; ++i) {
+ // Size attribute value.
+ Offset += Values[i]->SizeOf(*this, AbbrevData[i].getForm());
+ }
+
+ // Size the DIE children if any.
+ if (!Children.empty()) {
+ assert(Abbrev->getChildrenFlag() == DW_CHILDREN_yes &&
+ "Children flag not set");
+
+ for (unsigned j = 0, M = Children.size(); j < M; ++j) {
+ Offset = SizeAndOffsetDie(Children[j], Offset, (j + 1) == M);
+ }
+
+ // End of children marker.
+ Offset += sizeof(int8_t);
+ }
+
+ Die->setSize(Offset - Die->getOffset());
+ return Offset;
+ }
+
+ /// SizeAndOffsets - Compute the size and offset of all the DIEs.
+ ///
+ void SizeAndOffsets() {
+ // Process base compile unit.
+ CompileUnit *Unit = GetBaseCompileUnit();
+ // Compute size of compile unit header
+ unsigned Offset = sizeof(int32_t) + // Length of Compilation Unit Info
+ sizeof(int16_t) + // DWARF version number
+ sizeof(int32_t) + // Offset Into Abbrev. Section
+ sizeof(int8_t); // Pointer Size (in bytes)
+ SizeAndOffsetDie(Unit->getDie(), Offset, true);
+ }
+
+ /// EmitDebugInfo - Emit the debug info section.
+ ///
+ void EmitDebugInfo() {
+ // Start debug info section.
+ Asm->SwitchToDataSection(TAI->getDwarfInfoSection());
+
+ CompileUnit *Unit = GetBaseCompileUnit();
+ DIE *Die = Unit->getDie();
+ // Emit the compile units header.
+ EmitLabel("info_begin", Unit->getID());
+ // Emit size of content not including length itself
+ unsigned ContentSize = Die->getSize() +
+ sizeof(int16_t) + // DWARF version number
+ sizeof(int32_t) + // Offset Into Abbrev. Section
+ sizeof(int8_t) + // Pointer Size (in bytes)
+ sizeof(int32_t); // FIXME - extra pad for gdb bug.
+
+ Asm->EmitInt32(ContentSize); Asm->EOL("Length of Compilation Unit Info");
+ Asm->EmitInt16(DWARF_VERSION); Asm->EOL("DWARF version number");
+ EmitSectionOffset("abbrev_begin", "section_abbrev", 0, 0, true, false);
+ Asm->EOL("Offset Into Abbrev. Section");
+ Asm->EmitInt8(TAI->getAddressSize()); Asm->EOL("Address Size (in bytes)");
+
+ EmitDIE(Die);
+ // FIXME - extra padding for gdb bug.
+ Asm->EmitInt8(0); Asm->EOL("Extra Pad For GDB");
+ Asm->EmitInt8(0); Asm->EOL("Extra Pad For GDB");
+ Asm->EmitInt8(0); Asm->EOL("Extra Pad For GDB");
+ Asm->EmitInt8(0); Asm->EOL("Extra Pad For GDB");
+ EmitLabel("info_end", Unit->getID());
+
+ Asm->EOL();
+ }
+
+ /// EmitAbbreviations - Emit the abbreviation section.
+ ///
+ void EmitAbbreviations() const {
+ // Check to see if it is worth the effort.
+ if (!Abbreviations.empty()) {
+ // Start the debug abbrev section.
+ Asm->SwitchToDataSection(TAI->getDwarfAbbrevSection());
+
+ EmitLabel("abbrev_begin", 0);
+
+ // For each abbrevation.
+ for (unsigned i = 0, N = Abbreviations.size(); i < N; ++i) {
+ // Get abbreviation data
+ const DIEAbbrev *Abbrev = Abbreviations[i];
+
+ // Emit the abbrevations code (base 1 index.)
+ Asm->EmitULEB128Bytes(Abbrev->getNumber());
+ Asm->EOL("Abbreviation Code");
+
+ // Emit the abbreviations data.
+ Abbrev->Emit(*this);
+
+ Asm->EOL();
+ }
+
+ // Mark end of abbreviations.
+ Asm->EmitULEB128Bytes(0); Asm->EOL("EOM(3)");
+
+ EmitLabel("abbrev_end", 0);
+
+ Asm->EOL();
+ }
+ }
+
+ /// EmitDebugLines - Emit source line information.
+ ///
+ void EmitDebugLines() {
+ // Minimum line delta, thus ranging from -10..(255-10).
+ const int MinLineDelta = -(DW_LNS_fixed_advance_pc + 1);
+ // Maximum line delta, thus ranging from -10..(255-10).
+ const int MaxLineDelta = 255 + MinLineDelta;
+
+ // Start the dwarf line section.
+ Asm->SwitchToDataSection(TAI->getDwarfLineSection());
+
+ // Construct the section header.
+
+ EmitDifference("line_end", 0, "line_begin", 0, true);
+ Asm->EOL("Length of Source Line Info");
+ EmitLabel("line_begin", 0);
+
+ Asm->EmitInt16(DWARF_VERSION); Asm->EOL("DWARF version number");
+
+ EmitDifference("line_prolog_end", 0, "line_prolog_begin", 0, true);
+ Asm->EOL("Prolog Length");
+ EmitLabel("line_prolog_begin", 0);
+
+ Asm->EmitInt8(1); Asm->EOL("Minimum Instruction Length");
+
+ Asm->EmitInt8(1); Asm->EOL("Default is_stmt_start flag");
+
+ Asm->EmitInt8(MinLineDelta); Asm->EOL("Line Base Value (Special Opcodes)");
+
+ Asm->EmitInt8(MaxLineDelta); Asm->EOL("Line Range Value (Special Opcodes)");
+
+ Asm->EmitInt8(-MinLineDelta); Asm->EOL("Special Opcode Base");
+
+ // Line number standard opcode encodings argument count
+ Asm->EmitInt8(0); Asm->EOL("DW_LNS_copy arg count");
+ Asm->EmitInt8(1); Asm->EOL("DW_LNS_advance_pc arg count");
+ Asm->EmitInt8(1); Asm->EOL("DW_LNS_advance_line arg count");
+ Asm->EmitInt8(1); Asm->EOL("DW_LNS_set_file arg count");
+ Asm->EmitInt8(1); Asm->EOL("DW_LNS_set_column arg count");
+ Asm->EmitInt8(0); Asm->EOL("DW_LNS_negate_stmt arg count");
+ Asm->EmitInt8(0); Asm->EOL("DW_LNS_set_basic_block arg count");
+ Asm->EmitInt8(0); Asm->EOL("DW_LNS_const_add_pc arg count");
+ Asm->EmitInt8(1); Asm->EOL("DW_LNS_fixed_advance_pc arg count");
+
+ const UniqueVector<std::string> &Directories = MMI->getDirectories();
+ const UniqueVector<SourceFileInfo>
+ &SourceFiles = MMI->getSourceFiles();
+
+ // Emit directories.
+ for (unsigned DirectoryID = 1, NDID = Directories.size();
+ DirectoryID <= NDID; ++DirectoryID) {
+ Asm->EmitString(Directories[DirectoryID]); Asm->EOL("Directory");
+ }
+ Asm->EmitInt8(0); Asm->EOL("End of directories");
+
+ // Emit files.
+ for (unsigned SourceID = 1, NSID = SourceFiles.size();
+ SourceID <= NSID; ++SourceID) {
+ const SourceFileInfo &SourceFile = SourceFiles[SourceID];
+ Asm->EmitString(SourceFile.getName());
+ Asm->EOL("Source");
+ Asm->EmitULEB128Bytes(SourceFile.getDirectoryID());
+ Asm->EOL("Directory #");
+ Asm->EmitULEB128Bytes(0);
+ Asm->EOL("Mod date");
+ Asm->EmitULEB128Bytes(0);
+ Asm->EOL("File size");
+ }
+ Asm->EmitInt8(0); Asm->EOL("End of files");
+
+ EmitLabel("line_prolog_end", 0);
+
+ // A sequence for each text section.
+ for (unsigned j = 0, M = SectionSourceLines.size(); j < M; ++j) {
+ // Isolate current sections line info.
+ const std::vector<SourceLineInfo> &LineInfos = SectionSourceLines[j];
+
+ Asm->EOL(std::string("Section ") + SectionMap[j + 1]);
+
+ // Dwarf assumes we start with first line of first source file.
+ unsigned Source = 1;
+ unsigned Line = 1;
+
+ // Construct rows of the address, source, line, column matrix.
+ for (unsigned i = 0, N = LineInfos.size(); i < N; ++i) {
+ const SourceLineInfo &LineInfo = LineInfos[i];
+ unsigned LabelID = MMI->MappedLabel(LineInfo.getLabelID());
+ if (!LabelID) continue;
+
+ unsigned SourceID = LineInfo.getSourceID();
+ const SourceFileInfo &SourceFile = SourceFiles[SourceID];
+ unsigned DirectoryID = SourceFile.getDirectoryID();
+ Asm->EOL(Directories[DirectoryID]
+ + SourceFile.getName()
+ + ":"
+ + utostr_32(LineInfo.getLine()));
+
+ // Define the line address.
+ Asm->EmitInt8(0); Asm->EOL("Extended Op");
+ Asm->EmitInt8(TAI->getAddressSize() + 1); Asm->EOL("Op size");
+ Asm->EmitInt8(DW_LNE_set_address); Asm->EOL("DW_LNE_set_address");
+ EmitReference("label", LabelID); Asm->EOL("Location label");
+
+ // If change of source, then switch to the new source.
+ if (Source != LineInfo.getSourceID()) {
+ Source = LineInfo.getSourceID();
+ Asm->EmitInt8(DW_LNS_set_file); Asm->EOL("DW_LNS_set_file");
+ Asm->EmitULEB128Bytes(Source); Asm->EOL("New Source");
+ }
+
+ // If change of line.
+ if (Line != LineInfo.getLine()) {
+ // Determine offset.
+ int Offset = LineInfo.getLine() - Line;
+ int Delta = Offset - MinLineDelta;
+
+ // Update line.
+ Line = LineInfo.getLine();
+
+ // If delta is small enough and in range...
+ if (Delta >= 0 && Delta < (MaxLineDelta - 1)) {
+ // ... then use fast opcode.
+ Asm->EmitInt8(Delta - MinLineDelta); Asm->EOL("Line Delta");
+ } else {
+ // ... otherwise use long hand.
+ Asm->EmitInt8(DW_LNS_advance_line); Asm->EOL("DW_LNS_advance_line");
+ Asm->EmitSLEB128Bytes(Offset); Asm->EOL("Line Offset");
+ Asm->EmitInt8(DW_LNS_copy); Asm->EOL("DW_LNS_copy");
+ }
+ } else {
+ // Copy the previous row (different address or source)
+ Asm->EmitInt8(DW_LNS_copy); Asm->EOL("DW_LNS_copy");
+ }
+ }
+
+ // Define last address of section.
+ Asm->EmitInt8(0); Asm->EOL("Extended Op");
+ Asm->EmitInt8(TAI->getAddressSize() + 1); Asm->EOL("Op size");
+ Asm->EmitInt8(DW_LNE_set_address); Asm->EOL("DW_LNE_set_address");
+ EmitReference("section_end", j + 1); Asm->EOL("Section end label");
+
+ // Mark end of matrix.
+ Asm->EmitInt8(0); Asm->EOL("DW_LNE_end_sequence");
+ Asm->EmitULEB128Bytes(1); Asm->EOL();
+ Asm->EmitInt8(1); Asm->EOL();
+ }
+
+ EmitLabel("line_end", 0);
+
+ Asm->EOL();
+ }
+
+ /// EmitCommonDebugFrame - Emit common frame info into a debug frame section.
+ ///
+ void EmitCommonDebugFrame() {
+ if (!TAI->doesDwarfRequireFrameSection())
+ return;
+
+ int stackGrowth =
+ Asm->TM.getFrameInfo()->getStackGrowthDirection() ==
+ TargetFrameInfo::StackGrowsUp ?
+ TAI->getAddressSize() : -TAI->getAddressSize();
+
+ // Start the dwarf frame section.
+ Asm->SwitchToDataSection(TAI->getDwarfFrameSection());
+
+ EmitLabel("debug_frame_common", 0);
+ EmitDifference("debug_frame_common_end", 0,
+ "debug_frame_common_begin", 0, true);
+ Asm->EOL("Length of Common Information Entry");
+
+ EmitLabel("debug_frame_common_begin", 0);
+ Asm->EmitInt32((int)DW_CIE_ID);
+ Asm->EOL("CIE Identifier Tag");
+ Asm->EmitInt8(DW_CIE_VERSION);
+ Asm->EOL("CIE Version");
+ Asm->EmitString("");
+ Asm->EOL("CIE Augmentation");
+ Asm->EmitULEB128Bytes(1);
+ Asm->EOL("CIE Code Alignment Factor");
+ Asm->EmitSLEB128Bytes(stackGrowth);
+ Asm->EOL("CIE Data Alignment Factor");
+ Asm->EmitInt8(RI->getDwarfRegNum(RI->getRARegister()));
+ Asm->EOL("CIE RA Column");
+
+ std::vector<MachineMove> Moves;
+ RI->getInitialFrameState(Moves);
+
+ EmitFrameMoves(NULL, 0, Moves);
+
+ Asm->EmitAlignment(2);
+ EmitLabel("debug_frame_common_end", 0);
+
+ Asm->EOL();
+ }
+
+ /// EmitFunctionDebugFrame - Emit per function frame info into a debug frame
+ /// section.
+ void EmitFunctionDebugFrame(const FunctionDebugFrameInfo &DebugFrameInfo) {
+ if (!TAI->doesDwarfRequireFrameSection())
+ return;
+
+ // Start the dwarf frame section.
+ Asm->SwitchToDataSection(TAI->getDwarfFrameSection());
+
+ EmitDifference("debug_frame_end", DebugFrameInfo.Number,
+ "debug_frame_begin", DebugFrameInfo.Number, true);
+ Asm->EOL("Length of Frame Information Entry");
+
+ EmitLabel("debug_frame_begin", DebugFrameInfo.Number);
+
+ EmitSectionOffset("debug_frame_common", "section_debug_frame",
+ 0, 0, true, false);
+ Asm->EOL("FDE CIE offset");
+
+ EmitReference("func_begin", DebugFrameInfo.Number);
+ Asm->EOL("FDE initial location");
+ EmitDifference("func_end", DebugFrameInfo.Number,
+ "func_begin", DebugFrameInfo.Number);
+ Asm->EOL("FDE address range");
+
+ EmitFrameMoves("func_begin", DebugFrameInfo.Number, DebugFrameInfo.Moves);
+
+ Asm->EmitAlignment(2);
+ EmitLabel("debug_frame_end", DebugFrameInfo.Number);
+
+ Asm->EOL();
+ }
+
+ /// EmitDebugPubNames - Emit visible names into a debug pubnames section.
+ ///
+ void EmitDebugPubNames() {
+ // Start the dwarf pubnames section.
+ Asm->SwitchToDataSection(TAI->getDwarfPubNamesSection());
+
+ CompileUnit *Unit = GetBaseCompileUnit();
+
+ EmitDifference("pubnames_end", Unit->getID(),
+ "pubnames_begin", Unit->getID(), true);
+ Asm->EOL("Length of Public Names Info");
+
+ EmitLabel("pubnames_begin", Unit->getID());
+
+ Asm->EmitInt16(DWARF_VERSION); Asm->EOL("DWARF Version");
+
+ EmitSectionOffset("info_begin", "section_info",
+ Unit->getID(), 0, true, false);
+ Asm->EOL("Offset of Compilation Unit Info");
+
+ EmitDifference("info_end", Unit->getID(), "info_begin", Unit->getID(),true);
+ Asm->EOL("Compilation Unit Length");
+
+ std::map<std::string, DIE *> &Globals = Unit->getGlobals();
+
+ for (std::map<std::string, DIE *>::iterator GI = Globals.begin(),
+ GE = Globals.end();
+ GI != GE; ++GI) {
+ const std::string &Name = GI->first;
+ DIE * Entity = GI->second;
+
+ Asm->EmitInt32(Entity->getOffset()); Asm->EOL("DIE offset");
+ Asm->EmitString(Name); Asm->EOL("External Name");
+ }
+
+ Asm->EmitInt32(0); Asm->EOL("End Mark");
+ EmitLabel("pubnames_end", Unit->getID());
+
+ Asm->EOL();
+ }
+
+ /// EmitDebugStr - Emit visible names into a debug str section.
+ ///
+ void EmitDebugStr() {
+ // Check to see if it is worth the effort.
+ if (!StringPool.empty()) {
+ // Start the dwarf str section.
+ Asm->SwitchToDataSection(TAI->getDwarfStrSection());
+
+ // For each of strings in the string pool.
+ for (unsigned StringID = 1, N = StringPool.size();
+ StringID <= N; ++StringID) {
+ // Emit a label for reference from debug information entries.
+ EmitLabel("string", StringID);
+ // Emit the string itself.
+ const std::string &String = StringPool[StringID];
+ Asm->EmitString(String); Asm->EOL();
+ }
+
+ Asm->EOL();
+ }
+ }
+
+ /// EmitDebugLoc - Emit visible names into a debug loc section.
+ ///
+ void EmitDebugLoc() {
+ // Start the dwarf loc section.
+ Asm->SwitchToDataSection(TAI->getDwarfLocSection());
+
+ Asm->EOL();
+ }
+
+ /// EmitDebugARanges - Emit visible names into a debug aranges section.
+ ///
+ void EmitDebugARanges() {
+ // Start the dwarf aranges section.
+ Asm->SwitchToDataSection(TAI->getDwarfARangesSection());
+
+ // FIXME - Mock up
+ #if 0
+ CompileUnit *Unit = GetBaseCompileUnit();
+
+ // Don't include size of length
+ Asm->EmitInt32(0x1c); Asm->EOL("Length of Address Ranges Info");
+
+ Asm->EmitInt16(DWARF_VERSION); Asm->EOL("Dwarf Version");
+
+ EmitReference("info_begin", Unit->getID());
+ Asm->EOL("Offset of Compilation Unit Info");
+
+ Asm->EmitInt8(TAI->getAddressSize()); Asm->EOL("Size of Address");
+
+ Asm->EmitInt8(0); Asm->EOL("Size of Segment Descriptor");
+
+ Asm->EmitInt16(0); Asm->EOL("Pad (1)");
+ Asm->EmitInt16(0); Asm->EOL("Pad (2)");
+
+ // Range 1
+ EmitReference("text_begin", 0); Asm->EOL("Address");
+ EmitDifference("text_end", 0, "text_begin", 0, true); Asm->EOL("Length");
+
+ Asm->EmitInt32(0); Asm->EOL("EOM (1)");
+ Asm->EmitInt32(0); Asm->EOL("EOM (2)");
+
+ Asm->EOL();
+ #endif
+ }
+
+ /// EmitDebugRanges - Emit visible names into a debug ranges section.
+ ///
+ void EmitDebugRanges() {
+ // Start the dwarf ranges section.
+ Asm->SwitchToDataSection(TAI->getDwarfRangesSection());
+
+ Asm->EOL();
+ }
+
+ /// EmitDebugMacInfo - Emit visible names into a debug macinfo section.
+ ///
+ void EmitDebugMacInfo() {
+ // Start the dwarf macinfo section.
+ Asm->SwitchToDataSection(TAI->getDwarfMacInfoSection());
+
+ Asm->EOL();
+ }
+
+ /// ConstructCompileUnitDIEs - Create a compile unit DIE for each source and
+ /// header file.
+ void ConstructCompileUnitDIEs() {
+ const UniqueVector<CompileUnitDesc *> CUW = MMI->getCompileUnits();
+
+ for (unsigned i = 1, N = CUW.size(); i <= N; ++i) {
+ unsigned ID = MMI->RecordSource(CUW[i]);
+ CompileUnit *Unit = NewCompileUnit(CUW[i], ID);
+ CompileUnits.push_back(Unit);
+ }
+ }
+
+ /// ConstructGlobalDIEs - Create DIEs for each of the externally visible
+ /// global variables.
+ void ConstructGlobalDIEs() {
+ std::vector<GlobalVariableDesc *> GlobalVariables =
+ MMI->getAnchoredDescriptors<GlobalVariableDesc>(*M);
+
+ for (unsigned i = 0, N = GlobalVariables.size(); i < N; ++i) {
+ GlobalVariableDesc *GVD = GlobalVariables[i];
+ NewGlobalVariable(GVD);
+ }
+ }
+
+ /// ConstructSubprogramDIEs - Create DIEs for each of the externally visible
+ /// subprograms.
+ void ConstructSubprogramDIEs() {
+ std::vector<SubprogramDesc *> Subprograms =
+ MMI->getAnchoredDescriptors<SubprogramDesc>(*M);
+
+ for (unsigned i = 0, N = Subprograms.size(); i < N; ++i) {
+ SubprogramDesc *SPD = Subprograms[i];
+ NewSubprogram(SPD);
+ }
+ }
+
+public:
+ //===--------------------------------------------------------------------===//
+ // Main entry points.
+ //
+ DwarfDebug(std::ostream &OS, AsmPrinter *A, const TargetAsmInfo *T)
+ : Dwarf(OS, A, T)
+ , CompileUnits()
+ , AbbreviationsSet(InitAbbreviationsSetSize)
+ , Abbreviations()
+ , ValuesSet(InitValuesSetSize)
+ , Values()
+ , StringPool()
+ , DescToUnitMap()
+ , SectionMap()
+ , SectionSourceLines()
+ , didInitial(false)
+ , shouldEmit(false)
+ {
+ }
+ virtual ~DwarfDebug() {
+ for (unsigned i = 0, N = CompileUnits.size(); i < N; ++i)
+ delete CompileUnits[i];
+ for (unsigned j = 0, M = Values.size(); j < M; ++j)
+ delete Values[j];
+ }
+
+ /// SetModuleInfo - Set machine module information when it's known that pass
+ /// manager has created it. Set by the target AsmPrinter.
+ void SetModuleInfo(MachineModuleInfo *mmi) {
+ // Make sure initial declarations are made.
+ if (!MMI && mmi->hasDebugInfo()) {
+ MMI = mmi;
+ shouldEmit = true;
+
+ // Emit initial sections
+ EmitInitial();
+
+ // Create all the compile unit DIEs.
+ ConstructCompileUnitDIEs();
+
+ // Create DIEs for each of the externally visible global variables.
+ ConstructGlobalDIEs();
+
+ // Create DIEs for each of the externally visible subprograms.
+ ConstructSubprogramDIEs();
+
+ // Prime section data.
+ SectionMap.insert(TAI->getTextSection());
+ }
+ }
+
+ /// BeginModule - Emit all Dwarf sections that should come prior to the
+ /// content.
+ void BeginModule(Module *M) {
+ this->M = M;
+
+ if (!ShouldEmitDwarf()) return;
+ }
+
+ /// EndModule - Emit all Dwarf sections that should come after the content.
+ ///
+ void EndModule() {
+ if (!ShouldEmitDwarf()) return;
+
+ // Standard sections final addresses.
+ Asm->SwitchToTextSection(TAI->getTextSection());
+ EmitLabel("text_end", 0);
+ Asm->SwitchToDataSection(TAI->getDataSection());
+ EmitLabel("data_end", 0);
+
+ // End text sections.
+ for (unsigned i = 1, N = SectionMap.size(); i <= N; ++i) {
+ Asm->SwitchToTextSection(SectionMap[i].c_str());
+ EmitLabel("section_end", i);
+ }
+
+ // Emit common frame information.
+ EmitCommonDebugFrame();
+
+ // Emit function debug frame information
+ for (std::vector<FunctionDebugFrameInfo>::iterator I = DebugFrames.begin(),
+ E = DebugFrames.end(); I != E; ++I)
+ EmitFunctionDebugFrame(*I);
+
+ // Compute DIE offsets and sizes.
+ SizeAndOffsets();
+
+ // Emit all the DIEs into a debug info section
+ EmitDebugInfo();
+
+ // Corresponding abbreviations into a abbrev section.
+ EmitAbbreviations();
+
+ // Emit source line correspondence into a debug line section.
+ EmitDebugLines();
+
+ // Emit info into a debug pubnames section.
+ EmitDebugPubNames();
+
+ // Emit info into a debug str section.
+ EmitDebugStr();
+
+ // Emit info into a debug loc section.
+ EmitDebugLoc();
+
+ // Emit info into a debug aranges section.
+ EmitDebugARanges();
+
+ // Emit info into a debug ranges section.
+ EmitDebugRanges();
+
+ // Emit info into a debug macinfo section.
+ EmitDebugMacInfo();
+ }
+
+ /// BeginFunction - Gather pre-function debug information. Assumes being
+ /// emitted immediately after the function entry point.
+ void BeginFunction(MachineFunction *MF) {
+ this->MF = MF;
+
+ if (!ShouldEmitDwarf()) return;
+
+ // Begin accumulating function debug information.
+ MMI->BeginFunction(MF);
+
+ // Assumes in correct section after the entry point.
+ EmitLabel("func_begin", ++SubprogramCount);
+ }
+
+ /// EndFunction - Gather and emit post-function debug information.
+ ///
+ void EndFunction() {
+ if (!ShouldEmitDwarf()) return;
+
+ // Define end label for subprogram.
+ EmitLabel("func_end", SubprogramCount);
+
+ // Get function line info.
+ const std::vector<SourceLineInfo> &LineInfos = MMI->getSourceLines();
+
+ if (!LineInfos.empty()) {
+ // Get section line info.
+ unsigned ID = SectionMap.insert(Asm->CurrentSection);
+ if (SectionSourceLines.size() < ID) SectionSourceLines.resize(ID);
+ std::vector<SourceLineInfo> &SectionLineInfos = SectionSourceLines[ID-1];
+ // Append the function info to section info.
+ SectionLineInfos.insert(SectionLineInfos.end(),
+ LineInfos.begin(), LineInfos.end());
+ }
+
+ // Construct scopes for subprogram.
+ ConstructRootScope(MMI->getRootScope());
+
+ DebugFrames.push_back(FunctionDebugFrameInfo(SubprogramCount,
+ MMI->getFrameMoves()));
+ }
+};
+
+//===----------------------------------------------------------------------===//
+/// DwarfException - Emits Dwarf exception handling directives.
+///
+class DwarfException : public Dwarf {
+
+private:
+ struct FunctionEHFrameInfo {
+ std::string FnName;
+ unsigned Number;
+ unsigned PersonalityIndex;
+ bool hasCalls;
+ bool hasLandingPads;
+ std::vector<MachineMove> Moves;
+
+ FunctionEHFrameInfo(const std::string &FN, unsigned Num, unsigned P,
+ bool hC, bool hL,
+ const std::vector<MachineMove> &M):
+ FnName(FN), Number(Num), PersonalityIndex(P),
+ hasCalls(hC), hasLandingPads(hL), Moves(M) { };
+ };
+
+ std::vector<FunctionEHFrameInfo> EHFrames;
+
+ /// shouldEmit - Flag to indicate if debug information should be emitted.
+ ///
+ bool shouldEmit;
+
+ /// EmitCommonEHFrame - Emit the common eh unwind frame.
+ ///
+ void EmitCommonEHFrame(const Function *Personality, unsigned Index) {
+ // Size and sign of stack growth.
+ int stackGrowth =
+ Asm->TM.getFrameInfo()->getStackGrowthDirection() ==
+ TargetFrameInfo::StackGrowsUp ?
+ TAI->getAddressSize() : -TAI->getAddressSize();
+
+ // Begin eh frame section.
+ Asm->SwitchToTextSection(TAI->getDwarfEHFrameSection());
+ O << "EH_frame" << Index << ":\n";
+ EmitLabel("section_eh_frame", Index);
+
+ // Define base labels.
+ EmitLabel("eh_frame_common", Index);
+
+ // Define the eh frame length.
+ EmitDifference("eh_frame_common_end", Index,
+ "eh_frame_common_begin", Index, true);
+ Asm->EOL("Length of Common Information Entry");
+
+ // EH frame header.
+ EmitLabel("eh_frame_common_begin", Index);
+ Asm->EmitInt32((int)0);
+ Asm->EOL("CIE Identifier Tag");
+ Asm->EmitInt8(DW_CIE_VERSION);
+ Asm->EOL("CIE Version");
+
+ // The personality presence indicates that language specific information
+ // will show up in the eh frame.
+ Asm->EmitString(Personality ? "zPLR" : "zR");
+ Asm->EOL("CIE Augmentation");
+
+ // Round out reader.
+ Asm->EmitULEB128Bytes(1);
+ Asm->EOL("CIE Code Alignment Factor");
+ Asm->EmitSLEB128Bytes(stackGrowth);
+ Asm->EOL("CIE Data Alignment Factor");
+ Asm->EmitInt8(RI->getDwarfRegNum(RI->getRARegister()));
+ Asm->EOL("CIE RA Column");
+
+ // If there is a personality, we need to indicate the functions location.
+ if (Personality) {
+ Asm->EmitULEB128Bytes(7);
+ Asm->EOL("Augmentation Size");
+ Asm->EmitInt8(DW_EH_PE_pcrel | DW_EH_PE_sdata4);
+ Asm->EOL("Personality (pcrel sdata4)");
+
+ O << TAI->getData32bitsDirective();
+ Asm->EmitExternalGlobal((const GlobalVariable *)(Personality));
+ O << "-" << TAI->getPCSymbol();
+ Asm->EOL("Personality");
+
+ Asm->EmitULEB128Bytes(DW_EH_PE_pcrel);
+ Asm->EOL("LSDA Encoding (pcrel)");
+ Asm->EmitULEB128Bytes(DW_EH_PE_pcrel);
+ Asm->EOL("FDE Encoding (pcrel)");
+ } else {
+ Asm->EmitULEB128Bytes(1);
+ Asm->EOL("Augmentation Size");
+ Asm->EmitULEB128Bytes(DW_EH_PE_pcrel);
+ Asm->EOL("FDE Encoding (pcrel)");
+ }
+
+ // Indicate locations of general callee saved registers in frame.
+ std::vector<MachineMove> Moves;
+ RI->getInitialFrameState(Moves);
+ EmitFrameMoves(NULL, 0, Moves);
+
+ Asm->EmitAlignment(2);
+ EmitLabel("eh_frame_common_end", Index);
+
+ Asm->EOL();
+ }
+
+ /// EmitEHFrame - Emit function exception frame information.
+ ///
+ void EmitEHFrame(const FunctionEHFrameInfo &EHFrameInfo) {
+ Asm->SwitchToTextSection(TAI->getDwarfEHFrameSection());
+
+ // Externally visible entry into the functions eh frame info.
+ if (const char *GlobalDirective = TAI->getGlobalDirective())
+ O << GlobalDirective << EHFrameInfo.FnName << ".eh\n";
+
+ // If there are no calls then you can't unwind.
+ if (!EHFrameInfo.hasCalls) {
+ O << EHFrameInfo.FnName << ".eh = 0\n";
+ } else {
+ O << EHFrameInfo.FnName << ".eh:\n";
+
+ // EH frame header.
+ EmitDifference("eh_frame_end", EHFrameInfo.Number,
+ "eh_frame_begin", EHFrameInfo.Number, true);
+ Asm->EOL("Length of Frame Information Entry");
+
+ EmitLabel("eh_frame_begin", EHFrameInfo.Number);
+
+ EmitSectionOffset("eh_frame_begin", "eh_frame_common",
+ EHFrameInfo.Number, EHFrameInfo.PersonalityIndex,
+ true, true);
+ Asm->EOL("FDE CIE offset");
+
+ EmitReference("eh_func_begin", EHFrameInfo.Number, true);
+ Asm->EOL("FDE initial location");
+ EmitDifference("eh_func_end", EHFrameInfo.Number,
+ "eh_func_begin", EHFrameInfo.Number);
+ Asm->EOL("FDE address range");
+
+ // If there is a personality and landing pads then point to the language
+ // specific data area in the exception table.
+ if (EHFrameInfo.PersonalityIndex) {
+ Asm->EmitULEB128Bytes(4);
+ Asm->EOL("Augmentation size");
+
+ if (EHFrameInfo.hasLandingPads) {
+ EmitReference("exception", EHFrameInfo.Number, true);
+ } else if(TAI->getAddressSize() == 8) {
+ Asm->EmitInt64((int)0);
+ } else {
+ Asm->EmitInt32((int)0);
+ }
+ Asm->EOL("Language Specific Data Area");
+ } else {
+ Asm->EmitULEB128Bytes(0);
+ Asm->EOL("Augmentation size");
+ }
+
+ // Indicate locations of function specific callee saved registers in
+ // frame.
+ EmitFrameMoves("eh_func_begin", EHFrameInfo.Number, EHFrameInfo.Moves);
+
+ Asm->EmitAlignment(2);
+ EmitLabel("eh_frame_end", EHFrameInfo.Number);
+ }
+
+ if (const char *UsedDirective = TAI->getUsedDirective())
+ O << UsedDirective << EHFrameInfo.FnName << ".eh\n\n";
+ }
+
+ /// EmitExceptionTable - Emit landpads and actions.
+ ///
+ /// The general organization of the table is complex, but the basic concepts
+ /// are easy. First there is a header which describes the location and
+ /// organization of the three components that follow.
+ /// 1. The landing pad site information describes the range of code covered
+ /// by the try. In our case it's an accumulation of the ranges covered
+ /// by the invokes in the try. There is also a reference to the landing
+ /// pad that handles the exception once processed. Finally an index into
+ /// the actions table.
+ /// 2. The action table, in our case, is composed of pairs of type ids
+ /// and next action offset. Starting with the action index from the
+ /// landing pad site, each type Id is checked for a match to the current
+ /// exception. If it matches then the exception and type id are passed
+ /// on to the landing pad. Otherwise the next action is looked up. This
+ /// chain is terminated with a next action of zero. If no type id is
+ /// found the the frame is unwound and handling continues.
+ /// 3. Type id table contains references to all the C++ typeinfo for all
+ /// catches in the function. This tables is reversed indexed base 1.
+
+ /// SharedTypeIds - How many leading type ids two landing pads have in common.
+ static unsigned SharedTypeIds(const LandingPadInfo *L,
+ const LandingPadInfo *R) {
+ const std::vector<int> &LIds = L->TypeIds, &RIds = R->TypeIds;
+ unsigned LSize = LIds.size(), RSize = RIds.size();
+ unsigned MinSize = LSize < RSize ? LSize : RSize;
+ unsigned Count = 0;
+
+ for (; Count != MinSize; ++Count)
+ if (LIds[Count] != RIds[Count])
+ return Count;
+
+ return Count;
+ }
+
+ /// PadLT - Order landing pads lexicographically by type id.
+ static bool PadLT(const LandingPadInfo *L, const LandingPadInfo *R) {
+ const std::vector<int> &LIds = L->TypeIds, &RIds = R->TypeIds;
+ unsigned LSize = LIds.size(), RSize = RIds.size();
+ unsigned MinSize = LSize < RSize ? LSize : RSize;
+
+ for (unsigned i = 0; i != MinSize; ++i)
+ if (LIds[i] != RIds[i])
+ return LIds[i] < RIds[i];
+
+ return LSize < RSize;
+ }
+
+ struct KeyInfo {
+ static inline unsigned getEmptyKey() { return -1U; }
+ static inline unsigned getTombstoneKey() { return -2U; }
+ static unsigned getHashValue(const unsigned &Key) { return Key; }
+ static bool isPod() { return true; }
+ };
+
+ struct PadSite {
+ unsigned PadIndex;
+ unsigned SiteIndex;
+ };
+
+ typedef DenseMap<unsigned, PadSite, KeyInfo> PadMapType;
+
+ struct ActionEntry {
+ int ValueForTypeID; // The value to write - may not be equal to the type id.
+ int NextAction;
+ struct ActionEntry *Previous;
+ };
+
+ void EmitExceptionTable() {
+ // Map all labels and get rid of any dead landing pads.
+ MMI->TidyLandingPads();
+
+ const std::vector<GlobalVariable *> &TypeInfos = MMI->getTypeInfos();
+ const std::vector<unsigned> &FilterIds = MMI->getFilterIds();
+ const std::vector<LandingPadInfo> &PadInfos = MMI->getLandingPads();
+ if (PadInfos.empty()) return;
+
+ // Sort the landing pads in order of their type ids. This is used to fold
+ // duplicate actions.
+ SmallVector<const LandingPadInfo *, 64> LandingPads;
+ LandingPads.reserve(PadInfos.size());
+ for (unsigned i = 0, N = PadInfos.size(); i != N; ++i)
+ LandingPads.push_back(&PadInfos[i]);
+ std::sort(LandingPads.begin(), LandingPads.end(), PadLT);
+
+ // Gather first action index for each landing pad site.
+ SmallVector<unsigned, 64> FirstActions;
+ FirstActions.reserve(PadInfos.size());
+
+ // The actions table.
+ SmallVector<ActionEntry, 32> Actions;
+
+ // Negative type ids index into FilterIds, positive type ids index into
+ // TypeInfos. The value written for a positive type id is just the type
+ // id itself. For a negative type id, however, the value written is the
+ // (negative) byte offset of the corresponding FilterIds entry. The byte
+ // offset is usually equal to the type id, because the FilterIds entries
+ // are written using a variable width encoding which outputs one byte per
+ // entry as long as the value written is not too large, but can differ.
+ // This kind of complication does not occur for positive type ids because
+ // type infos are output using a fixed width encoding.
+ // FilterOffsets[i] holds the byte offset corresponding to FilterIds[i].
+ SmallVector<int, 16> FilterOffsets;
+ FilterOffsets.reserve(FilterIds.size());
+ int Offset = -1;
+ for(std::vector<unsigned>::const_iterator I = FilterIds.begin(),
+ E = FilterIds.end(); I != E; ++I) {
+ FilterOffsets.push_back(Offset);
+ Offset -= Asm->SizeULEB128(*I);
+ }
+
+ // Compute sizes for exception table.
+ unsigned SizeSites = 0;
+ unsigned SizeActions = 0;
+
+ // Look at each landing pad site to compute size. We need the size of each
+ // landing pad site info and the size of the landing pad's actions.
+ int FirstAction = 0;
+
+ for (unsigned i = 0, N = LandingPads.size(); i != N; ++i) {
+ const LandingPadInfo *LP = LandingPads[i];
+ const std::vector<int> &TypeIds = LP->TypeIds;
+ const unsigned NumShared = i ? SharedTypeIds(LP, LandingPads[i-1]) : 0;
+ unsigned SizeSiteActions = 0;
+
+ if (NumShared < TypeIds.size()) {
+ unsigned SizeAction = 0;
+ ActionEntry *PrevAction = 0;
+
+ if (NumShared) {
+ const unsigned SizePrevIds = LandingPads[i-1]->TypeIds.size();
+ assert(Actions.size());
+ PrevAction = &Actions.back();
+ SizeAction = Asm->SizeSLEB128(PrevAction->NextAction) +
+ Asm->SizeSLEB128(PrevAction->ValueForTypeID);
+ for (unsigned j = NumShared; j != SizePrevIds; ++j) {
+ SizeAction -= Asm->SizeSLEB128(PrevAction->ValueForTypeID);
+ SizeAction += -PrevAction->NextAction;
+ PrevAction = PrevAction->Previous;
+ }
+ }
+
+ // Compute the actions.
+ for (unsigned I = NumShared, M = TypeIds.size(); I != M; ++I) {
+ int TypeID = TypeIds[I];
+ assert(-1-TypeID < (int)FilterOffsets.size() && "Unknown filter id!");
+ int ValueForTypeID = TypeID < 0 ? FilterOffsets[-1 - TypeID] : TypeID;
+ unsigned SizeTypeID = Asm->SizeSLEB128(ValueForTypeID);
+
+ int NextAction = SizeAction ? -(SizeAction + SizeTypeID) : 0;
+ SizeAction = SizeTypeID + Asm->SizeSLEB128(NextAction);
+ SizeSiteActions += SizeAction;
+
+ ActionEntry Action = {ValueForTypeID, NextAction, PrevAction};
+ Actions.push_back(Action);
+
+ PrevAction = &Actions.back();
+ }
+
+ // Record the first action of the landing pad site.
+ FirstAction = SizeActions + SizeSiteActions - SizeAction + 1;
+ } // else identical - re-use previous FirstAction
+
+ FirstActions.push_back(FirstAction);
+
+ // Compute this sites contribution to size.
+ SizeActions += SizeSiteActions;
+ unsigned M = LP->BeginLabels.size();
+ SizeSites += M*(sizeof(int32_t) + // Site start.
+ sizeof(int32_t) + // Site length.
+ sizeof(int32_t) + // Landing pad.
+ Asm->SizeULEB128(FirstAction)); // Action.
+ }
+
+ // Final tallies.
+ unsigned SizeTypes = TypeInfos.size() * TAI->getAddressSize();
+
+ unsigned TypeOffset = sizeof(int8_t) + // Call site format
+ Asm->SizeULEB128(SizeSites) + // Call-site table length
+ SizeSites + SizeActions + SizeTypes;
+
+ unsigned TotalSize = sizeof(int8_t) + // LPStart format
+ sizeof(int8_t) + // TType format
+ Asm->SizeULEB128(TypeOffset) + // TType base offset
+ TypeOffset;
+
+ unsigned SizeAlign = (4 - TotalSize) & 3;
+
+ // Begin the exception table.
+ Asm->SwitchToDataSection(TAI->getDwarfExceptionSection());
+ O << "GCC_except_table" << SubprogramCount << ":\n";
+ Asm->EmitAlignment(2);
+ for (unsigned i = 0; i != SizeAlign; ++i) {
+ Asm->EmitInt8(0);
+ Asm->EOL("Padding");
+ }
+ EmitLabel("exception", SubprogramCount);
+
+ // Emit the header.
+ Asm->EmitInt8(DW_EH_PE_omit);
+ Asm->EOL("LPStart format (DW_EH_PE_omit)");
+ Asm->EmitInt8(DW_EH_PE_absptr);
+ Asm->EOL("TType format (DW_EH_PE_absptr)");
+ Asm->EmitULEB128Bytes(TypeOffset);
+ Asm->EOL("TType base offset");
+ Asm->EmitInt8(DW_EH_PE_udata4);
+ Asm->EOL("Call site format (DW_EH_PE_udata4)");
+ Asm->EmitULEB128Bytes(SizeSites);
+ Asm->EOL("Call-site table length");
+
+ // Emit the landing pad site information in order of address.
+ PadMapType PadMap;
+
+ for (unsigned i = 0, N = LandingPads.size(); i != N; ++i) {
+ const LandingPadInfo *LandingPad = LandingPads[i];
+ for (unsigned j=0, E = LandingPad->BeginLabels.size(); j != E; ++j) {
+ unsigned BeginLabel = LandingPad->BeginLabels[j];
+ assert(!PadMap.count(BeginLabel) && "duplicate landing pad labels!");
+ PadSite P = { i, j };
+ PadMap[BeginLabel] = P;
+ }
+ }
+
+ for (MachineFunction::const_iterator I = MF->begin(), E = MF->end();
+ I != E; ++I) {
+ for (MachineBasicBlock::const_iterator MI = I->begin(), E = I->end();
+ MI != E; ++MI) {
+ if (MI->getOpcode() != TargetInstrInfo::LABEL)
+ continue;
+
+ unsigned BeginLabel = MI->getOperand(0).getImmedValue();
+ PadMapType::iterator L = PadMap.find(BeginLabel);
+
+ if (L == PadMap.end())
+ continue;
+
+ PadSite P = L->second;
+ const LandingPadInfo *LandingPad = LandingPads[P.PadIndex];
+
+ assert(BeginLabel == LandingPad->BeginLabels[P.SiteIndex] &&
+ "Inconsistent landing pad map!");
+
+ EmitSectionOffset("label", "eh_func_begin", BeginLabel, SubprogramCount,
+ false, true);
+ Asm->EOL("Region start");
+
+ EmitDifference("label", LandingPad->EndLabels[P.SiteIndex],
+ "label", BeginLabel);
+ Asm->EOL("Region length");
+
+ if (LandingPad->TypeIds.empty()) {
+ if (TAI->getAddressSize() == sizeof(int32_t))
+ Asm->EmitInt32(0);
+ else
+ Asm->EmitInt64(0);
+ } else {
+ EmitSectionOffset("label", "eh_func_begin",
+ LandingPad->LandingPadLabel, SubprogramCount,
+ false, true);
+ }
+ Asm->EOL("Landing pad");
+
+ Asm->EmitULEB128Bytes(FirstActions[P.PadIndex]);
+ Asm->EOL("Action");
+ }
+ }
+
+ // Emit the actions.
+ for (unsigned I = 0, N = Actions.size(); I != N; ++I) {
+ ActionEntry &Action = Actions[I];
+
+ Asm->EmitSLEB128Bytes(Action.ValueForTypeID);
+ Asm->EOL("TypeInfo index");
+ Asm->EmitSLEB128Bytes(Action.NextAction);
+ Asm->EOL("Next action");
+ }
+
+ // Emit the type ids.
+ for (unsigned M = TypeInfos.size(); M; --M) {
+ GlobalVariable *GV = TypeInfos[M - 1];
+
+ if (TAI->getAddressSize() == sizeof(int32_t))
+ O << TAI->getData32bitsDirective();
+ else
+ O << TAI->getData64bitsDirective();
+
+ if (GV)
+ O << Asm->getGlobalLinkName(GV);
+ else
+ O << "0";
+
+ Asm->EOL("TypeInfo");
+ }
+
+ // Emit the filter typeids.
+ for (unsigned j = 0, M = FilterIds.size(); j < M; ++j) {
+ unsigned TypeID = FilterIds[j];
+ Asm->EmitULEB128Bytes(TypeID);
+ Asm->EOL("Filter TypeInfo index");
+ }
+
+ Asm->EmitAlignment(2);
+ }
+
+public:
+ //===--------------------------------------------------------------------===//
+ // Main entry points.
+ //
+ DwarfException(std::ostream &OS, AsmPrinter *A, const TargetAsmInfo *T)
+ : Dwarf(OS, A, T)
+ , shouldEmit(false)
+ {}
+
+ virtual ~DwarfException() {}
+
+ /// SetModuleInfo - Set machine module information when it's known that pass
+ /// manager has created it. Set by the target AsmPrinter.
+ void SetModuleInfo(MachineModuleInfo *mmi) {
+ MMI = mmi;
+ }
+
+ /// BeginModule - Emit all exception information that should come prior to the
+ /// content.
+ void BeginModule(Module *M) {
+ this->M = M;
+ }
+
+ /// EndModule - Emit all exception information that should come after the
+ /// content.
+ void EndModule() {
+ if (!shouldEmit) return;
+
+ const std::vector<Function *> Personalities = MMI->getPersonalities();
+ for (unsigned i =0; i < Personalities.size(); ++i)
+ EmitCommonEHFrame(Personalities[i], i);
+
+ for (std::vector<FunctionEHFrameInfo>::iterator I = EHFrames.begin(),
+ E = EHFrames.end(); I != E; ++I)
+ EmitEHFrame(*I);
+ }
+
+ /// BeginFunction - Gather pre-function exception information. Assumes being
+ /// emitted immediately after the function entry point.
+ void BeginFunction(MachineFunction *MF) {
+ this->MF = MF;
+
+ if (MMI &&
+ ExceptionHandling &&
+ TAI->doesSupportExceptionHandling()) {
+ shouldEmit = true;
+ // Assumes in correct section after the entry point.
+ EmitLabel("eh_func_begin", ++SubprogramCount);
+ }
+ }
+
+ /// EndFunction - Gather and emit post-function exception information.
+ ///
+ void EndFunction() {
+ if (!shouldEmit) return;
+
+ EmitLabel("eh_func_end", SubprogramCount);
+ EmitExceptionTable();
+
+ // Save EH frame information
+ EHFrames.push_back(FunctionEHFrameInfo(getAsm()->CurrentFnName,
+ SubprogramCount,
+ MMI->getPersonalityIndex(),
+ MF->getFrameInfo()->hasCalls(),
+ !MMI->getLandingPads().empty(),
+ MMI->getFrameMoves()));
+ }
+};
+
+} // End of namespace llvm
+
+//===----------------------------------------------------------------------===//
+
+/// Emit - Print the abbreviation using the specified Dwarf writer.
+///
+void DIEAbbrev::Emit(const DwarfDebug &DD) const {
+ // Emit its Dwarf tag type.
+ DD.getAsm()->EmitULEB128Bytes(Tag);
+ DD.getAsm()->EOL(TagString(Tag));
+
+ // Emit whether it has children DIEs.
+ DD.getAsm()->EmitULEB128Bytes(ChildrenFlag);
+ DD.getAsm()->EOL(ChildrenString(ChildrenFlag));
+
+ // For each attribute description.
+ for (unsigned i = 0, N = Data.size(); i < N; ++i) {
+ const DIEAbbrevData &AttrData = Data[i];
+
+ // Emit attribute type.
+ DD.getAsm()->EmitULEB128Bytes(AttrData.getAttribute());
+ DD.getAsm()->EOL(AttributeString(AttrData.getAttribute()));
+
+ // Emit form type.
+ DD.getAsm()->EmitULEB128Bytes(AttrData.getForm());
+ DD.getAsm()->EOL(FormEncodingString(AttrData.getForm()));
+ }
+
+ // Mark end of abbreviation.
+ DD.getAsm()->EmitULEB128Bytes(0); DD.getAsm()->EOL("EOM(1)");
+ DD.getAsm()->EmitULEB128Bytes(0); DD.getAsm()->EOL("EOM(2)");
+}
+
+#ifndef NDEBUG
+void DIEAbbrev::print(std::ostream &O) {
+ O << "Abbreviation @"
+ << std::hex << (intptr_t)this << std::dec
+ << " "
+ << TagString(Tag)
+ << " "
+ << ChildrenString(ChildrenFlag)
+ << "\n";
+
+ for (unsigned i = 0, N = Data.size(); i < N; ++i) {
+ O << " "
+ << AttributeString(Data[i].getAttribute())
+ << " "
+ << FormEncodingString(Data[i].getForm())
+ << "\n";
+ }
+}
+void DIEAbbrev::dump() { print(cerr); }
+#endif
+
+//===----------------------------------------------------------------------===//
+
+#ifndef NDEBUG
+void DIEValue::dump() {
+ print(cerr);
+}
+#endif
+
+//===----------------------------------------------------------------------===//
+
+/// EmitValue - Emit integer of appropriate size.
+///
+void DIEInteger::EmitValue(DwarfDebug &DD, unsigned Form) {
+ switch (Form) {
+ case DW_FORM_flag: // Fall thru
+ case DW_FORM_ref1: // Fall thru
+ case DW_FORM_data1: DD.getAsm()->EmitInt8(Integer); break;
+ case DW_FORM_ref2: // Fall thru
+ case DW_FORM_data2: DD.getAsm()->EmitInt16(Integer); break;
+ case DW_FORM_ref4: // Fall thru
+ case DW_FORM_data4: DD.getAsm()->EmitInt32(Integer); break;
+ case DW_FORM_ref8: // Fall thru
+ case DW_FORM_data8: DD.getAsm()->EmitInt64(Integer); break;
+ case DW_FORM_udata: DD.getAsm()->EmitULEB128Bytes(Integer); break;
+ case DW_FORM_sdata: DD.getAsm()->EmitSLEB128Bytes(Integer); break;
+ default: assert(0 && "DIE Value form not supported yet"); break;
+ }
+}
+
+/// SizeOf - Determine size of integer value in bytes.
+///
+unsigned DIEInteger::SizeOf(const DwarfDebug &DD, unsigned Form) const {
+ switch (Form) {
+ case DW_FORM_flag: // Fall thru
+ case DW_FORM_ref1: // Fall thru
+ case DW_FORM_data1: return sizeof(int8_t);
+ case DW_FORM_ref2: // Fall thru
+ case DW_FORM_data2: return sizeof(int16_t);
+ case DW_FORM_ref4: // Fall thru
+ case DW_FORM_data4: return sizeof(int32_t);
+ case DW_FORM_ref8: // Fall thru
+ case DW_FORM_data8: return sizeof(int64_t);
+ case DW_FORM_udata: return DD.getAsm()->SizeULEB128(Integer);
+ case DW_FORM_sdata: return DD.getAsm()->SizeSLEB128(Integer);
+ default: assert(0 && "DIE Value form not supported yet"); break;
+ }
+ return 0;
+}
+
+//===----------------------------------------------------------------------===//
+
+/// EmitValue - Emit string value.
+///
+void DIEString::EmitValue(DwarfDebug &DD, unsigned Form) {
+ DD.getAsm()->EmitString(String);
+}
+
+//===----------------------------------------------------------------------===//
+
+/// EmitValue - Emit label value.
+///
+void DIEDwarfLabel::EmitValue(DwarfDebug &DD, unsigned Form) {
+ DD.EmitReference(Label);
+}
+
+/// SizeOf - Determine size of label value in bytes.
+///
+unsigned DIEDwarfLabel::SizeOf(const DwarfDebug &DD, unsigned Form) const {
+ return DD.getTargetAsmInfo()->getAddressSize();
+}
+
+//===----------------------------------------------------------------------===//
+
+/// EmitValue - Emit label value.
+///
+void DIEObjectLabel::EmitValue(DwarfDebug &DD, unsigned Form) {
+ DD.EmitReference(Label);
+}
+
+/// SizeOf - Determine size of label value in bytes.
+///
+unsigned DIEObjectLabel::SizeOf(const DwarfDebug &DD, unsigned Form) const {
+ return DD.getTargetAsmInfo()->getAddressSize();
+}
+
+//===----------------------------------------------------------------------===//
+
+/// EmitValue - Emit delta value.
+///
+void DIEDelta::EmitValue(DwarfDebug &DD, unsigned Form) {
+ bool IsSmall = Form == DW_FORM_data4;
+ DD.EmitDifference(LabelHi, LabelLo, IsSmall);
+}
+
+/// SizeOf - Determine size of delta value in bytes.
+///
+unsigned DIEDelta::SizeOf(const DwarfDebug &DD, unsigned Form) const {
+ if (Form == DW_FORM_data4) return 4;
+ return DD.getTargetAsmInfo()->getAddressSize();
+}
+
+//===----------------------------------------------------------------------===//
+
+/// EmitValue - Emit debug information entry offset.
+///
+void DIEntry::EmitValue(DwarfDebug &DD, unsigned Form) {
+ DD.getAsm()->EmitInt32(Entry->getOffset());
+}
+
+//===----------------------------------------------------------------------===//
+
+/// ComputeSize - calculate the size of the block.
+///
+unsigned DIEBlock::ComputeSize(DwarfDebug &DD) {
+ if (!Size) {
+ const std::vector<DIEAbbrevData> &AbbrevData = Abbrev.getData();
+
+ for (unsigned i = 0, N = Values.size(); i < N; ++i) {
+ Size += Values[i]->SizeOf(DD, AbbrevData[i].getForm());
+ }
+ }
+ return Size;
+}
+
+/// EmitValue - Emit block data.
+///
+void DIEBlock::EmitValue(DwarfDebug &DD, unsigned Form) {
+ switch (Form) {
+ case DW_FORM_block1: DD.getAsm()->EmitInt8(Size); break;
+ case DW_FORM_block2: DD.getAsm()->EmitInt16(Size); break;
+ case DW_FORM_block4: DD.getAsm()->EmitInt32(Size); break;
+ case DW_FORM_block: DD.getAsm()->EmitULEB128Bytes(Size); break;
+ default: assert(0 && "Improper form for block"); break;
+ }
+
+ const std::vector<DIEAbbrevData> &AbbrevData = Abbrev.getData();
+
+ for (unsigned i = 0, N = Values.size(); i < N; ++i) {
+ DD.getAsm()->EOL();
+ Values[i]->EmitValue(DD, AbbrevData[i].getForm());
+ }
+}
+
+/// SizeOf - Determine size of block data in bytes.
+///
+unsigned DIEBlock::SizeOf(const DwarfDebug &DD, unsigned Form) const {
+ switch (Form) {
+ case DW_FORM_block1: return Size + sizeof(int8_t);
+ case DW_FORM_block2: return Size + sizeof(int16_t);
+ case DW_FORM_block4: return Size + sizeof(int32_t);
+ case DW_FORM_block: return Size + DD.getAsm()->SizeULEB128(Size);
+ default: assert(0 && "Improper form for block"); break;
+ }
+ return 0;
+}
+
+//===----------------------------------------------------------------------===//
+/// DIE Implementation
+
+DIE::~DIE() {
+ for (unsigned i = 0, N = Children.size(); i < N; ++i)
+ delete Children[i];
+}
+
+/// AddSiblingOffset - Add a sibling offset field to the front of the DIE.
+///
+void DIE::AddSiblingOffset() {
+ DIEInteger *DI = new DIEInteger(0);
+ Values.insert(Values.begin(), DI);
+ Abbrev.AddFirstAttribute(DW_AT_sibling, DW_FORM_ref4);
+}
+
+/// Profile - Used to gather unique data for the value folding set.
+///
+void DIE::Profile(FoldingSetNodeID &ID) {
+ Abbrev.Profile(ID);
+
+ for (unsigned i = 0, N = Children.size(); i < N; ++i)
+ ID.AddPointer(Children[i]);
+
+ for (unsigned j = 0, M = Values.size(); j < M; ++j)
+ ID.AddPointer(Values[j]);
+}
+
+#ifndef NDEBUG
+void DIE::print(std::ostream &O, unsigned IncIndent) {
+ static unsigned IndentCount = 0;
+ IndentCount += IncIndent;
+ const std::string Indent(IndentCount, ' ');
+ bool isBlock = Abbrev.getTag() == 0;
+
+ if (!isBlock) {
+ O << Indent
+ << "Die: "
+ << "0x" << std::hex << (intptr_t)this << std::dec
+ << ", Offset: " << Offset
+ << ", Size: " << Size
+ << "\n";
+
+ O << Indent
+ << TagString(Abbrev.getTag())
+ << " "
+ << ChildrenString(Abbrev.getChildrenFlag());
+ } else {
+ O << "Size: " << Size;
+ }
+ O << "\n";
+
+ const std::vector<DIEAbbrevData> &Data = Abbrev.getData();
+
+ IndentCount += 2;
+ for (unsigned i = 0, N = Data.size(); i < N; ++i) {
+ O << Indent;
+ if (!isBlock) {
+ O << AttributeString(Data[i].getAttribute());
+ } else {
+ O << "Blk[" << i << "]";
+ }
+ O << " "
+ << FormEncodingString(Data[i].getForm())
+ << " ";
+ Values[i]->print(O);
+ O << "\n";
+ }
+ IndentCount -= 2;
+
+ for (unsigned j = 0, M = Children.size(); j < M; ++j) {
+ Children[j]->print(O, 4);
+ }
+
+ if (!isBlock) O << "\n";
+ IndentCount -= IncIndent;
+}
+
+void DIE::dump() {
+ print(cerr);
+}
+#endif
+
+//===----------------------------------------------------------------------===//
+/// DwarfWriter Implementation
+///
+
+DwarfWriter::DwarfWriter(std::ostream &OS, AsmPrinter *A,
+ const TargetAsmInfo *T) {
+ DE = new DwarfException(OS, A, T);
+ DD = new DwarfDebug(OS, A, T);
+}
+
+DwarfWriter::~DwarfWriter() {
+ delete DE;
+ delete DD;
+}
+
+/// SetModuleInfo - Set machine module info when it's known that pass manager
+/// has created it. Set by the target AsmPrinter.
+void DwarfWriter::SetModuleInfo(MachineModuleInfo *MMI) {
+ DD->SetModuleInfo(MMI);
+ DE->SetModuleInfo(MMI);
+}
+
+/// BeginModule - Emit all Dwarf sections that should come prior to the
+/// content.
+void DwarfWriter::BeginModule(Module *M) {
+ DE->BeginModule(M);
+ DD->BeginModule(M);
+}
+
+/// EndModule - Emit all Dwarf sections that should come after the content.
+///
+void DwarfWriter::EndModule() {
+ DE->EndModule();
+ DD->EndModule();
+}
+
+/// BeginFunction - Gather pre-function debug information. Assumes being
+/// emitted immediately after the function entry point.
+void DwarfWriter::BeginFunction(MachineFunction *MF) {
+ DE->BeginFunction(MF);
+ DD->BeginFunction(MF);
+}
+
+/// EndFunction - Gather and emit post-function debug information.
+///
+void DwarfWriter::EndFunction() {
+ DD->EndFunction();
+ DE->EndFunction();
+
+ if (MachineModuleInfo *MMI = DD->getMMI() ? DD->getMMI() : DE->getMMI()) {
+ // Clear function debug information.
+ MMI->EndFunction();
+ }
+}
diff --git a/lib/CodeGen/ELFWriter.cpp b/lib/CodeGen/ELFWriter.cpp
new file mode 100644
index 0000000..8ecddb8
--- /dev/null
+++ b/lib/CodeGen/ELFWriter.cpp
@@ -0,0 +1,547 @@
+//===-- ELFWriter.cpp - Target-independent ELF Writer code ----------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file was developed by Chris Lattner and is distributed under the
+// University of Illinois Open Source License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements the target-independent ELF writer. This file writes out
+// the ELF file in the following order:
+//
+// #1. ELF Header
+// #2. '.text' section
+// #3. '.data' section
+// #4. '.bss' section (conceptual position in file)
+// ...
+// #X. '.shstrtab' section
+// #Y. Section Table
+//
+// The entries in the section table are laid out as:
+// #0. Null entry [required]
+// #1. ".text" entry - the program code
+// #2. ".data" entry - global variables with initializers. [ if needed ]
+// #3. ".bss" entry - global variables without initializers. [ if needed ]
+// ...
+// #N. ".shstrtab" entry - String table for the section names.
+//
+// NOTE: This code should eventually be extended to support 64-bit ELF (this
+// won't be hard), but we haven't done so yet!
+//
+//===----------------------------------------------------------------------===//
+
+#include "ELFWriter.h"
+#include "llvm/Module.h"
+#include "llvm/PassManager.h"
+#include "llvm/CodeGen/FileWriters.h"
+#include "llvm/CodeGen/MachineCodeEmitter.h"
+#include "llvm/CodeGen/MachineConstantPool.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/Target/TargetData.h"
+#include "llvm/Target/TargetELFWriterInfo.h"
+#include "llvm/Target/TargetMachine.h"
+#include "llvm/Support/Mangler.h"
+#include "llvm/Support/OutputBuffer.h"
+#include "llvm/Support/Streams.h"
+#include <list>
+using namespace llvm;
+
+char ELFWriter::ID = 0;
+/// AddELFWriter - Concrete function to add the ELF writer to the function pass
+/// manager.
+MachineCodeEmitter *llvm::AddELFWriter(FunctionPassManager &FPM,
+ std::ostream &O,
+ TargetMachine &TM) {
+ ELFWriter *EW = new ELFWriter(O, TM);
+ FPM.add(EW);
+ return &EW->getMachineCodeEmitter();
+}
+
+//===----------------------------------------------------------------------===//
+// ELFCodeEmitter Implementation
+//===----------------------------------------------------------------------===//
+
+namespace llvm {
+ /// ELFCodeEmitter - This class is used by the ELFWriter to emit the code for
+ /// functions to the ELF file.
+ class ELFCodeEmitter : public MachineCodeEmitter {
+ ELFWriter &EW;
+ TargetMachine &TM;
+ ELFWriter::ELFSection *ES; // Section to write to.
+ std::vector<unsigned char> *OutBuffer;
+ size_t FnStart;
+ public:
+ ELFCodeEmitter(ELFWriter &ew) : EW(ew), TM(EW.TM), OutBuffer(0) {}
+
+ void startFunction(MachineFunction &F);
+ bool finishFunction(MachineFunction &F);
+
+ void addRelocation(const MachineRelocation &MR) {
+ assert(0 && "relo not handled yet!");
+ }
+
+ virtual void StartMachineBasicBlock(MachineBasicBlock *MBB) {
+ }
+
+ virtual intptr_t getConstantPoolEntryAddress(unsigned Index) const {
+ assert(0 && "CP not implementated yet!");
+ return 0;
+ }
+ virtual intptr_t getJumpTableEntryAddress(unsigned Index) const {
+ assert(0 && "JT not implementated yet!");
+ return 0;
+ }
+
+ virtual intptr_t getMachineBasicBlockAddress(MachineBasicBlock *MBB) const {
+ assert(0 && "JT not implementated yet!");
+ return 0;
+ }
+
+ /// JIT SPECIFIC FUNCTIONS - DO NOT IMPLEMENT THESE HERE!
+ void startFunctionStub(unsigned StubSize, unsigned Alignment = 1) {
+ assert(0 && "JIT specific function called!");
+ abort();
+ }
+ void *finishFunctionStub(const Function *F) {
+ assert(0 && "JIT specific function called!");
+ abort();
+ return 0;
+ }
+ };
+}
+
+/// startFunction - This callback is invoked when a new machine function is
+/// about to be emitted.
+void ELFCodeEmitter::startFunction(MachineFunction &F) {
+ // Align the output buffer to the appropriate alignment.
+ unsigned Align = 16; // FIXME: GENERICIZE!!
+ // Get the ELF Section that this function belongs in.
+ ES = &EW.getSection(".text", ELFWriter::ELFSection::SHT_PROGBITS,
+ ELFWriter::ELFSection::SHF_EXECINSTR |
+ ELFWriter::ELFSection::SHF_ALLOC);
+ OutBuffer = &ES->SectionData;
+ cerr << "FIXME: This code needs to be updated for changes in the "
+ << "CodeEmitter interfaces. In particular, this should set "
+ << "BufferBegin/BufferEnd/CurBufferPtr, not deal with OutBuffer!";
+ abort();
+
+ // Upgrade the section alignment if required.
+ if (ES->Align < Align) ES->Align = Align;
+
+ // Add padding zeros to the end of the buffer to make sure that the
+ // function will start on the correct byte alignment within the section.
+ OutputBuffer OB(*OutBuffer,
+ TM.getTargetData()->getPointerSizeInBits() == 64,
+ TM.getTargetData()->isLittleEndian());
+ OB.align(Align);
+ FnStart = OutBuffer->size();
+}
+
+/// finishFunction - This callback is invoked after the function is completely
+/// finished.
+bool ELFCodeEmitter::finishFunction(MachineFunction &F) {
+ // We now know the size of the function, add a symbol to represent it.
+ ELFWriter::ELFSym FnSym(F.getFunction());
+
+ // Figure out the binding (linkage) of the symbol.
+ switch (F.getFunction()->getLinkage()) {
+ default:
+ // appending linkage is illegal for functions.
+ assert(0 && "Unknown linkage type!");
+ case GlobalValue::ExternalLinkage:
+ FnSym.SetBind(ELFWriter::ELFSym::STB_GLOBAL);
+ break;
+ case GlobalValue::LinkOnceLinkage:
+ case GlobalValue::WeakLinkage:
+ FnSym.SetBind(ELFWriter::ELFSym::STB_WEAK);
+ break;
+ case GlobalValue::InternalLinkage:
+ FnSym.SetBind(ELFWriter::ELFSym::STB_LOCAL);
+ break;
+ }
+
+ ES->Size = OutBuffer->size();
+
+ FnSym.SetType(ELFWriter::ELFSym::STT_FUNC);
+ FnSym.SectionIdx = ES->SectionIdx;
+ FnSym.Value = FnStart; // Value = Offset from start of Section.
+ FnSym.Size = OutBuffer->size()-FnStart;
+
+ // Finally, add it to the symtab.
+ EW.SymbolTable.push_back(FnSym);
+ return false;
+}
+
+//===----------------------------------------------------------------------===//
+// ELFWriter Implementation
+//===----------------------------------------------------------------------===//
+
+ELFWriter::ELFWriter(std::ostream &o, TargetMachine &tm)
+ : MachineFunctionPass((intptr_t)&ID), O(o), TM(tm) {
+ e_flags = 0; // e_flags defaults to 0, no flags.
+
+ is64Bit = TM.getTargetData()->getPointerSizeInBits() == 64;
+ isLittleEndian = TM.getTargetData()->isLittleEndian();
+
+ // Create the machine code emitter object for this target.
+ MCE = new ELFCodeEmitter(*this);
+ NumSections = 0;
+}
+
+ELFWriter::~ELFWriter() {
+ delete MCE;
+}
+
+// doInitialization - Emit the file header and all of the global variables for
+// the module to the ELF file.
+bool ELFWriter::doInitialization(Module &M) {
+ Mang = new Mangler(M);
+
+ // Local alias to shortenify coming code.
+ std::vector<unsigned char> &FH = FileHeader;
+ OutputBuffer FHOut(FH, is64Bit, isLittleEndian);
+
+ FHOut.outbyte(0x7F); // EI_MAG0
+ FHOut.outbyte('E'); // EI_MAG1
+ FHOut.outbyte('L'); // EI_MAG2
+ FHOut.outbyte('F'); // EI_MAG3
+ FHOut.outbyte(is64Bit ? 2 : 1); // EI_CLASS
+ FHOut.outbyte(isLittleEndian ? 1 : 2); // EI_DATA
+ FHOut.outbyte(1); // EI_VERSION
+ FH.resize(16); // EI_PAD up to 16 bytes.
+
+ // This should change for shared objects.
+ FHOut.outhalf(1); // e_type = ET_REL
+ FHOut.outword(TM.getELFWriterInfo()->getEMachine()); // target-defined
+ FHOut.outword(1); // e_version = 1
+ FHOut.outaddr(0); // e_entry = 0 -> no entry point in .o file
+ FHOut.outaddr(0); // e_phoff = 0 -> no program header for .o
+
+ ELFHeader_e_shoff_Offset = FH.size();
+ FHOut.outaddr(0); // e_shoff
+ FHOut.outword(e_flags); // e_flags = whatever the target wants
+
+ FHOut.outhalf(is64Bit ? 64 : 52); // e_ehsize = ELF header size
+ FHOut.outhalf(0); // e_phentsize = prog header entry size
+ FHOut.outhalf(0); // e_phnum = # prog header entries = 0
+ FHOut.outhalf(is64Bit ? 64 : 40); // e_shentsize = sect hdr entry size
+
+
+ ELFHeader_e_shnum_Offset = FH.size();
+ FHOut.outhalf(0); // e_shnum = # of section header ents
+ ELFHeader_e_shstrndx_Offset = FH.size();
+ FHOut.outhalf(0); // e_shstrndx = Section # of '.shstrtab'
+
+ // Add the null section, which is required to be first in the file.
+ getSection("", 0, 0);
+
+ // Start up the symbol table. The first entry in the symtab is the null
+ // entry.
+ SymbolTable.push_back(ELFSym(0));
+
+ return false;
+}
+
+void ELFWriter::EmitGlobal(GlobalVariable *GV) {
+ // If this is an external global, emit it now. TODO: Note that it would be
+ // better to ignore the symbol here and only add it to the symbol table if
+ // referenced.
+ if (!GV->hasInitializer()) {
+ ELFSym ExternalSym(GV);
+ ExternalSym.SetBind(ELFSym::STB_GLOBAL);
+ ExternalSym.SetType(ELFSym::STT_NOTYPE);
+ ExternalSym.SectionIdx = ELFSection::SHN_UNDEF;
+ SymbolTable.push_back(ExternalSym);
+ return;
+ }
+
+ const Type *GVType = (const Type*)GV->getType();
+ unsigned Align = TM.getTargetData()->getPrefTypeAlignment(GVType);
+ unsigned Size = TM.getTargetData()->getTypeSize(GVType);
+
+ // If this global has a zero initializer, it is part of the .bss or common
+ // section.
+ if (GV->getInitializer()->isNullValue()) {
+ // If this global is part of the common block, add it now. Variables are
+ // part of the common block if they are zero initialized and allowed to be
+ // merged with other symbols.
+ if (GV->hasLinkOnceLinkage() || GV->hasWeakLinkage()) {
+ ELFSym CommonSym(GV);
+ // Value for common symbols is the alignment required.
+ CommonSym.Value = Align;
+ CommonSym.Size = Size;
+ CommonSym.SetBind(ELFSym::STB_GLOBAL);
+ CommonSym.SetType(ELFSym::STT_OBJECT);
+ // TODO SOMEDAY: add ELF visibility.
+ CommonSym.SectionIdx = ELFSection::SHN_COMMON;
+ SymbolTable.push_back(CommonSym);
+ return;
+ }
+
+ // Otherwise, this symbol is part of the .bss section. Emit it now.
+
+ // Handle alignment. Ensure section is aligned at least as much as required
+ // by this symbol.
+ ELFSection &BSSSection = getBSSSection();
+ BSSSection.Align = std::max(BSSSection.Align, Align);
+
+ // Within the section, emit enough virtual padding to get us to an alignment
+ // boundary.
+ if (Align)
+ BSSSection.Size = (BSSSection.Size + Align - 1) & ~(Align-1);
+
+ ELFSym BSSSym(GV);
+ BSSSym.Value = BSSSection.Size;
+ BSSSym.Size = Size;
+ BSSSym.SetType(ELFSym::STT_OBJECT);
+
+ switch (GV->getLinkage()) {
+ default: // weak/linkonce handled above
+ assert(0 && "Unexpected linkage type!");
+ case GlobalValue::AppendingLinkage: // FIXME: This should be improved!
+ case GlobalValue::ExternalLinkage:
+ BSSSym.SetBind(ELFSym::STB_GLOBAL);
+ break;
+ case GlobalValue::InternalLinkage:
+ BSSSym.SetBind(ELFSym::STB_LOCAL);
+ break;
+ }
+
+ // Set the idx of the .bss section
+ BSSSym.SectionIdx = BSSSection.SectionIdx;
+ SymbolTable.push_back(BSSSym);
+
+ // Reserve space in the .bss section for this symbol.
+ BSSSection.Size += Size;
+ return;
+ }
+
+ // FIXME: handle .rodata
+ //assert(!GV->isConstant() && "unimp");
+
+ // FIXME: handle .data
+ //assert(0 && "unimp");
+}
+
+
+bool ELFWriter::runOnMachineFunction(MachineFunction &MF) {
+ // Nothing to do here, this is all done through the MCE object above.
+ return false;
+}
+
+/// doFinalization - Now that the module has been completely processed, emit
+/// the ELF file to 'O'.
+bool ELFWriter::doFinalization(Module &M) {
+ // Okay, the ELF header and .text sections have been completed, build the
+ // .data, .bss, and "common" sections next.
+ for (Module::global_iterator I = M.global_begin(), E = M.global_end();
+ I != E; ++I)
+ EmitGlobal(I);
+
+ // Emit the symbol table now, if non-empty.
+ EmitSymbolTable();
+
+ // FIXME: Emit the relocations now.
+
+ // Emit the string table for the sections in the ELF file we have.
+ EmitSectionTableStringTable();
+
+ // Emit the sections to the .o file, and emit the section table for the file.
+ OutputSectionsAndSectionTable();
+
+ // We are done with the abstract symbols.
+ SectionList.clear();
+ NumSections = 0;
+
+ // Release the name mangler object.
+ delete Mang; Mang = 0;
+ return false;
+}
+
+/// EmitSymbolTable - If the current symbol table is non-empty, emit the string
+/// table for it and then the symbol table itself.
+void ELFWriter::EmitSymbolTable() {
+ if (SymbolTable.size() == 1) return; // Only the null entry.
+
+ // FIXME: compact all local symbols to the start of the symtab.
+ unsigned FirstNonLocalSymbol = 1;
+
+ ELFSection &StrTab = getSection(".strtab", ELFSection::SHT_STRTAB, 0);
+ StrTab.Align = 1;
+
+ DataBuffer &StrTabBuf = StrTab.SectionData;
+ OutputBuffer StrTabOut(StrTabBuf, is64Bit, isLittleEndian);
+
+ // Set the zero'th symbol to a null byte, as required.
+ StrTabOut.outbyte(0);
+ SymbolTable[0].NameIdx = 0;
+ unsigned Index = 1;
+ for (unsigned i = 1, e = SymbolTable.size(); i != e; ++i) {
+ // Use the name mangler to uniquify the LLVM symbol.
+ std::string Name = Mang->getValueName(SymbolTable[i].GV);
+
+ if (Name.empty()) {
+ SymbolTable[i].NameIdx = 0;
+ } else {
+ SymbolTable[i].NameIdx = Index;
+
+ // Add the name to the output buffer, including the null terminator.
+ StrTabBuf.insert(StrTabBuf.end(), Name.begin(), Name.end());
+
+ // Add a null terminator.
+ StrTabBuf.push_back(0);
+
+ // Keep track of the number of bytes emitted to this section.
+ Index += Name.size()+1;
+ }
+ }
+ assert(Index == StrTabBuf.size());
+ StrTab.Size = Index;
+
+ // Now that we have emitted the string table and know the offset into the
+ // string table of each symbol, emit the symbol table itself.
+ ELFSection &SymTab = getSection(".symtab", ELFSection::SHT_SYMTAB, 0);
+ SymTab.Align = is64Bit ? 8 : 4;
+ SymTab.Link = SymTab.SectionIdx; // Section Index of .strtab.
+ SymTab.Info = FirstNonLocalSymbol; // First non-STB_LOCAL symbol.
+ SymTab.EntSize = 16; // Size of each symtab entry. FIXME: wrong for ELF64
+ DataBuffer &SymTabBuf = SymTab.SectionData;
+ OutputBuffer SymTabOut(SymTabBuf, is64Bit, isLittleEndian);
+
+ if (!is64Bit) { // 32-bit and 64-bit formats are shuffled a bit.
+ for (unsigned i = 0, e = SymbolTable.size(); i != e; ++i) {
+ ELFSym &Sym = SymbolTable[i];
+ SymTabOut.outword(Sym.NameIdx);
+ SymTabOut.outaddr32(Sym.Value);
+ SymTabOut.outword(Sym.Size);
+ SymTabOut.outbyte(Sym.Info);
+ SymTabOut.outbyte(Sym.Other);
+ SymTabOut.outhalf(Sym.SectionIdx);
+ }
+ } else {
+ for (unsigned i = 0, e = SymbolTable.size(); i != e; ++i) {
+ ELFSym &Sym = SymbolTable[i];
+ SymTabOut.outword(Sym.NameIdx);
+ SymTabOut.outbyte(Sym.Info);
+ SymTabOut.outbyte(Sym.Other);
+ SymTabOut.outhalf(Sym.SectionIdx);
+ SymTabOut.outaddr64(Sym.Value);
+ SymTabOut.outxword(Sym.Size);
+ }
+ }
+
+ SymTab.Size = SymTabBuf.size();
+}
+
+/// EmitSectionTableStringTable - This method adds and emits a section for the
+/// ELF Section Table string table: the string table that holds all of the
+/// section names.
+void ELFWriter::EmitSectionTableStringTable() {
+ // First step: add the section for the string table to the list of sections:
+ ELFSection &SHStrTab = getSection(".shstrtab", ELFSection::SHT_STRTAB, 0);
+
+ // Now that we know which section number is the .shstrtab section, update the
+ // e_shstrndx entry in the ELF header.
+ OutputBuffer FHOut(FileHeader, is64Bit, isLittleEndian);
+ FHOut.fixhalf(SHStrTab.SectionIdx, ELFHeader_e_shstrndx_Offset);
+
+ // Set the NameIdx of each section in the string table and emit the bytes for
+ // the string table.
+ unsigned Index = 0;
+ DataBuffer &Buf = SHStrTab.SectionData;
+
+ for (std::list<ELFSection>::iterator I = SectionList.begin(),
+ E = SectionList.end(); I != E; ++I) {
+ // Set the index into the table. Note if we have lots of entries with
+ // common suffixes, we could memoize them here if we cared.
+ I->NameIdx = Index;
+
+ // Add the name to the output buffer, including the null terminator.
+ Buf.insert(Buf.end(), I->Name.begin(), I->Name.end());
+
+ // Add a null terminator.
+ Buf.push_back(0);
+
+ // Keep track of the number of bytes emitted to this section.
+ Index += I->Name.size()+1;
+ }
+
+ // Set the size of .shstrtab now that we know what it is.
+ assert(Index == Buf.size());
+ SHStrTab.Size = Index;
+}
+
+/// OutputSectionsAndSectionTable - Now that we have constructed the file header
+/// and all of the sections, emit these to the ostream destination and emit the
+/// SectionTable.
+void ELFWriter::OutputSectionsAndSectionTable() {
+ // Pass #1: Compute the file offset for each section.
+ size_t FileOff = FileHeader.size(); // File header first.
+
+ // Emit all of the section data in order.
+ for (std::list<ELFSection>::iterator I = SectionList.begin(),
+ E = SectionList.end(); I != E; ++I) {
+ // Align FileOff to whatever the alignment restrictions of the section are.
+ if (I->Align)
+ FileOff = (FileOff+I->Align-1) & ~(I->Align-1);
+ I->Offset = FileOff;
+ FileOff += I->SectionData.size();
+ }
+
+ // Align Section Header.
+ unsigned TableAlign = is64Bit ? 8 : 4;
+ FileOff = (FileOff+TableAlign-1) & ~(TableAlign-1);
+
+ // Now that we know where all of the sections will be emitted, set the e_shnum
+ // entry in the ELF header.
+ OutputBuffer FHOut(FileHeader, is64Bit, isLittleEndian);
+ FHOut.fixhalf(NumSections, ELFHeader_e_shnum_Offset);
+
+ // Now that we know the offset in the file of the section table, update the
+ // e_shoff address in the ELF header.
+ FHOut.fixaddr(FileOff, ELFHeader_e_shoff_Offset);
+
+ // Now that we know all of the data in the file header, emit it and all of the
+ // sections!
+ O.write((char*)&FileHeader[0], FileHeader.size());
+ FileOff = FileHeader.size();
+ DataBuffer().swap(FileHeader);
+
+ DataBuffer Table;
+ OutputBuffer TableOut(Table, is64Bit, isLittleEndian);
+
+ // Emit all of the section data and build the section table itself.
+ while (!SectionList.empty()) {
+ const ELFSection &S = *SectionList.begin();
+
+ // Align FileOff to whatever the alignment restrictions of the section are.
+ if (S.Align)
+ for (size_t NewFileOff = (FileOff+S.Align-1) & ~(S.Align-1);
+ FileOff != NewFileOff; ++FileOff)
+ O.put((char)0xAB);
+ O.write((char*)&S.SectionData[0], S.SectionData.size());
+ FileOff += S.SectionData.size();
+
+ TableOut.outword(S.NameIdx); // sh_name - Symbol table name idx
+ TableOut.outword(S.Type); // sh_type - Section contents & semantics
+ TableOut.outword(S.Flags); // sh_flags - Section flags.
+ TableOut.outaddr(S.Addr); // sh_addr - The mem addr this section is in.
+ TableOut.outaddr(S.Offset); // sh_offset - Offset from the file start.
+ TableOut.outword(S.Size); // sh_size - The section size.
+ TableOut.outword(S.Link); // sh_link - Section header table index link.
+ TableOut.outword(S.Info); // sh_info - Auxillary information.
+ TableOut.outword(S.Align); // sh_addralign - Alignment of section.
+ TableOut.outword(S.EntSize); // sh_entsize - Size of entries in the section
+
+ SectionList.pop_front();
+ }
+
+ // Align output for the section table.
+ for (size_t NewFileOff = (FileOff+TableAlign-1) & ~(TableAlign-1);
+ FileOff != NewFileOff; ++FileOff)
+ O.put((char)0xAB);
+
+ // Emit the section table itself.
+ O.write((char*)&Table[0], Table.size());
+}
diff --git a/lib/CodeGen/ELFWriter.h b/lib/CodeGen/ELFWriter.h
new file mode 100644
index 0000000..f27d78f
--- /dev/null
+++ b/lib/CodeGen/ELFWriter.h
@@ -0,0 +1,228 @@
+//===-- ELFWriter.h - Target-independent ELF writer support -----*- C++ -*-===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file was developed by Chris Lattner and is distributed under the
+// University of Illinois Open Source License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines the ELFWriter class.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef ELFWRITER_H
+#define ELFWRITER_H
+
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include <list>
+
+namespace llvm {
+ class GlobalVariable;
+ class Mangler;
+ class MachineCodeEmitter;
+ class ELFCodeEmitter;
+
+ /// ELFWriter - This class implements the common target-independent code for
+ /// writing ELF files. Targets should derive a class from this to
+ /// parameterize the output format.
+ ///
+ class ELFWriter : public MachineFunctionPass {
+ friend class ELFCodeEmitter;
+ public:
+ static char ID;
+
+ MachineCodeEmitter &getMachineCodeEmitter() const {
+ return *(MachineCodeEmitter*)MCE;
+ }
+
+ ELFWriter(std::ostream &O, TargetMachine &TM);
+ ~ELFWriter();
+
+ typedef std::vector<unsigned char> DataBuffer;
+
+ protected:
+ /// Output stream to send the resultant object file to.
+ ///
+ std::ostream &O;
+
+ /// Target machine description.
+ ///
+ TargetMachine &TM;
+
+ /// Mang - The object used to perform name mangling for this module.
+ ///
+ Mangler *Mang;
+
+ /// MCE - The MachineCodeEmitter object that we are exposing to emit machine
+ /// code for functions to the .o file.
+ ELFCodeEmitter *MCE;
+
+ //===------------------------------------------------------------------===//
+ // Properties to be set by the derived class ctor, used to configure the
+ // ELFWriter.
+
+ // e_machine - This field is the target specific value to emit as the
+ // e_machine member of the ELF header.
+ unsigned short e_machine;
+
+ // e_flags - The machine flags for the target. This defaults to zero.
+ unsigned e_flags;
+
+ //===------------------------------------------------------------------===//
+ // Properties inferred automatically from the target machine.
+ //
+
+ /// is64Bit/isLittleEndian - This information is inferred from the target
+ /// machine directly, indicating whether to emit a 32- or 64-bit ELF file.
+ bool is64Bit, isLittleEndian;
+
+ /// doInitialization - Emit the file header and all of the global variables
+ /// for the module to the ELF file.
+ bool doInitialization(Module &M);
+
+ bool runOnMachineFunction(MachineFunction &MF);
+
+
+ /// doFinalization - Now that the module has been completely processed, emit
+ /// the ELF file to 'O'.
+ bool doFinalization(Module &M);
+
+ private:
+ // The buffer we accumulate the file header into. Note that this should be
+ // changed into something much more efficient later (and the bitcode writer
+ // as well!).
+ DataBuffer FileHeader;
+
+ /// ELFSection - This struct contains information about each section that is
+ /// emitted to the file. This is eventually turned into the section header
+ /// table at the end of the file.
+ struct ELFSection {
+ std::string Name; // Name of the section.
+ unsigned NameIdx; // Index in .shstrtab of name, once emitted.
+ unsigned Type;
+ unsigned Flags;
+ uint64_t Addr;
+ unsigned Offset;
+ unsigned Size;
+ unsigned Link;
+ unsigned Info;
+ unsigned Align;
+ unsigned EntSize;
+
+ /// SectionIdx - The number of the section in the Section Table.
+ ///
+ unsigned short SectionIdx;
+
+ /// SectionData - The actual data for this section which we are building
+ /// up for emission to the file.
+ DataBuffer SectionData;
+
+ enum { SHT_NULL = 0, SHT_PROGBITS = 1, SHT_SYMTAB = 2, SHT_STRTAB = 3,
+ SHT_RELA = 4, SHT_HASH = 5, SHT_DYNAMIC = 6, SHT_NOTE = 7,
+ SHT_NOBITS = 8, SHT_REL = 9, SHT_SHLIB = 10, SHT_DYNSYM = 11 };
+ enum { SHN_UNDEF = 0, SHN_ABS = 0xFFF1, SHN_COMMON = 0xFFF2 };
+ enum { // SHF - ELF Section Header Flags
+ SHF_WRITE = 1 << 0, // Writable
+ SHF_ALLOC = 1 << 1, // Mapped into the process addr space
+ SHF_EXECINSTR = 1 << 2, // Executable
+ SHF_MERGE = 1 << 4, // Might be merged if equal
+ SHF_STRINGS = 1 << 5, // Contains null-terminated strings
+ SHF_INFO_LINK = 1 << 6, // 'sh_info' contains SHT index
+ SHF_LINK_ORDER = 1 << 7, // Preserve order after combining
+ SHF_OS_NONCONFORMING = 1 << 8, // nonstandard OS support required
+ SHF_GROUP = 1 << 9, // Section is a member of a group
+ SHF_TLS = 1 << 10 // Section holds thread-local data
+ };
+
+ ELFSection(const std::string &name)
+ : Name(name), Type(0), Flags(0), Addr(0), Offset(0), Size(0),
+ Link(0), Info(0), Align(0), EntSize(0) {
+ }
+ };
+
+ /// SectionList - This is the list of sections that we have emitted to the
+ /// file. Once the file has been completely built, the section header table
+ /// is constructed from this info.
+ std::list<ELFSection> SectionList;
+ unsigned NumSections; // Always = SectionList.size()
+
+ /// SectionLookup - This is a mapping from section name to section number in
+ /// the SectionList.
+ std::map<std::string, ELFSection*> SectionLookup;
+
+ /// getSection - Return the section with the specified name, creating a new
+ /// section if one does not already exist.
+ ELFSection &getSection(const std::string &Name,
+ unsigned Type, unsigned Flags = 0) {
+ ELFSection *&SN = SectionLookup[Name];
+ if (SN) return *SN;
+
+ SectionList.push_back(Name);
+ SN = &SectionList.back();
+ SN->SectionIdx = NumSections++;
+ SN->Type = Type;
+ SN->Flags = Flags;
+ return *SN;
+ }
+
+ ELFSection &getDataSection() {
+ return getSection(".data", ELFSection::SHT_PROGBITS,
+ ELFSection::SHF_WRITE | ELFSection::SHF_ALLOC);
+ }
+ ELFSection &getBSSSection() {
+ return getSection(".bss", ELFSection::SHT_NOBITS,
+ ELFSection::SHF_WRITE | ELFSection::SHF_ALLOC);
+ }
+
+ /// ELFSym - This struct contains information about each symbol that is
+ /// added to logical symbol table for the module. This is eventually
+ /// turned into a real symbol table in the file.
+ struct ELFSym {
+ const GlobalValue *GV; // The global value this corresponds to.
+ unsigned NameIdx; // Index in .strtab of name, once emitted.
+ uint64_t Value;
+ unsigned Size;
+ unsigned char Info;
+ unsigned char Other;
+ unsigned short SectionIdx;
+
+ enum { STB_LOCAL = 0, STB_GLOBAL = 1, STB_WEAK = 2 };
+ enum { STT_NOTYPE = 0, STT_OBJECT = 1, STT_FUNC = 2, STT_SECTION = 3,
+ STT_FILE = 4 };
+ ELFSym(const GlobalValue *gv) : GV(gv), Value(0), Size(0), Info(0),
+ Other(0), SectionIdx(0) {}
+
+ void SetBind(unsigned X) {
+ assert(X == (X & 0xF) && "Bind value out of range!");
+ Info = (Info & 0x0F) | (X << 4);
+ }
+ void SetType(unsigned X) {
+ assert(X == (X & 0xF) && "Type value out of range!");
+ Info = (Info & 0xF0) | X;
+ }
+ };
+
+ /// SymbolTable - This is the list of symbols we have emitted to the file.
+ /// This actually gets rearranged before emission to the file (to put the
+ /// local symbols first in the list).
+ std::vector<ELFSym> SymbolTable;
+
+ // As we complete the ELF file, we need to update fields in the ELF header
+ // (e.g. the location of the section table). These members keep track of
+ // the offset in ELFHeader of these various pieces to update and other
+ // locations in the file.
+ unsigned ELFHeader_e_shoff_Offset; // e_shoff in ELF header.
+ unsigned ELFHeader_e_shstrndx_Offset; // e_shstrndx in ELF header.
+ unsigned ELFHeader_e_shnum_Offset; // e_shnum in ELF header.
+ private:
+ void EmitGlobal(GlobalVariable *GV);
+
+ void EmitSymbolTable();
+
+ void EmitSectionTableStringTable();
+ void OutputSectionsAndSectionTable();
+ };
+}
+
+#endif
diff --git a/lib/CodeGen/IfConversion.cpp b/lib/CodeGen/IfConversion.cpp
new file mode 100644
index 0000000..3bddc77
--- /dev/null
+++ b/lib/CodeGen/IfConversion.cpp
@@ -0,0 +1,1226 @@
+//===-- IfConversion.cpp - Machine code if conversion pass. ---------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file was developed by the Evan Cheng and is distributed under
+// the University of Illinois Open Source License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements the machine instruction level if-conversion pass.
+//
+//===----------------------------------------------------------------------===//
+
+#define DEBUG_TYPE "ifcvt"
+#include "llvm/Function.h"
+#include "llvm/CodeGen/Passes.h"
+#include "llvm/CodeGen/MachineModuleInfo.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/Target/TargetInstrInfo.h"
+#include "llvm/Target/TargetLowering.h"
+#include "llvm/Target/TargetMachine.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/ADT/DepthFirstIterator.h"
+#include "llvm/ADT/Statistic.h"
+#include "llvm/ADT/STLExtras.h"
+using namespace llvm;
+
+namespace {
+ // Hidden options for help debugging.
+ cl::opt<int> IfCvtFnStart("ifcvt-fn-start", cl::init(-1), cl::Hidden);
+ cl::opt<int> IfCvtFnStop("ifcvt-fn-stop", cl::init(-1), cl::Hidden);
+ cl::opt<int> IfCvtLimit("ifcvt-limit", cl::init(-1), cl::Hidden);
+ cl::opt<bool> DisableSimple("disable-ifcvt-simple",
+ cl::init(false), cl::Hidden);
+ cl::opt<bool> DisableSimpleF("disable-ifcvt-simple-false",
+ cl::init(false), cl::Hidden);
+ cl::opt<bool> DisableTriangle("disable-ifcvt-triangle",
+ cl::init(false), cl::Hidden);
+ cl::opt<bool> DisableTriangleR("disable-ifcvt-triangle-rev",
+ cl::init(false), cl::Hidden);
+ cl::opt<bool> DisableTriangleF("disable-ifcvt-triangle-false",
+ cl::init(false), cl::Hidden);
+ cl::opt<bool> DisableTriangleFR("disable-ifcvt-triangle-false-rev",
+ cl::init(false), cl::Hidden);
+ cl::opt<bool> DisableDiamond("disable-ifcvt-diamond",
+ cl::init(false), cl::Hidden);
+}
+
+STATISTIC(NumSimple, "Number of simple if-conversions performed");
+STATISTIC(NumSimpleFalse, "Number of simple (F) if-conversions performed");
+STATISTIC(NumTriangle, "Number of triangle if-conversions performed");
+STATISTIC(NumTriangleRev, "Number of triangle (R) if-conversions performed");
+STATISTIC(NumTriangleFalse,"Number of triangle (F) if-conversions performed");
+STATISTIC(NumTriangleFRev, "Number of triangle (F/R) if-conversions performed");
+STATISTIC(NumDiamonds, "Number of diamond if-conversions performed");
+STATISTIC(NumIfConvBBs, "Number of if-converted blocks");
+STATISTIC(NumDupBBs, "Number of duplicated blocks");
+
+namespace {
+ class IfConverter : public MachineFunctionPass {
+ enum IfcvtKind {
+ ICNotClassfied, // BB data valid, but not classified.
+ ICSimpleFalse, // Same as ICSimple, but on the false path.
+ ICSimple, // BB is entry of an one split, no rejoin sub-CFG.
+ ICTriangleFRev, // Same as ICTriangleFalse, but false path rev condition.
+ ICTriangleRev, // Same as ICTriangle, but true path rev condition.
+ ICTriangleFalse, // Same as ICTriangle, but on the false path.
+ ICTriangle, // BB is entry of a triangle sub-CFG.
+ ICDiamond // BB is entry of a diamond sub-CFG.
+ };
+
+ /// BBInfo - One per MachineBasicBlock, this is used to cache the result
+ /// if-conversion feasibility analysis. This includes results from
+ /// TargetInstrInfo::AnalyzeBranch() (i.e. TBB, FBB, and Cond), and its
+ /// classification, and common tail block of its successors (if it's a
+ /// diamond shape), its size, whether it's predicable, and whether any
+ /// instruction can clobber the 'would-be' predicate.
+ ///
+ /// IsDone - True if BB is not to be considered for ifcvt.
+ /// IsBeingAnalyzed - True if BB is currently being analyzed.
+ /// IsAnalyzed - True if BB has been analyzed (info is still valid).
+ /// IsEnqueued - True if BB has been enqueued to be ifcvt'ed.
+ /// IsBrAnalyzable - True if AnalyzeBranch() returns false.
+ /// HasFallThrough - True if BB may fallthrough to the following BB.
+ /// IsUnpredicable - True if BB is known to be unpredicable.
+ /// ClobbersPred - True if BB could modify predicates (e.g. has
+ /// cmp, call, etc.)
+ /// NonPredSize - Number of non-predicated instructions.
+ /// BB - Corresponding MachineBasicBlock.
+ /// TrueBB / FalseBB- See AnalyzeBranch().
+ /// BrCond - Conditions for end of block conditional branches.
+ /// Predicate - Predicate used in the BB.
+ struct BBInfo {
+ bool IsDone : 1;
+ bool IsBeingAnalyzed : 1;
+ bool IsAnalyzed : 1;
+ bool IsEnqueued : 1;
+ bool IsBrAnalyzable : 1;
+ bool HasFallThrough : 1;
+ bool IsUnpredicable : 1;
+ bool CannotBeCopied : 1;
+ bool ClobbersPred : 1;
+ unsigned NonPredSize;
+ MachineBasicBlock *BB;
+ MachineBasicBlock *TrueBB;
+ MachineBasicBlock *FalseBB;
+ std::vector<MachineOperand> BrCond;
+ std::vector<MachineOperand> Predicate;
+ BBInfo() : IsDone(false), IsBeingAnalyzed(false),
+ IsAnalyzed(false), IsEnqueued(false), IsBrAnalyzable(false),
+ HasFallThrough(false), IsUnpredicable(false),
+ CannotBeCopied(false), ClobbersPred(false), NonPredSize(0),
+ BB(0), TrueBB(0), FalseBB(0) {}
+ };
+
+ /// IfcvtToken - Record information about pending if-conversions to attemp:
+ /// BBI - Corresponding BBInfo.
+ /// Kind - Type of block. See IfcvtKind.
+ /// NeedSubsumsion - True if the to be predicated BB has already been
+ /// predicated.
+ /// NumDups - Number of instructions that would be duplicated due
+ /// to this if-conversion. (For diamonds, the number of
+ /// identical instructions at the beginnings of both
+ /// paths).
+ /// NumDups2 - For diamonds, the number of identical instructions
+ /// at the ends of both paths.
+ struct IfcvtToken {
+ BBInfo &BBI;
+ IfcvtKind Kind;
+ bool NeedSubsumsion;
+ unsigned NumDups;
+ unsigned NumDups2;
+ IfcvtToken(BBInfo &b, IfcvtKind k, bool s, unsigned d, unsigned d2 = 0)
+ : BBI(b), Kind(k), NeedSubsumsion(s), NumDups(d), NumDups2(d2) {}
+ };
+
+ /// Roots - Basic blocks that do not have successors. These are the starting
+ /// points of Graph traversal.
+ std::vector<MachineBasicBlock*> Roots;
+
+ /// BBAnalysis - Results of if-conversion feasibility analysis indexed by
+ /// basic block number.
+ std::vector<BBInfo> BBAnalysis;
+
+ const TargetLowering *TLI;
+ const TargetInstrInfo *TII;
+ bool MadeChange;
+ public:
+ static char ID;
+ IfConverter() : MachineFunctionPass((intptr_t)&ID) {}
+
+ virtual bool runOnMachineFunction(MachineFunction &MF);
+ virtual const char *getPassName() const { return "If converter"; }
+
+ private:
+ bool ReverseBranchCondition(BBInfo &BBI);
+ bool ValidSimple(BBInfo &TrueBBI, unsigned &Dups) const;
+ bool ValidTriangle(BBInfo &TrueBBI, BBInfo &FalseBBI,
+ bool FalseBranch, unsigned &Dups) const;
+ bool ValidDiamond(BBInfo &TrueBBI, BBInfo &FalseBBI,
+ unsigned &Dups1, unsigned &Dups2) const;
+ void ScanInstructions(BBInfo &BBI);
+ BBInfo &AnalyzeBlock(MachineBasicBlock *BB,
+ std::vector<IfcvtToken*> &Tokens);
+ bool FeasibilityAnalysis(BBInfo &BBI, std::vector<MachineOperand> &Cond,
+ bool isTriangle = false, bool RevBranch = false);
+ bool AnalyzeBlocks(MachineFunction &MF,
+ std::vector<IfcvtToken*> &Tokens);
+ void InvalidatePreds(MachineBasicBlock *BB);
+ void RemoveExtraEdges(BBInfo &BBI);
+ bool IfConvertSimple(BBInfo &BBI, IfcvtKind Kind);
+ bool IfConvertTriangle(BBInfo &BBI, IfcvtKind Kind);
+ bool IfConvertDiamond(BBInfo &BBI, IfcvtKind Kind,
+ unsigned NumDups1, unsigned NumDups2);
+ void PredicateBlock(BBInfo &BBI,
+ MachineBasicBlock::iterator E,
+ std::vector<MachineOperand> &Cond);
+ void CopyAndPredicateBlock(BBInfo &ToBBI, BBInfo &FromBBI,
+ std::vector<MachineOperand> &Cond,
+ bool IgnoreBr = false);
+ void MergeBlocks(BBInfo &ToBBI, BBInfo &FromBBI);
+
+ bool MeetIfcvtSizeLimit(unsigned Size) const {
+ return Size > 0 && Size <= TLI->getIfCvtBlockSizeLimit();
+ }
+
+ // blockAlwaysFallThrough - Block ends without a terminator.
+ bool blockAlwaysFallThrough(BBInfo &BBI) const {
+ return BBI.IsBrAnalyzable && BBI.TrueBB == NULL;
+ }
+
+ // IfcvtTokenCmp - Used to sort if-conversion candidates.
+ static bool IfcvtTokenCmp(IfcvtToken *C1, IfcvtToken *C2) {
+ int Incr1 = (C1->Kind == ICDiamond)
+ ? -(int)(C1->NumDups + C1->NumDups2) : (int)C1->NumDups;
+ int Incr2 = (C2->Kind == ICDiamond)
+ ? -(int)(C2->NumDups + C2->NumDups2) : (int)C2->NumDups;
+ if (Incr1 > Incr2)
+ return true;
+ else if (Incr1 == Incr2) {
+ // Favors subsumsion.
+ if (C1->NeedSubsumsion == false && C2->NeedSubsumsion == true)
+ return true;
+ else if (C1->NeedSubsumsion == C2->NeedSubsumsion) {
+ // Favors diamond over triangle, etc.
+ if ((unsigned)C1->Kind < (unsigned)C2->Kind)
+ return true;
+ else if (C1->Kind == C2->Kind)
+ return C1->BBI.BB->getNumber() < C2->BBI.BB->getNumber();
+ }
+ }
+ return false;
+ }
+ };
+
+ char IfConverter::ID = 0;
+}
+
+FunctionPass *llvm::createIfConverterPass() { return new IfConverter(); }
+
+bool IfConverter::runOnMachineFunction(MachineFunction &MF) {
+ TLI = MF.getTarget().getTargetLowering();
+ TII = MF.getTarget().getInstrInfo();
+ if (!TII) return false;
+
+ static int FnNum = -1;
+ DOUT << "\nIfcvt: function (" << ++FnNum << ") \'"
+ << MF.getFunction()->getName() << "\'";
+
+ if (FnNum < IfCvtFnStart || (IfCvtFnStop != -1 && FnNum > IfCvtFnStop)) {
+ DOUT << " skipped\n";
+ return false;
+ }
+ DOUT << "\n";
+
+ MF.RenumberBlocks();
+ BBAnalysis.resize(MF.getNumBlockIDs());
+
+ // Look for root nodes, i.e. blocks without successors.
+ for (MachineFunction::iterator I = MF.begin(), E = MF.end(); I != E; ++I)
+ if (I->succ_size() == 0)
+ Roots.push_back(I);
+
+ std::vector<IfcvtToken*> Tokens;
+ MadeChange = false;
+ unsigned NumIfCvts = NumSimple + NumSimpleFalse + NumTriangle +
+ NumTriangleRev + NumTriangleFalse + NumTriangleFRev + NumDiamonds;
+ while (IfCvtLimit == -1 || (int)NumIfCvts < IfCvtLimit) {
+ // Do an intial analysis for each basic block and finding all the potential
+ // candidates to perform if-convesion.
+ bool Change = AnalyzeBlocks(MF, Tokens);
+ while (!Tokens.empty()) {
+ IfcvtToken *Token = Tokens.back();
+ Tokens.pop_back();
+ BBInfo &BBI = Token->BBI;
+ IfcvtKind Kind = Token->Kind;
+
+ // If the block has been evicted out of the queue or it has already been
+ // marked dead (due to it being predicated), then skip it.
+ if (BBI.IsDone)
+ BBI.IsEnqueued = false;
+ if (!BBI.IsEnqueued)
+ continue;
+
+ BBI.IsEnqueued = false;
+
+ bool RetVal = false;
+ switch (Kind) {
+ default: assert(false && "Unexpected!");
+ break;
+ case ICSimple:
+ case ICSimpleFalse: {
+ bool isFalse = Kind == ICSimpleFalse;
+ if ((isFalse && DisableSimpleF) || (!isFalse && DisableSimple)) break;
+ DOUT << "Ifcvt (Simple" << (Kind == ICSimpleFalse ? " false" :"")
+ << "): BB#" << BBI.BB->getNumber() << " ("
+ << ((Kind == ICSimpleFalse)
+ ? BBI.FalseBB->getNumber()
+ : BBI.TrueBB->getNumber()) << ") ";
+ RetVal = IfConvertSimple(BBI, Kind);
+ DOUT << (RetVal ? "succeeded!" : "failed!") << "\n";
+ if (RetVal)
+ if (isFalse) NumSimpleFalse++;
+ else NumSimple++;
+ break;
+ }
+ case ICTriangle:
+ case ICTriangleRev:
+ case ICTriangleFalse:
+ case ICTriangleFRev: {
+ bool isFalse = Kind == ICTriangleFalse;
+ bool isRev = (Kind == ICTriangleRev || Kind == ICTriangleFRev);
+ if (DisableTriangle && !isFalse && !isRev) break;
+ if (DisableTriangleR && !isFalse && isRev) break;
+ if (DisableTriangleF && isFalse && !isRev) break;
+ if (DisableTriangleFR && isFalse && isRev) break;
+ DOUT << "Ifcvt (Triangle";
+ if (isFalse)
+ DOUT << " false";
+ if (isRev)
+ DOUT << " rev";
+ DOUT << "): BB#" << BBI.BB->getNumber() << " (T:"
+ << BBI.TrueBB->getNumber() << ",F:"
+ << BBI.FalseBB->getNumber() << ") ";
+ RetVal = IfConvertTriangle(BBI, Kind);
+ DOUT << (RetVal ? "succeeded!" : "failed!") << "\n";
+ if (RetVal) {
+ if (isFalse) {
+ if (isRev) NumTriangleFRev++;
+ else NumTriangleFalse++;
+ } else {
+ if (isRev) NumTriangleRev++;
+ else NumTriangle++;
+ }
+ }
+ break;
+ }
+ case ICDiamond: {
+ if (DisableDiamond) break;
+ DOUT << "Ifcvt (Diamond): BB#" << BBI.BB->getNumber() << " (T:"
+ << BBI.TrueBB->getNumber() << ",F:"
+ << BBI.FalseBB->getNumber() << ") ";
+ RetVal = IfConvertDiamond(BBI, Kind, Token->NumDups, Token->NumDups2);
+ DOUT << (RetVal ? "succeeded!" : "failed!") << "\n";
+ if (RetVal) NumDiamonds++;
+ break;
+ }
+ }
+
+ Change |= RetVal;
+
+ NumIfCvts = NumSimple + NumSimpleFalse + NumTriangle + NumTriangleRev +
+ NumTriangleFalse + NumTriangleFRev + NumDiamonds;
+ if (IfCvtLimit != -1 && (int)NumIfCvts >= IfCvtLimit)
+ break;
+ }
+
+ if (!Change)
+ break;
+ MadeChange |= Change;
+ }
+
+ // Delete tokens in case of early exit.
+ while (!Tokens.empty()) {
+ IfcvtToken *Token = Tokens.back();
+ Tokens.pop_back();
+ delete Token;
+ }
+
+ Tokens.clear();
+ Roots.clear();
+ BBAnalysis.clear();
+
+ return MadeChange;
+}
+
+/// findFalseBlock - BB has a fallthrough. Find its 'false' successor given
+/// its 'true' successor.
+static MachineBasicBlock *findFalseBlock(MachineBasicBlock *BB,
+ MachineBasicBlock *TrueBB) {
+ for (MachineBasicBlock::succ_iterator SI = BB->succ_begin(),
+ E = BB->succ_end(); SI != E; ++SI) {
+ MachineBasicBlock *SuccBB = *SI;
+ if (SuccBB != TrueBB)
+ return SuccBB;
+ }
+ return NULL;
+}
+
+/// ReverseBranchCondition - Reverse the condition of the end of the block
+/// branchs. Swap block's 'true' and 'false' successors.
+bool IfConverter::ReverseBranchCondition(BBInfo &BBI) {
+ if (!TII->ReverseBranchCondition(BBI.BrCond)) {
+ TII->RemoveBranch(*BBI.BB);
+ TII->InsertBranch(*BBI.BB, BBI.FalseBB, BBI.TrueBB, BBI.BrCond);
+ std::swap(BBI.TrueBB, BBI.FalseBB);
+ return true;
+ }
+ return false;
+}
+
+/// getNextBlock - Returns the next block in the function blocks ordering. If
+/// it is the end, returns NULL.
+static inline MachineBasicBlock *getNextBlock(MachineBasicBlock *BB) {
+ MachineFunction::iterator I = BB;
+ MachineFunction::iterator E = BB->getParent()->end();
+ if (++I == E)
+ return NULL;
+ return I;
+}
+
+/// ValidSimple - Returns true if the 'true' block (along with its
+/// predecessor) forms a valid simple shape for ifcvt. It also returns the
+/// number of instructions that the ifcvt would need to duplicate if performed
+/// in Dups.
+bool IfConverter::ValidSimple(BBInfo &TrueBBI, unsigned &Dups) const {
+ Dups = 0;
+ if (TrueBBI.IsBeingAnalyzed || TrueBBI.IsDone)
+ return false;
+
+ if (TrueBBI.IsBrAnalyzable)
+ return false;
+
+ if (TrueBBI.BB->pred_size() > 1) {
+ if (TrueBBI.CannotBeCopied ||
+ TrueBBI.NonPredSize > TLI->getIfCvtDupBlockSizeLimit())
+ return false;
+ Dups = TrueBBI.NonPredSize;
+ }
+
+ return true;
+}
+
+/// ValidTriangle - Returns true if the 'true' and 'false' blocks (along
+/// with their common predecessor) forms a valid triangle shape for ifcvt.
+/// If 'FalseBranch' is true, it checks if 'true' block's false branch
+/// branches to the false branch rather than the other way around. It also
+/// returns the number of instructions that the ifcvt would need to duplicate
+/// if performed in 'Dups'.
+bool IfConverter::ValidTriangle(BBInfo &TrueBBI, BBInfo &FalseBBI,
+ bool FalseBranch, unsigned &Dups) const {
+ Dups = 0;
+ if (TrueBBI.IsBeingAnalyzed || TrueBBI.IsDone)
+ return false;
+
+ if (TrueBBI.BB->pred_size() > 1) {
+ if (TrueBBI.CannotBeCopied)
+ return false;
+
+ unsigned Size = TrueBBI.NonPredSize;
+ if (TrueBBI.IsBrAnalyzable) {
+ if (TrueBBI.TrueBB && TrueBBI.BrCond.size() == 0)
+ // End with an unconditional branch. It will be removed.
+ --Size;
+ else {
+ MachineBasicBlock *FExit = FalseBranch
+ ? TrueBBI.TrueBB : TrueBBI.FalseBB;
+ if (FExit)
+ // Require a conditional branch
+ ++Size;
+ }
+ }
+ if (Size > TLI->getIfCvtDupBlockSizeLimit())
+ return false;
+ Dups = Size;
+ }
+
+ MachineBasicBlock *TExit = FalseBranch ? TrueBBI.FalseBB : TrueBBI.TrueBB;
+ if (!TExit && blockAlwaysFallThrough(TrueBBI)) {
+ MachineFunction::iterator I = TrueBBI.BB;
+ if (++I == TrueBBI.BB->getParent()->end())
+ return false;
+ TExit = I;
+ }
+ return TExit && TExit == FalseBBI.BB;
+}
+
+static
+MachineBasicBlock::iterator firstNonBranchInst(MachineBasicBlock *BB,
+ const TargetInstrInfo *TII) {
+ MachineBasicBlock::iterator I = BB->end();
+ while (I != BB->begin()) {
+ --I;
+ const TargetInstrDescriptor *TID = I->getInstrDescriptor();
+ if ((TID->Flags & M_BRANCH_FLAG) == 0)
+ break;
+ }
+ return I;
+}
+
+/// ValidDiamond - Returns true if the 'true' and 'false' blocks (along
+/// with their common predecessor) forms a valid diamond shape for ifcvt.
+bool IfConverter::ValidDiamond(BBInfo &TrueBBI, BBInfo &FalseBBI,
+ unsigned &Dups1, unsigned &Dups2) const {
+ Dups1 = Dups2 = 0;
+ if (TrueBBI.IsBeingAnalyzed || TrueBBI.IsDone ||
+ FalseBBI.IsBeingAnalyzed || FalseBBI.IsDone)
+ return false;
+
+ MachineBasicBlock *TT = TrueBBI.TrueBB;
+ MachineBasicBlock *FT = FalseBBI.TrueBB;
+
+ if (!TT && blockAlwaysFallThrough(TrueBBI))
+ TT = getNextBlock(TrueBBI.BB);
+ if (!FT && blockAlwaysFallThrough(FalseBBI))
+ FT = getNextBlock(FalseBBI.BB);
+ if (TT != FT)
+ return false;
+ if (TT == NULL && (TrueBBI.IsBrAnalyzable || FalseBBI.IsBrAnalyzable))
+ return false;
+ if (TrueBBI.BB->pred_size() > 1 || FalseBBI.BB->pred_size() > 1)
+ return false;
+
+ // FIXME: Allow true block to have an early exit?
+ if (TrueBBI.FalseBB || FalseBBI.FalseBB ||
+ (TrueBBI.ClobbersPred && FalseBBI.ClobbersPred))
+ return false;
+
+ MachineBasicBlock::iterator TI = TrueBBI.BB->begin();
+ MachineBasicBlock::iterator FI = FalseBBI.BB->begin();
+ while (TI != TrueBBI.BB->end() && FI != FalseBBI.BB->end()) {
+ if (!TI->isIdenticalTo(FI))
+ break;
+ ++Dups1;
+ ++TI;
+ ++FI;
+ }
+
+ TI = firstNonBranchInst(TrueBBI.BB, TII);
+ FI = firstNonBranchInst(FalseBBI.BB, TII);
+ while (TI != TrueBBI.BB->begin() && FI != FalseBBI.BB->begin()) {
+ if (!TI->isIdenticalTo(FI))
+ break;
+ ++Dups2;
+ --TI;
+ --FI;
+ }
+
+ return true;
+}
+
+/// ScanInstructions - Scan all the instructions in the block to determine if
+/// the block is predicable. In most cases, that means all the instructions
+/// in the block has M_PREDICABLE flag. Also checks if the block contains any
+/// instruction which can clobber a predicate (e.g. condition code register).
+/// If so, the block is not predicable unless it's the last instruction.
+void IfConverter::ScanInstructions(BBInfo &BBI) {
+ if (BBI.IsDone)
+ return;
+
+ bool AlreadyPredicated = BBI.Predicate.size() > 0;
+ // First analyze the end of BB branches.
+ BBI.TrueBB = BBI.FalseBB = NULL;
+ BBI.BrCond.clear();
+ BBI.IsBrAnalyzable =
+ !TII->AnalyzeBranch(*BBI.BB, BBI.TrueBB, BBI.FalseBB, BBI.BrCond);
+ BBI.HasFallThrough = BBI.IsBrAnalyzable && BBI.FalseBB == NULL;
+
+ if (BBI.BrCond.size()) {
+ // No false branch. This BB must end with a conditional branch and a
+ // fallthrough.
+ if (!BBI.FalseBB)
+ BBI.FalseBB = findFalseBlock(BBI.BB, BBI.TrueBB);
+ assert(BBI.FalseBB && "Expected to find the fallthrough block!");
+ }
+
+ // Then scan all the instructions.
+ BBI.NonPredSize = 0;
+ BBI.ClobbersPred = false;
+ bool SeenCondBr = false;
+ for (MachineBasicBlock::iterator I = BBI.BB->begin(), E = BBI.BB->end();
+ I != E; ++I) {
+ const TargetInstrDescriptor *TID = I->getInstrDescriptor();
+ if ((TID->Flags & M_NOT_DUPLICABLE) != 0)
+ BBI.CannotBeCopied = true;
+
+ bool isPredicated = TII->isPredicated(I);
+ bool isCondBr = BBI.IsBrAnalyzable &&
+ (TID->Flags & M_BRANCH_FLAG) != 0 && (TID->Flags & M_BARRIER_FLAG) == 0;
+
+ if (!isCondBr) {
+ if (!isPredicated)
+ BBI.NonPredSize++;
+ else if (!AlreadyPredicated) {
+ // FIXME: This instruction is already predicated before the
+ // if-conversion pass. It's probably something like a conditional move.
+ // Mark this block unpredicable for now.
+ BBI.IsUnpredicable = true;
+ return;
+ }
+
+ }
+
+ if (BBI.ClobbersPred && !isPredicated) {
+ // Predicate modification instruction should end the block (except for
+ // already predicated instructions and end of block branches).
+ if (isCondBr) {
+ SeenCondBr = true;
+
+ // Conditional branches is not predicable. But it may be eliminated.
+ continue;
+ }
+
+ // Predicate may have been modified, the subsequent (currently)
+ // unpredicated instructions cannot be correctly predicated.
+ BBI.IsUnpredicable = true;
+ return;
+ }
+
+ // FIXME: Make use of PredDefs? e.g. ADDC, SUBC sets predicates but are
+ // still potentially predicable.
+ std::vector<MachineOperand> PredDefs;
+ if (TII->DefinesPredicate(I, PredDefs))
+ BBI.ClobbersPred = true;
+
+ if ((TID->Flags & M_PREDICABLE) == 0) {
+ BBI.IsUnpredicable = true;
+ return;
+ }
+ }
+}
+
+/// FeasibilityAnalysis - Determine if the block is a suitable candidate to be
+/// predicated by the specified predicate.
+bool IfConverter::FeasibilityAnalysis(BBInfo &BBI,
+ std::vector<MachineOperand> &Pred,
+ bool isTriangle, bool RevBranch) {
+ // If the block is dead or unpredicable, then it cannot be predicated.
+ if (BBI.IsDone || BBI.IsUnpredicable)
+ return false;
+
+ // If it is already predicated, check if its predicate subsumes the new
+ // predicate.
+ if (BBI.Predicate.size() && !TII->SubsumesPredicate(BBI.Predicate, Pred))
+ return false;
+
+ if (BBI.BrCond.size()) {
+ if (!isTriangle)
+ return false;
+
+ // Test predicate subsumsion.
+ std::vector<MachineOperand> RevPred(Pred);
+ std::vector<MachineOperand> Cond(BBI.BrCond);
+ if (RevBranch) {
+ if (TII->ReverseBranchCondition(Cond))
+ return false;
+ }
+ if (TII->ReverseBranchCondition(RevPred) ||
+ !TII->SubsumesPredicate(Cond, RevPred))
+ return false;
+ }
+
+ return true;
+}
+
+/// AnalyzeBlock - Analyze the structure of the sub-CFG starting from
+/// the specified block. Record its successors and whether it looks like an
+/// if-conversion candidate.
+IfConverter::BBInfo &IfConverter::AnalyzeBlock(MachineBasicBlock *BB,
+ std::vector<IfcvtToken*> &Tokens) {
+ BBInfo &BBI = BBAnalysis[BB->getNumber()];
+
+ if (BBI.IsAnalyzed || BBI.IsBeingAnalyzed)
+ return BBI;
+
+ BBI.BB = BB;
+ BBI.IsBeingAnalyzed = true;
+
+ ScanInstructions(BBI);
+
+ // Unanalyable or ends with fallthrough or unconditional branch.
+ if (!BBI.IsBrAnalyzable || BBI.BrCond.size() == 0) {
+ BBI.IsBeingAnalyzed = false;
+ BBI.IsAnalyzed = true;
+ return BBI;
+ }
+
+ // Do not ifcvt if either path is a back edge to the entry block.
+ if (BBI.TrueBB == BB || BBI.FalseBB == BB) {
+ BBI.IsBeingAnalyzed = false;
+ BBI.IsAnalyzed = true;
+ return BBI;
+ }
+
+ BBInfo &TrueBBI = AnalyzeBlock(BBI.TrueBB, Tokens);
+ BBInfo &FalseBBI = AnalyzeBlock(BBI.FalseBB, Tokens);
+
+ if (TrueBBI.IsDone && FalseBBI.IsDone) {
+ BBI.IsBeingAnalyzed = false;
+ BBI.IsAnalyzed = true;
+ return BBI;
+ }
+
+ std::vector<MachineOperand> RevCond(BBI.BrCond);
+ bool CanRevCond = !TII->ReverseBranchCondition(RevCond);
+
+ unsigned Dups = 0;
+ unsigned Dups2 = 0;
+ bool TNeedSub = TrueBBI.Predicate.size() > 0;
+ bool FNeedSub = FalseBBI.Predicate.size() > 0;
+ bool Enqueued = false;
+ if (CanRevCond && ValidDiamond(TrueBBI, FalseBBI, Dups, Dups2) &&
+ MeetIfcvtSizeLimit(TrueBBI.NonPredSize - (Dups + Dups2)) &&
+ MeetIfcvtSizeLimit(FalseBBI.NonPredSize - (Dups + Dups2)) &&
+ FeasibilityAnalysis(TrueBBI, BBI.BrCond) &&
+ FeasibilityAnalysis(FalseBBI, RevCond)) {
+ // Diamond:
+ // EBB
+ // / \_
+ // | |
+ // TBB FBB
+ // \ /
+ // TailBB
+ // Note TailBB can be empty.
+ Tokens.push_back(new IfcvtToken(BBI, ICDiamond, TNeedSub|FNeedSub, Dups,
+ Dups2));
+ Enqueued = true;
+ }
+
+ if (ValidTriangle(TrueBBI, FalseBBI, false, Dups) &&
+ MeetIfcvtSizeLimit(TrueBBI.NonPredSize) &&
+ FeasibilityAnalysis(TrueBBI, BBI.BrCond, true)) {
+ // Triangle:
+ // EBB
+ // | \_
+ // | |
+ // | TBB
+ // | /
+ // FBB
+ Tokens.push_back(new IfcvtToken(BBI, ICTriangle, TNeedSub, Dups));
+ Enqueued = true;
+ }
+
+ if (ValidTriangle(TrueBBI, FalseBBI, true, Dups) &&
+ MeetIfcvtSizeLimit(TrueBBI.NonPredSize) &&
+ FeasibilityAnalysis(TrueBBI, BBI.BrCond, true, true)) {
+ Tokens.push_back(new IfcvtToken(BBI, ICTriangleRev, TNeedSub, Dups));
+ Enqueued = true;
+ }
+
+ if (ValidSimple(TrueBBI, Dups) &&
+ MeetIfcvtSizeLimit(TrueBBI.NonPredSize) &&
+ FeasibilityAnalysis(TrueBBI, BBI.BrCond)) {
+ // Simple (split, no rejoin):
+ // EBB
+ // | \_
+ // | |
+ // | TBB---> exit
+ // |
+ // FBB
+ Tokens.push_back(new IfcvtToken(BBI, ICSimple, TNeedSub, Dups));
+ Enqueued = true;
+ }
+
+ if (CanRevCond) {
+ // Try the other path...
+ if (ValidTriangle(FalseBBI, TrueBBI, false, Dups) &&
+ MeetIfcvtSizeLimit(FalseBBI.NonPredSize) &&
+ FeasibilityAnalysis(FalseBBI, RevCond, true)) {
+ Tokens.push_back(new IfcvtToken(BBI, ICTriangleFalse, FNeedSub, Dups));
+ Enqueued = true;
+ }
+
+ if (ValidTriangle(FalseBBI, TrueBBI, true, Dups) &&
+ MeetIfcvtSizeLimit(FalseBBI.NonPredSize) &&
+ FeasibilityAnalysis(FalseBBI, RevCond, true, true)) {
+ Tokens.push_back(new IfcvtToken(BBI, ICTriangleFRev, FNeedSub, Dups));
+ Enqueued = true;
+ }
+
+ if (ValidSimple(FalseBBI, Dups) &&
+ MeetIfcvtSizeLimit(FalseBBI.NonPredSize) &&
+ FeasibilityAnalysis(FalseBBI, RevCond)) {
+ Tokens.push_back(new IfcvtToken(BBI, ICSimpleFalse, FNeedSub, Dups));
+ Enqueued = true;
+ }
+ }
+
+ BBI.IsEnqueued = Enqueued;
+ BBI.IsBeingAnalyzed = false;
+ BBI.IsAnalyzed = true;
+ return BBI;
+}
+
+/// AnalyzeBlocks - Analyze all blocks and find entries for all if-conversion
+/// candidates. It returns true if any CFG restructuring is done to expose more
+/// if-conversion opportunities.
+bool IfConverter::AnalyzeBlocks(MachineFunction &MF,
+ std::vector<IfcvtToken*> &Tokens) {
+ bool Change = false;
+ std::set<MachineBasicBlock*> Visited;
+ for (unsigned i = 0, e = Roots.size(); i != e; ++i) {
+ for (idf_ext_iterator<MachineBasicBlock*> I=idf_ext_begin(Roots[i],Visited),
+ E = idf_ext_end(Roots[i], Visited); I != E; ++I) {
+ MachineBasicBlock *BB = *I;
+ AnalyzeBlock(BB, Tokens);
+ }
+ }
+
+ // Sort to favor more complex ifcvt scheme.
+ std::stable_sort(Tokens.begin(), Tokens.end(), IfcvtTokenCmp);
+
+ return Change;
+}
+
+/// canFallThroughTo - Returns true either if ToBB is the next block after BB or
+/// that all the intervening blocks are empty (given BB can fall through to its
+/// next block).
+static bool canFallThroughTo(MachineBasicBlock *BB, MachineBasicBlock *ToBB) {
+ MachineFunction::iterator I = BB;
+ MachineFunction::iterator TI = ToBB;
+ MachineFunction::iterator E = BB->getParent()->end();
+ while (++I != TI)
+ if (I == E || !I->empty())
+ return false;
+ return true;
+}
+
+/// InvalidatePreds - Invalidate predecessor BB info so it would be re-analyzed
+/// to determine if it can be if-converted. If predecessor is already enqueued,
+/// dequeue it!
+void IfConverter::InvalidatePreds(MachineBasicBlock *BB) {
+ for (MachineBasicBlock::pred_iterator PI = BB->pred_begin(),
+ E = BB->pred_end(); PI != E; ++PI) {
+ BBInfo &PBBI = BBAnalysis[(*PI)->getNumber()];
+ if (PBBI.IsDone || PBBI.BB == BB)
+ continue;
+ PBBI.IsAnalyzed = false;
+ PBBI.IsEnqueued = false;
+ }
+}
+
+/// InsertUncondBranch - Inserts an unconditional branch from BB to ToBB.
+///
+static void InsertUncondBranch(MachineBasicBlock *BB, MachineBasicBlock *ToBB,
+ const TargetInstrInfo *TII) {
+ std::vector<MachineOperand> NoCond;
+ TII->InsertBranch(*BB, ToBB, NULL, NoCond);
+}
+
+/// RemoveExtraEdges - Remove true / false edges if either / both are no longer
+/// successors.
+void IfConverter::RemoveExtraEdges(BBInfo &BBI) {
+ MachineBasicBlock *TBB = NULL, *FBB = NULL;
+ std::vector<MachineOperand> Cond;
+ if (!TII->AnalyzeBranch(*BBI.BB, TBB, FBB, Cond))
+ BBI.BB->CorrectExtraCFGEdges(TBB, FBB, !Cond.empty());
+}
+
+/// IfConvertSimple - If convert a simple (split, no rejoin) sub-CFG.
+///
+bool IfConverter::IfConvertSimple(BBInfo &BBI, IfcvtKind Kind) {
+ BBInfo &TrueBBI = BBAnalysis[BBI.TrueBB->getNumber()];
+ BBInfo &FalseBBI = BBAnalysis[BBI.FalseBB->getNumber()];
+ BBInfo *CvtBBI = &TrueBBI;
+ BBInfo *NextBBI = &FalseBBI;
+
+ std::vector<MachineOperand> Cond(BBI.BrCond);
+ if (Kind == ICSimpleFalse)
+ std::swap(CvtBBI, NextBBI);
+
+ if (CvtBBI->IsDone ||
+ (CvtBBI->CannotBeCopied && CvtBBI->BB->pred_size() > 1)) {
+ // Something has changed. It's no longer safe to predicate this block.
+ BBI.IsAnalyzed = false;
+ CvtBBI->IsAnalyzed = false;
+ return false;
+ }
+
+ if (Kind == ICSimpleFalse)
+ TII->ReverseBranchCondition(Cond);
+
+ if (CvtBBI->BB->pred_size() > 1) {
+ BBI.NonPredSize -= TII->RemoveBranch(*BBI.BB);
+ // Copy instructions in the true block, predicate them add them to
+ // the entry block.
+ CopyAndPredicateBlock(BBI, *CvtBBI, Cond);
+ } else {
+ PredicateBlock(*CvtBBI, CvtBBI->BB->end(), Cond);
+
+ // Merge converted block into entry block.
+ BBI.NonPredSize -= TII->RemoveBranch(*BBI.BB);
+ MergeBlocks(BBI, *CvtBBI);
+ }
+
+ bool IterIfcvt = true;
+ if (!canFallThroughTo(BBI.BB, NextBBI->BB)) {
+ InsertUncondBranch(BBI.BB, NextBBI->BB, TII);
+ BBI.HasFallThrough = false;
+ // Now ifcvt'd block will look like this:
+ // BB:
+ // ...
+ // t, f = cmp
+ // if t op
+ // b BBf
+ //
+ // We cannot further ifcvt this block because the unconditional branch
+ // will have to be predicated on the new condition, that will not be
+ // available if cmp executes.
+ IterIfcvt = false;
+ }
+
+ RemoveExtraEdges(BBI);
+
+ // Update block info. BB can be iteratively if-converted.
+ if (!IterIfcvt)
+ BBI.IsDone = true;
+ InvalidatePreds(BBI.BB);
+ CvtBBI->IsDone = true;
+
+ // FIXME: Must maintain LiveIns.
+ return true;
+}
+
+/// IfConvertTriangle - If convert a triangle sub-CFG.
+///
+bool IfConverter::IfConvertTriangle(BBInfo &BBI, IfcvtKind Kind) {
+ BBInfo &TrueBBI = BBAnalysis[BBI.TrueBB->getNumber()];
+ BBInfo &FalseBBI = BBAnalysis[BBI.FalseBB->getNumber()];
+ BBInfo *CvtBBI = &TrueBBI;
+ BBInfo *NextBBI = &FalseBBI;
+
+ std::vector<MachineOperand> Cond(BBI.BrCond);
+ if (Kind == ICTriangleFalse || Kind == ICTriangleFRev)
+ std::swap(CvtBBI, NextBBI);
+
+ if (CvtBBI->IsDone ||
+ (CvtBBI->CannotBeCopied && CvtBBI->BB->pred_size() > 1)) {
+ // Something has changed. It's no longer safe to predicate this block.
+ BBI.IsAnalyzed = false;
+ CvtBBI->IsAnalyzed = false;
+ return false;
+ }
+
+ if (Kind == ICTriangleFalse || Kind == ICTriangleFRev)
+ TII->ReverseBranchCondition(Cond);
+
+ if (Kind == ICTriangleRev || Kind == ICTriangleFRev) {
+ ReverseBranchCondition(*CvtBBI);
+ // BB has been changed, modify its predecessors (except for this
+ // one) so they don't get ifcvt'ed based on bad intel.
+ for (MachineBasicBlock::pred_iterator PI = CvtBBI->BB->pred_begin(),
+ E = CvtBBI->BB->pred_end(); PI != E; ++PI) {
+ MachineBasicBlock *PBB = *PI;
+ if (PBB == BBI.BB)
+ continue;
+ BBInfo &PBBI = BBAnalysis[PBB->getNumber()];
+ if (PBBI.IsEnqueued) {
+ PBBI.IsAnalyzed = false;
+ PBBI.IsEnqueued = false;
+ }
+ }
+ }
+
+ bool HasEarlyExit = CvtBBI->FalseBB != NULL;
+ bool DupBB = CvtBBI->BB->pred_size() > 1;
+ if (DupBB) {
+ BBI.NonPredSize -= TII->RemoveBranch(*BBI.BB);
+ // Copy instructions in the true block, predicate them add them to
+ // the entry block.
+ CopyAndPredicateBlock(BBI, *CvtBBI, Cond, true);
+ } else {
+ // Predicate the 'true' block after removing its branch.
+ CvtBBI->NonPredSize -= TII->RemoveBranch(*CvtBBI->BB);
+ PredicateBlock(*CvtBBI, CvtBBI->BB->end(), Cond);
+ }
+
+ if (!DupBB) {
+ // Now merge the entry of the triangle with the true block.
+ BBI.NonPredSize -= TII->RemoveBranch(*BBI.BB);
+ MergeBlocks(BBI, *CvtBBI);
+ }
+
+ // If 'true' block has a 'false' successor, add an exit branch to it.
+ if (HasEarlyExit) {
+ std::vector<MachineOperand> RevCond(CvtBBI->BrCond);
+ if (TII->ReverseBranchCondition(RevCond))
+ assert(false && "Unable to reverse branch condition!");
+ TII->InsertBranch(*BBI.BB, CvtBBI->FalseBB, NULL, RevCond);
+ BBI.BB->addSuccessor(CvtBBI->FalseBB);
+ }
+
+ // Merge in the 'false' block if the 'false' block has no other
+ // predecessors. Otherwise, add a unconditional branch from to 'false'.
+ bool FalseBBDead = false;
+ bool IterIfcvt = true;
+ bool isFallThrough = canFallThroughTo(BBI.BB, NextBBI->BB);
+ if (!isFallThrough) {
+ // Only merge them if the true block does not fallthrough to the false
+ // block. By not merging them, we make it possible to iteratively
+ // ifcvt the blocks.
+ if (!HasEarlyExit &&
+ NextBBI->BB->pred_size() == 1 && !NextBBI->HasFallThrough) {
+ MergeBlocks(BBI, *NextBBI);
+ FalseBBDead = true;
+ } else {
+ InsertUncondBranch(BBI.BB, NextBBI->BB, TII);
+ BBI.HasFallThrough = false;
+ }
+ // Mixed predicated and unpredicated code. This cannot be iteratively
+ // predicated.
+ IterIfcvt = false;
+ }
+
+ RemoveExtraEdges(BBI);
+
+ // Update block info. BB can be iteratively if-converted.
+ if (!IterIfcvt)
+ BBI.IsDone = true;
+ InvalidatePreds(BBI.BB);
+ CvtBBI->IsDone = true;
+ if (FalseBBDead)
+ NextBBI->IsDone = true;
+
+ // FIXME: Must maintain LiveIns.
+ return true;
+}
+
+/// IfConvertDiamond - If convert a diamond sub-CFG.
+///
+bool IfConverter::IfConvertDiamond(BBInfo &BBI, IfcvtKind Kind,
+ unsigned NumDups1, unsigned NumDups2) {
+ BBInfo &TrueBBI = BBAnalysis[BBI.TrueBB->getNumber()];
+ BBInfo &FalseBBI = BBAnalysis[BBI.FalseBB->getNumber()];
+ MachineBasicBlock *TailBB = TrueBBI.TrueBB;
+ // True block must fall through or ended with unanalyzable terminator.
+ if (!TailBB) {
+ if (blockAlwaysFallThrough(TrueBBI))
+ TailBB = FalseBBI.TrueBB;
+ assert((TailBB || !TrueBBI.IsBrAnalyzable) && "Unexpected!");
+ }
+
+ if (TrueBBI.IsDone || FalseBBI.IsDone ||
+ TrueBBI.BB->pred_size() > 1 ||
+ FalseBBI.BB->pred_size() > 1) {
+ // Something has changed. It's no longer safe to predicate these blocks.
+ BBI.IsAnalyzed = false;
+ TrueBBI.IsAnalyzed = false;
+ FalseBBI.IsAnalyzed = false;
+ return false;
+ }
+
+ // Merge the 'true' and 'false' blocks by copying the instructions
+ // from the 'false' block to the 'true' block. That is, unless the true
+ // block would clobber the predicate, in that case, do the opposite.
+ BBInfo *BBI1 = &TrueBBI;
+ BBInfo *BBI2 = &FalseBBI;
+ std::vector<MachineOperand> RevCond(BBI.BrCond);
+ TII->ReverseBranchCondition(RevCond);
+ std::vector<MachineOperand> *Cond1 = &BBI.BrCond;
+ std::vector<MachineOperand> *Cond2 = &RevCond;
+
+ // Figure out the more profitable ordering.
+ bool DoSwap = false;
+ if (TrueBBI.ClobbersPred && !FalseBBI.ClobbersPred)
+ DoSwap = true;
+ else if (TrueBBI.ClobbersPred == FalseBBI.ClobbersPred) {
+ if (TrueBBI.NonPredSize > FalseBBI.NonPredSize)
+ DoSwap = true;
+ }
+ if (DoSwap) {
+ std::swap(BBI1, BBI2);
+ std::swap(Cond1, Cond2);
+ }
+
+ // Remove the conditional branch from entry to the blocks.
+ BBI.NonPredSize -= TII->RemoveBranch(*BBI.BB);
+
+ // Remove the duplicated instructions at the beginnings of both paths.
+ MachineBasicBlock::iterator DI1 = BBI1->BB->begin();
+ MachineBasicBlock::iterator DI2 = BBI2->BB->begin();
+ BBI1->NonPredSize -= NumDups1;
+ BBI2->NonPredSize -= NumDups1;
+ while (NumDups1 != 0) {
+ ++DI1;
+ ++DI2;
+ --NumDups1;
+ }
+ BBI.BB->splice(BBI.BB->end(), BBI1->BB, BBI1->BB->begin(), DI1);
+ BBI2->BB->erase(BBI2->BB->begin(), DI2);
+
+ // Predicate the 'true' block after removing its branch.
+ BBI1->NonPredSize -= TII->RemoveBranch(*BBI1->BB);
+ DI1 = BBI1->BB->end();
+ for (unsigned i = 0; i != NumDups2; ++i)
+ --DI1;
+ BBI1->BB->erase(DI1, BBI1->BB->end());
+ PredicateBlock(*BBI1, BBI1->BB->end(), *Cond1);
+
+ // Predicate the 'false' block.
+ BBI2->NonPredSize -= TII->RemoveBranch(*BBI2->BB);
+ DI2 = BBI2->BB->end();
+ while (NumDups2 != 0) {
+ --DI2;
+ --NumDups2;
+ }
+ PredicateBlock(*BBI2, DI2, *Cond2);
+
+ // Merge the true block into the entry of the diamond.
+ MergeBlocks(BBI, *BBI1);
+ MergeBlocks(BBI, *BBI2);
+
+ // If the if-converted block fallthrough or unconditionally branch into the
+ // tail block, and the tail block does not have other predecessors, then
+ // fold the tail block in as well. Otherwise, unless it falls through to the
+ // tail, add a unconditional branch to it.
+ if (TailBB) {
+ BBInfo TailBBI = BBAnalysis[TailBB->getNumber()];
+ if (TailBB->pred_size() == 1 && !TailBBI.HasFallThrough) {
+ BBI.NonPredSize -= TII->RemoveBranch(*BBI.BB);
+ MergeBlocks(BBI, TailBBI);
+ TailBBI.IsDone = true;
+ } else {
+ InsertUncondBranch(BBI.BB, TailBB, TII);
+ BBI.HasFallThrough = false;
+ }
+ }
+
+ RemoveExtraEdges(BBI);
+
+ // Update block info.
+ BBI.IsDone = TrueBBI.IsDone = FalseBBI.IsDone = true;
+ InvalidatePreds(BBI.BB);
+
+ // FIXME: Must maintain LiveIns.
+ return true;
+}
+
+/// PredicateBlock - Predicate instructions from the start of the block to the
+/// specified end with the specified condition.
+void IfConverter::PredicateBlock(BBInfo &BBI,
+ MachineBasicBlock::iterator E,
+ std::vector<MachineOperand> &Cond) {
+ for (MachineBasicBlock::iterator I = BBI.BB->begin(); I != E; ++I) {
+ if (TII->isPredicated(I))
+ continue;
+ if (!TII->PredicateInstruction(I, Cond)) {
+ cerr << "Unable to predicate " << *I << "!\n";
+ abort();
+ }
+ }
+
+ std::copy(Cond.begin(), Cond.end(), std::back_inserter(BBI.Predicate));
+
+ BBI.IsAnalyzed = false;
+ BBI.NonPredSize = 0;
+
+ NumIfConvBBs++;
+}
+
+/// CopyAndPredicateBlock - Copy and predicate instructions from source BB to
+/// the destination block. Skip end of block branches if IgnoreBr is true.
+void IfConverter::CopyAndPredicateBlock(BBInfo &ToBBI, BBInfo &FromBBI,
+ std::vector<MachineOperand> &Cond,
+ bool IgnoreBr) {
+ for (MachineBasicBlock::iterator I = FromBBI.BB->begin(),
+ E = FromBBI.BB->end(); I != E; ++I) {
+ const TargetInstrDescriptor *TID = I->getInstrDescriptor();
+ bool isPredicated = TII->isPredicated(I);
+ // Do not copy the end of the block branches.
+ if (IgnoreBr && !isPredicated && (TID->Flags & M_BRANCH_FLAG) != 0)
+ break;
+
+ MachineInstr *MI = I->clone();
+ ToBBI.BB->insert(ToBBI.BB->end(), MI);
+ ToBBI.NonPredSize++;
+
+ if (!isPredicated)
+ if (!TII->PredicateInstruction(MI, Cond)) {
+ cerr << "Unable to predicate " << *MI << "!\n";
+ abort();
+ }
+ }
+
+ std::vector<MachineBasicBlock *> Succs(FromBBI.BB->succ_begin(),
+ FromBBI.BB->succ_end());
+ MachineBasicBlock *NBB = getNextBlock(FromBBI.BB);
+ MachineBasicBlock *FallThrough = FromBBI.HasFallThrough ? NBB : NULL;
+
+ for (unsigned i = 0, e = Succs.size(); i != e; ++i) {
+ MachineBasicBlock *Succ = Succs[i];
+ // Fallthrough edge can't be transferred.
+ if (Succ == FallThrough)
+ continue;
+ if (!ToBBI.BB->isSuccessor(Succ))
+ ToBBI.BB->addSuccessor(Succ);
+ }
+
+ std::copy(FromBBI.Predicate.begin(), FromBBI.Predicate.end(),
+ std::back_inserter(ToBBI.Predicate));
+ std::copy(Cond.begin(), Cond.end(), std::back_inserter(ToBBI.Predicate));
+
+ ToBBI.ClobbersPred |= FromBBI.ClobbersPred;
+ ToBBI.IsAnalyzed = false;
+
+ NumDupBBs++;
+}
+
+/// MergeBlocks - Move all instructions from FromBB to the end of ToBB.
+///
+void IfConverter::MergeBlocks(BBInfo &ToBBI, BBInfo &FromBBI) {
+ ToBBI.BB->splice(ToBBI.BB->end(),
+ FromBBI.BB, FromBBI.BB->begin(), FromBBI.BB->end());
+
+ // Redirect all branches to FromBB to ToBB.
+ std::vector<MachineBasicBlock *> Preds(FromBBI.BB->pred_begin(),
+ FromBBI.BB->pred_end());
+ for (unsigned i = 0, e = Preds.size(); i != e; ++i) {
+ MachineBasicBlock *Pred = Preds[i];
+ if (Pred == ToBBI.BB)
+ continue;
+ Pred->ReplaceUsesOfBlockWith(FromBBI.BB, ToBBI.BB);
+ }
+
+ std::vector<MachineBasicBlock *> Succs(FromBBI.BB->succ_begin(),
+ FromBBI.BB->succ_end());
+ MachineBasicBlock *NBB = getNextBlock(FromBBI.BB);
+ MachineBasicBlock *FallThrough = FromBBI.HasFallThrough ? NBB : NULL;
+
+ for (unsigned i = 0, e = Succs.size(); i != e; ++i) {
+ MachineBasicBlock *Succ = Succs[i];
+ // Fallthrough edge can't be transferred.
+ if (Succ == FallThrough)
+ continue;
+ FromBBI.BB->removeSuccessor(Succ);
+ if (!ToBBI.BB->isSuccessor(Succ))
+ ToBBI.BB->addSuccessor(Succ);
+ }
+
+ // Now FromBBI always fall through to the next block!
+ if (NBB && !FromBBI.BB->isSuccessor(NBB))
+ FromBBI.BB->addSuccessor(NBB);
+
+ std::copy(FromBBI.Predicate.begin(), FromBBI.Predicate.end(),
+ std::back_inserter(ToBBI.Predicate));
+ FromBBI.Predicate.clear();
+
+ ToBBI.NonPredSize += FromBBI.NonPredSize;
+ FromBBI.NonPredSize = 0;
+
+ ToBBI.ClobbersPred |= FromBBI.ClobbersPred;
+ ToBBI.HasFallThrough = FromBBI.HasFallThrough;
+ ToBBI.IsAnalyzed = false;
+ FromBBI.IsAnalyzed = false;
+}
diff --git a/lib/CodeGen/IntrinsicLowering.cpp b/lib/CodeGen/IntrinsicLowering.cpp
new file mode 100644
index 0000000..8ae4df6
--- /dev/null
+++ b/lib/CodeGen/IntrinsicLowering.cpp
@@ -0,0 +1,799 @@
+//===-- IntrinsicLowering.cpp - Intrinsic Lowering default implementation -===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file was developed by the LLVM research group and is distributed under
+// the University of Illinois Open Source License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements the IntrinsicLowering class.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Constants.h"
+#include "llvm/DerivedTypes.h"
+#include "llvm/Module.h"
+#include "llvm/Instructions.h"
+#include "llvm/Type.h"
+#include "llvm/CodeGen/IntrinsicLowering.h"
+#include "llvm/Support/Streams.h"
+#include "llvm/Target/TargetData.h"
+#include "llvm/ADT/SmallVector.h"
+using namespace llvm;
+
+template <class ArgIt>
+static void EnsureFunctionExists(Module &M, const char *Name,
+ ArgIt ArgBegin, ArgIt ArgEnd,
+ const Type *RetTy) {
+ // Insert a correctly-typed definition now.
+ std::vector<const Type *> ParamTys;
+ for (ArgIt I = ArgBegin; I != ArgEnd; ++I)
+ ParamTys.push_back(I->getType());
+ M.getOrInsertFunction(Name, FunctionType::get(RetTy, ParamTys, false));
+}
+
+/// ReplaceCallWith - This function is used when we want to lower an intrinsic
+/// call to a call of an external function. This handles hard cases such as
+/// when there was already a prototype for the external function, and if that
+/// prototype doesn't match the arguments we expect to pass in.
+template <class ArgIt>
+static CallInst *ReplaceCallWith(const char *NewFn, CallInst *CI,
+ ArgIt ArgBegin, ArgIt ArgEnd,
+ const Type *RetTy, Constant *&FCache) {
+ if (!FCache) {
+ // If we haven't already looked up this function, check to see if the
+ // program already contains a function with this name.
+ Module *M = CI->getParent()->getParent()->getParent();
+ // Get or insert the definition now.
+ std::vector<const Type *> ParamTys;
+ for (ArgIt I = ArgBegin; I != ArgEnd; ++I)
+ ParamTys.push_back((*I)->getType());
+ FCache = M->getOrInsertFunction(NewFn,
+ FunctionType::get(RetTy, ParamTys, false));
+ }
+
+ SmallVector<Value*, 8> Operands(ArgBegin, ArgEnd);
+ CallInst *NewCI = new CallInst(FCache, &Operands[0], Operands.size(),
+ CI->getName(), CI);
+ if (!CI->use_empty())
+ CI->replaceAllUsesWith(NewCI);
+ return NewCI;
+}
+
+void IntrinsicLowering::AddPrototypes(Module &M) {
+ for (Module::iterator I = M.begin(), E = M.end(); I != E; ++I)
+ if (I->isDeclaration() && !I->use_empty())
+ switch (I->getIntrinsicID()) {
+ default: break;
+ case Intrinsic::setjmp:
+ EnsureFunctionExists(M, "setjmp", I->arg_begin(), I->arg_end(),
+ Type::Int32Ty);
+ break;
+ case Intrinsic::longjmp:
+ EnsureFunctionExists(M, "longjmp", I->arg_begin(), I->arg_end(),
+ Type::VoidTy);
+ break;
+ case Intrinsic::siglongjmp:
+ EnsureFunctionExists(M, "abort", I->arg_end(), I->arg_end(),
+ Type::VoidTy);
+ break;
+ case Intrinsic::memcpy_i32:
+ case Intrinsic::memcpy_i64:
+ M.getOrInsertFunction("memcpy", PointerType::get(Type::Int8Ty),
+ PointerType::get(Type::Int8Ty),
+ PointerType::get(Type::Int8Ty),
+ TD.getIntPtrType(), (Type *)0);
+ break;
+ case Intrinsic::memmove_i32:
+ case Intrinsic::memmove_i64:
+ M.getOrInsertFunction("memmove", PointerType::get(Type::Int8Ty),
+ PointerType::get(Type::Int8Ty),
+ PointerType::get(Type::Int8Ty),
+ TD.getIntPtrType(), (Type *)0);
+ break;
+ case Intrinsic::memset_i32:
+ case Intrinsic::memset_i64:
+ M.getOrInsertFunction("memset", PointerType::get(Type::Int8Ty),
+ PointerType::get(Type::Int8Ty), Type::Int32Ty,
+ TD.getIntPtrType(), (Type *)0);
+ break;
+ case Intrinsic::sqrt_f32:
+ case Intrinsic::sqrt_f64:
+ if(I->arg_begin()->getType() == Type::FloatTy)
+ EnsureFunctionExists(M, "sqrtf", I->arg_begin(), I->arg_end(),
+ Type::FloatTy);
+ else
+ EnsureFunctionExists(M, "sqrt", I->arg_begin(), I->arg_end(),
+ Type::DoubleTy);
+ break;
+ }
+}
+
+/// LowerBSWAP - Emit the code to lower bswap of V before the specified
+/// instruction IP.
+static Value *LowerBSWAP(Value *V, Instruction *IP) {
+ assert(V->getType()->isInteger() && "Can't bswap a non-integer type!");
+
+ unsigned BitSize = V->getType()->getPrimitiveSizeInBits();
+
+ switch(BitSize) {
+ default: assert(0 && "Unhandled type size of value to byteswap!");
+ case 16: {
+ Value *Tmp1 = BinaryOperator::createShl(V,
+ ConstantInt::get(V->getType(),8),"bswap.2",IP);
+ Value *Tmp2 = BinaryOperator::createLShr(V,
+ ConstantInt::get(V->getType(),8),"bswap.1",IP);
+ V = BinaryOperator::createOr(Tmp1, Tmp2, "bswap.i16", IP);
+ break;
+ }
+ case 32: {
+ Value *Tmp4 = BinaryOperator::createShl(V,
+ ConstantInt::get(V->getType(),24),"bswap.4", IP);
+ Value *Tmp3 = BinaryOperator::createShl(V,
+ ConstantInt::get(V->getType(),8),"bswap.3",IP);
+ Value *Tmp2 = BinaryOperator::createLShr(V,
+ ConstantInt::get(V->getType(),8),"bswap.2",IP);
+ Value *Tmp1 = BinaryOperator::createLShr(V,
+ ConstantInt::get(V->getType(),24),"bswap.1", IP);
+ Tmp3 = BinaryOperator::createAnd(Tmp3,
+ ConstantInt::get(Type::Int32Ty, 0xFF0000),
+ "bswap.and3", IP);
+ Tmp2 = BinaryOperator::createAnd(Tmp2,
+ ConstantInt::get(Type::Int32Ty, 0xFF00),
+ "bswap.and2", IP);
+ Tmp4 = BinaryOperator::createOr(Tmp4, Tmp3, "bswap.or1", IP);
+ Tmp2 = BinaryOperator::createOr(Tmp2, Tmp1, "bswap.or2", IP);
+ V = BinaryOperator::createOr(Tmp4, Tmp2, "bswap.i32", IP);
+ break;
+ }
+ case 64: {
+ Value *Tmp8 = BinaryOperator::createShl(V,
+ ConstantInt::get(V->getType(),56),"bswap.8", IP);
+ Value *Tmp7 = BinaryOperator::createShl(V,
+ ConstantInt::get(V->getType(),40),"bswap.7", IP);
+ Value *Tmp6 = BinaryOperator::createShl(V,
+ ConstantInt::get(V->getType(),24),"bswap.6", IP);
+ Value *Tmp5 = BinaryOperator::createShl(V,
+ ConstantInt::get(V->getType(),8),"bswap.5", IP);
+ Value* Tmp4 = BinaryOperator::createLShr(V,
+ ConstantInt::get(V->getType(),8),"bswap.4", IP);
+ Value* Tmp3 = BinaryOperator::createLShr(V,
+ ConstantInt::get(V->getType(),24),"bswap.3", IP);
+ Value* Tmp2 = BinaryOperator::createLShr(V,
+ ConstantInt::get(V->getType(),40),"bswap.2", IP);
+ Value* Tmp1 = BinaryOperator::createLShr(V,
+ ConstantInt::get(V->getType(),56),"bswap.1", IP);
+ Tmp7 = BinaryOperator::createAnd(Tmp7,
+ ConstantInt::get(Type::Int64Ty,
+ 0xFF000000000000ULL),
+ "bswap.and7", IP);
+ Tmp6 = BinaryOperator::createAnd(Tmp6,
+ ConstantInt::get(Type::Int64Ty, 0xFF0000000000ULL),
+ "bswap.and6", IP);
+ Tmp5 = BinaryOperator::createAnd(Tmp5,
+ ConstantInt::get(Type::Int64Ty, 0xFF00000000ULL),
+ "bswap.and5", IP);
+ Tmp4 = BinaryOperator::createAnd(Tmp4,
+ ConstantInt::get(Type::Int64Ty, 0xFF000000ULL),
+ "bswap.and4", IP);
+ Tmp3 = BinaryOperator::createAnd(Tmp3,
+ ConstantInt::get(Type::Int64Ty, 0xFF0000ULL),
+ "bswap.and3", IP);
+ Tmp2 = BinaryOperator::createAnd(Tmp2,
+ ConstantInt::get(Type::Int64Ty, 0xFF00ULL),
+ "bswap.and2", IP);
+ Tmp8 = BinaryOperator::createOr(Tmp8, Tmp7, "bswap.or1", IP);
+ Tmp6 = BinaryOperator::createOr(Tmp6, Tmp5, "bswap.or2", IP);
+ Tmp4 = BinaryOperator::createOr(Tmp4, Tmp3, "bswap.or3", IP);
+ Tmp2 = BinaryOperator::createOr(Tmp2, Tmp1, "bswap.or4", IP);
+ Tmp8 = BinaryOperator::createOr(Tmp8, Tmp6, "bswap.or5", IP);
+ Tmp4 = BinaryOperator::createOr(Tmp4, Tmp2, "bswap.or6", IP);
+ V = BinaryOperator::createOr(Tmp8, Tmp4, "bswap.i64", IP);
+ break;
+ }
+ }
+ return V;
+}
+
+/// LowerCTPOP - Emit the code to lower ctpop of V before the specified
+/// instruction IP.
+static Value *LowerCTPOP(Value *V, Instruction *IP) {
+ assert(V->getType()->isInteger() && "Can't ctpop a non-integer type!");
+
+ static const uint64_t MaskValues[6] = {
+ 0x5555555555555555ULL, 0x3333333333333333ULL,
+ 0x0F0F0F0F0F0F0F0FULL, 0x00FF00FF00FF00FFULL,
+ 0x0000FFFF0000FFFFULL, 0x00000000FFFFFFFFULL
+ };
+
+ unsigned BitSize = V->getType()->getPrimitiveSizeInBits();
+ unsigned WordSize = (BitSize + 63) / 64;
+ Value *Count = ConstantInt::get(V->getType(), 0);
+
+ for (unsigned n = 0; n < WordSize; ++n) {
+ Value *PartValue = V;
+ for (unsigned i = 1, ct = 0; i < (BitSize>64 ? 64 : BitSize);
+ i <<= 1, ++ct) {
+ Value *MaskCst = ConstantInt::get(V->getType(), MaskValues[ct]);
+ Value *LHS = BinaryOperator::createAnd(
+ PartValue, MaskCst, "cppop.and1", IP);
+ Value *VShift = BinaryOperator::createLShr(PartValue,
+ ConstantInt::get(V->getType(), i), "ctpop.sh", IP);
+ Value *RHS = BinaryOperator::createAnd(VShift, MaskCst, "cppop.and2", IP);
+ PartValue = BinaryOperator::createAdd(LHS, RHS, "ctpop.step", IP);
+ }
+ Count = BinaryOperator::createAdd(PartValue, Count, "ctpop.part", IP);
+ if (BitSize > 64) {
+ V = BinaryOperator::createLShr(V, ConstantInt::get(V->getType(), 64),
+ "ctpop.part.sh", IP);
+ BitSize -= 64;
+ }
+ }
+
+ return CastInst::createIntegerCast(Count, Type::Int32Ty, false, "ctpop", IP);
+}
+
+/// LowerCTLZ - Emit the code to lower ctlz of V before the specified
+/// instruction IP.
+static Value *LowerCTLZ(Value *V, Instruction *IP) {
+
+ unsigned BitSize = V->getType()->getPrimitiveSizeInBits();
+ for (unsigned i = 1; i < BitSize; i <<= 1) {
+ Value *ShVal = ConstantInt::get(V->getType(), i);
+ ShVal = BinaryOperator::createLShr(V, ShVal, "ctlz.sh", IP);
+ V = BinaryOperator::createOr(V, ShVal, "ctlz.step", IP);
+ }
+
+ V = BinaryOperator::createNot(V, "", IP);
+ return LowerCTPOP(V, IP);
+}
+
+/// Convert the llvm.part.select.iX.iY intrinsic. This intrinsic takes
+/// three integer arguments. The first argument is the Value from which the
+/// bits will be selected. It may be of any bit width. The second and third
+/// arguments specify a range of bits to select with the second argument
+/// specifying the low bit and the third argument specifying the high bit. Both
+/// must be type i32. The result is the corresponding selected bits from the
+/// Value in the same width as the Value (first argument). If the low bit index
+/// is higher than the high bit index then the inverse selection is done and
+/// the bits are returned in inverse order.
+/// @brief Lowering of llvm.part.select intrinsic.
+static Instruction *LowerPartSelect(CallInst *CI) {
+ // Make sure we're dealing with a part select intrinsic here
+ Function *F = CI->getCalledFunction();
+ const FunctionType *FT = F->getFunctionType();
+ if (!F->isDeclaration() || !FT->getReturnType()->isInteger() ||
+ FT->getNumParams() != 3 || !FT->getParamType(0)->isInteger() ||
+ !FT->getParamType(1)->isInteger() || !FT->getParamType(2)->isInteger())
+ return CI;
+
+ // Get the intrinsic implementation function by converting all the . to _
+ // in the intrinsic's function name and then reconstructing the function
+ // declaration.
+ std::string Name(F->getName());
+ for (unsigned i = 4; i < Name.length(); ++i)
+ if (Name[i] == '.')
+ Name[i] = '_';
+ Module* M = F->getParent();
+ F = cast<Function>(M->getOrInsertFunction(Name, FT));
+ F->setLinkage(GlobalValue::WeakLinkage);
+
+ // If we haven't defined the impl function yet, do so now
+ if (F->isDeclaration()) {
+
+ // Get the arguments to the function
+ Function::arg_iterator args = F->arg_begin();
+ Value* Val = args++; Val->setName("Val");
+ Value* Lo = args++; Lo->setName("Lo");
+ Value* Hi = args++; Hi->setName("High");
+
+ // We want to select a range of bits here such that [Hi, Lo] is shifted
+ // down to the low bits. However, it is quite possible that Hi is smaller
+ // than Lo in which case the bits have to be reversed.
+
+ // Create the blocks we will need for the two cases (forward, reverse)
+ BasicBlock* CurBB = new BasicBlock("entry", F);
+ BasicBlock *RevSize = new BasicBlock("revsize", CurBB->getParent());
+ BasicBlock *FwdSize = new BasicBlock("fwdsize", CurBB->getParent());
+ BasicBlock *Compute = new BasicBlock("compute", CurBB->getParent());
+ BasicBlock *Reverse = new BasicBlock("reverse", CurBB->getParent());
+ BasicBlock *RsltBlk = new BasicBlock("result", CurBB->getParent());
+
+ // Cast Hi and Lo to the size of Val so the widths are all the same
+ if (Hi->getType() != Val->getType())
+ Hi = CastInst::createIntegerCast(Hi, Val->getType(), false,
+ "tmp", CurBB);
+ if (Lo->getType() != Val->getType())
+ Lo = CastInst::createIntegerCast(Lo, Val->getType(), false,
+ "tmp", CurBB);
+
+ // Compute a few things that both cases will need, up front.
+ Constant* Zero = ConstantInt::get(Val->getType(), 0);
+ Constant* One = ConstantInt::get(Val->getType(), 1);
+ Constant* AllOnes = ConstantInt::getAllOnesValue(Val->getType());
+
+ // Compare the Hi and Lo bit positions. This is used to determine
+ // which case we have (forward or reverse)
+ ICmpInst *Cmp = new ICmpInst(ICmpInst::ICMP_ULT, Hi, Lo, "less",CurBB);
+ new BranchInst(RevSize, FwdSize, Cmp, CurBB);
+
+ // First, copmute the number of bits in the forward case.
+ Instruction* FBitSize =
+ BinaryOperator::createSub(Hi, Lo,"fbits", FwdSize);
+ new BranchInst(Compute, FwdSize);
+
+ // Second, compute the number of bits in the reverse case.
+ Instruction* RBitSize =
+ BinaryOperator::createSub(Lo, Hi, "rbits", RevSize);
+ new BranchInst(Compute, RevSize);
+
+ // Now, compute the bit range. Start by getting the bitsize and the shift
+ // amount (either Hi or Lo) from PHI nodes. Then we compute a mask for
+ // the number of bits we want in the range. We shift the bits down to the
+ // least significant bits, apply the mask to zero out unwanted high bits,
+ // and we have computed the "forward" result. It may still need to be
+ // reversed.
+
+ // Get the BitSize from one of the two subtractions
+ PHINode *BitSize = new PHINode(Val->getType(), "bits", Compute);
+ BitSize->reserveOperandSpace(2);
+ BitSize->addIncoming(FBitSize, FwdSize);
+ BitSize->addIncoming(RBitSize, RevSize);
+
+ // Get the ShiftAmount as the smaller of Hi/Lo
+ PHINode *ShiftAmt = new PHINode(Val->getType(), "shiftamt", Compute);
+ ShiftAmt->reserveOperandSpace(2);
+ ShiftAmt->addIncoming(Lo, FwdSize);
+ ShiftAmt->addIncoming(Hi, RevSize);
+
+ // Increment the bit size
+ Instruction *BitSizePlusOne =
+ BinaryOperator::createAdd(BitSize, One, "bits", Compute);
+
+ // Create a Mask to zero out the high order bits.
+ Instruction* Mask =
+ BinaryOperator::createShl(AllOnes, BitSizePlusOne, "mask", Compute);
+ Mask = BinaryOperator::createNot(Mask, "mask", Compute);
+
+ // Shift the bits down and apply the mask
+ Instruction* FRes =
+ BinaryOperator::createLShr(Val, ShiftAmt, "fres", Compute);
+ FRes = BinaryOperator::createAnd(FRes, Mask, "fres", Compute);
+ new BranchInst(Reverse, RsltBlk, Cmp, Compute);
+
+ // In the Reverse block we have the mask already in FRes but we must reverse
+ // it by shifting FRes bits right and putting them in RRes by shifting them
+ // in from left.
+
+ // First set up our loop counters
+ PHINode *Count = new PHINode(Val->getType(), "count", Reverse);
+ Count->reserveOperandSpace(2);
+ Count->addIncoming(BitSizePlusOne, Compute);
+
+ // Next, get the value that we are shifting.
+ PHINode *BitsToShift = new PHINode(Val->getType(), "val", Reverse);
+ BitsToShift->reserveOperandSpace(2);
+ BitsToShift->addIncoming(FRes, Compute);
+
+ // Finally, get the result of the last computation
+ PHINode *RRes = new PHINode(Val->getType(), "rres", Reverse);
+ RRes->reserveOperandSpace(2);
+ RRes->addIncoming(Zero, Compute);
+
+ // Decrement the counter
+ Instruction *Decr = BinaryOperator::createSub(Count, One, "decr", Reverse);
+ Count->addIncoming(Decr, Reverse);
+
+ // Compute the Bit that we want to move
+ Instruction *Bit =
+ BinaryOperator::createAnd(BitsToShift, One, "bit", Reverse);
+
+ // Compute the new value for next iteration.
+ Instruction *NewVal =
+ BinaryOperator::createLShr(BitsToShift, One, "rshift", Reverse);
+ BitsToShift->addIncoming(NewVal, Reverse);
+
+ // Shift the bit into the low bits of the result.
+ Instruction *NewRes =
+ BinaryOperator::createShl(RRes, One, "lshift", Reverse);
+ NewRes = BinaryOperator::createOr(NewRes, Bit, "addbit", Reverse);
+ RRes->addIncoming(NewRes, Reverse);
+
+ // Terminate loop if we've moved all the bits.
+ ICmpInst *Cond =
+ new ICmpInst(ICmpInst::ICMP_EQ, Decr, Zero, "cond", Reverse);
+ new BranchInst(RsltBlk, Reverse, Cond, Reverse);
+
+ // Finally, in the result block, select one of the two results with a PHI
+ // node and return the result;
+ CurBB = RsltBlk;
+ PHINode *BitSelect = new PHINode(Val->getType(), "part_select", CurBB);
+ BitSelect->reserveOperandSpace(2);
+ BitSelect->addIncoming(FRes, Compute);
+ BitSelect->addIncoming(NewRes, Reverse);
+ new ReturnInst(BitSelect, CurBB);
+ }
+
+ // Return a call to the implementation function
+ Value *Args[] = {
+ CI->getOperand(1),
+ CI->getOperand(2),
+ CI->getOperand(3)
+ };
+ return new CallInst(F, Args, sizeof(Args)/sizeof(Args[0]), CI->getName(), CI);
+}
+
+/// Convert the llvm.part.set.iX.iY.iZ intrinsic. This intrinsic takes
+/// four integer arguments (iAny %Value, iAny %Replacement, i32 %Low, i32 %High)
+/// The first two arguments can be any bit width. The result is the same width
+/// as %Value. The operation replaces bits between %Low and %High with the value
+/// in %Replacement. If %Replacement is not the same width, it is truncated or
+/// zero extended as appropriate to fit the bits being replaced. If %Low is
+/// greater than %High then the inverse set of bits are replaced.
+/// @brief Lowering of llvm.bit.part.set intrinsic.
+static Instruction *LowerPartSet(CallInst *CI) {
+ // Make sure we're dealing with a part select intrinsic here
+ Function *F = CI->getCalledFunction();
+ const FunctionType *FT = F->getFunctionType();
+ if (!F->isDeclaration() || !FT->getReturnType()->isInteger() ||
+ FT->getNumParams() != 4 || !FT->getParamType(0)->isInteger() ||
+ !FT->getParamType(1)->isInteger() || !FT->getParamType(2)->isInteger() ||
+ !FT->getParamType(3)->isInteger())
+ return CI;
+
+ // Get the intrinsic implementation function by converting all the . to _
+ // in the intrinsic's function name and then reconstructing the function
+ // declaration.
+ std::string Name(F->getName());
+ for (unsigned i = 4; i < Name.length(); ++i)
+ if (Name[i] == '.')
+ Name[i] = '_';
+ Module* M = F->getParent();
+ F = cast<Function>(M->getOrInsertFunction(Name, FT));
+ F->setLinkage(GlobalValue::WeakLinkage);
+
+ // If we haven't defined the impl function yet, do so now
+ if (F->isDeclaration()) {
+ // Get the arguments for the function.
+ Function::arg_iterator args = F->arg_begin();
+ Value* Val = args++; Val->setName("Val");
+ Value* Rep = args++; Rep->setName("Rep");
+ Value* Lo = args++; Lo->setName("Lo");
+ Value* Hi = args++; Hi->setName("Hi");
+
+ // Get some types we need
+ const IntegerType* ValTy = cast<IntegerType>(Val->getType());
+ const IntegerType* RepTy = cast<IntegerType>(Rep->getType());
+ uint32_t ValBits = ValTy->getBitWidth();
+ uint32_t RepBits = RepTy->getBitWidth();
+
+ // Constant Definitions
+ ConstantInt* RepBitWidth = ConstantInt::get(Type::Int32Ty, RepBits);
+ ConstantInt* RepMask = ConstantInt::getAllOnesValue(RepTy);
+ ConstantInt* ValMask = ConstantInt::getAllOnesValue(ValTy);
+ ConstantInt* One = ConstantInt::get(Type::Int32Ty, 1);
+ ConstantInt* ValOne = ConstantInt::get(ValTy, 1);
+ ConstantInt* Zero = ConstantInt::get(Type::Int32Ty, 0);
+ ConstantInt* ValZero = ConstantInt::get(ValTy, 0);
+
+ // Basic blocks we fill in below.
+ BasicBlock* entry = new BasicBlock("entry", F, 0);
+ BasicBlock* large = new BasicBlock("large", F, 0);
+ BasicBlock* small = new BasicBlock("small", F, 0);
+ BasicBlock* reverse = new BasicBlock("reverse", F, 0);
+ BasicBlock* result = new BasicBlock("result", F, 0);
+
+ // BASIC BLOCK: entry
+ // First, get the number of bits that we're placing as an i32
+ ICmpInst* is_forward =
+ new ICmpInst(ICmpInst::ICMP_ULT, Lo, Hi, "", entry);
+ SelectInst* Hi_pn = new SelectInst(is_forward, Hi, Lo, "", entry);
+ SelectInst* Lo_pn = new SelectInst(is_forward, Lo, Hi, "", entry);
+ BinaryOperator* NumBits = BinaryOperator::createSub(Hi_pn, Lo_pn, "",entry);
+ NumBits = BinaryOperator::createAdd(NumBits, One, "", entry);
+ // Now, convert Lo and Hi to ValTy bit width
+ if (ValBits > 32) {
+ Lo = new ZExtInst(Lo_pn, ValTy, "", entry);
+ } else if (ValBits < 32) {
+ Lo = new TruncInst(Lo_pn, ValTy, "", entry);
+ }
+ // Determine if the replacement bits are larger than the number of bits we
+ // are replacing and deal with it.
+ ICmpInst* is_large =
+ new ICmpInst(ICmpInst::ICMP_ULT, NumBits, RepBitWidth, "", entry);
+ new BranchInst(large, small, is_large, entry);
+
+ // BASIC BLOCK: large
+ Instruction* MaskBits =
+ BinaryOperator::createSub(RepBitWidth, NumBits, "", large);
+ MaskBits = CastInst::createIntegerCast(MaskBits, RepMask->getType(),
+ false, "", large);
+ BinaryOperator* Mask1 =
+ BinaryOperator::createLShr(RepMask, MaskBits, "", large);
+ BinaryOperator* Rep2 = BinaryOperator::createAnd(Mask1, Rep, "", large);
+ new BranchInst(small, large);
+
+ // BASIC BLOCK: small
+ PHINode* Rep3 = new PHINode(RepTy, "", small);
+ Rep3->reserveOperandSpace(2);
+ Rep3->addIncoming(Rep2, large);
+ Rep3->addIncoming(Rep, entry);
+ Value* Rep4 = Rep3;
+ if (ValBits > RepBits)
+ Rep4 = new ZExtInst(Rep3, ValTy, "", small);
+ else if (ValBits < RepBits)
+ Rep4 = new TruncInst(Rep3, ValTy, "", small);
+ new BranchInst(result, reverse, is_forward, small);
+
+ // BASIC BLOCK: reverse (reverses the bits of the replacement)
+ // Set up our loop counter as a PHI so we can decrement on each iteration.
+ // We will loop for the number of bits in the replacement value.
+ PHINode *Count = new PHINode(Type::Int32Ty, "count", reverse);
+ Count->reserveOperandSpace(2);
+ Count->addIncoming(NumBits, small);
+
+ // Get the value that we are shifting bits out of as a PHI because
+ // we'll change this with each iteration.
+ PHINode *BitsToShift = new PHINode(Val->getType(), "val", reverse);
+ BitsToShift->reserveOperandSpace(2);
+ BitsToShift->addIncoming(Rep4, small);
+
+ // Get the result of the last computation or zero on first iteration
+ PHINode *RRes = new PHINode(Val->getType(), "rres", reverse);
+ RRes->reserveOperandSpace(2);
+ RRes->addIncoming(ValZero, small);
+
+ // Decrement the loop counter by one
+ Instruction *Decr = BinaryOperator::createSub(Count, One, "", reverse);
+ Count->addIncoming(Decr, reverse);
+
+ // Get the bit that we want to move into the result
+ Value *Bit = BinaryOperator::createAnd(BitsToShift, ValOne, "", reverse);
+
+ // Compute the new value of the bits to shift for the next iteration.
+ Value *NewVal = BinaryOperator::createLShr(BitsToShift, ValOne,"", reverse);
+ BitsToShift->addIncoming(NewVal, reverse);
+
+ // Shift the bit we extracted into the low bit of the result.
+ Instruction *NewRes = BinaryOperator::createShl(RRes, ValOne, "", reverse);
+ NewRes = BinaryOperator::createOr(NewRes, Bit, "", reverse);
+ RRes->addIncoming(NewRes, reverse);
+
+ // Terminate loop if we've moved all the bits.
+ ICmpInst *Cond = new ICmpInst(ICmpInst::ICMP_EQ, Decr, Zero, "", reverse);
+ new BranchInst(result, reverse, Cond, reverse);
+
+ // BASIC BLOCK: result
+ PHINode *Rplcmnt = new PHINode(Val->getType(), "", result);
+ Rplcmnt->reserveOperandSpace(2);
+ Rplcmnt->addIncoming(NewRes, reverse);
+ Rplcmnt->addIncoming(Rep4, small);
+ Value* t0 = CastInst::createIntegerCast(NumBits,ValTy,false,"",result);
+ Value* t1 = BinaryOperator::createShl(ValMask, Lo, "", result);
+ Value* t2 = BinaryOperator::createNot(t1, "", result);
+ Value* t3 = BinaryOperator::createShl(t1, t0, "", result);
+ Value* t4 = BinaryOperator::createOr(t2, t3, "", result);
+ Value* t5 = BinaryOperator::createAnd(t4, Val, "", result);
+ Value* t6 = BinaryOperator::createShl(Rplcmnt, Lo, "", result);
+ Value* Rslt = BinaryOperator::createOr(t5, t6, "part_set", result);
+ new ReturnInst(Rslt, result);
+ }
+
+ // Return a call to the implementation function
+ Value *Args[] = {
+ CI->getOperand(1),
+ CI->getOperand(2),
+ CI->getOperand(3),
+ CI->getOperand(4)
+ };
+ return new CallInst(F, Args, sizeof(Args)/sizeof(Args[0]), CI->getName(), CI);
+}
+
+
+void IntrinsicLowering::LowerIntrinsicCall(CallInst *CI) {
+ Function *Callee = CI->getCalledFunction();
+ assert(Callee && "Cannot lower an indirect call!");
+
+ switch (Callee->getIntrinsicID()) {
+ case Intrinsic::not_intrinsic:
+ cerr << "Cannot lower a call to a non-intrinsic function '"
+ << Callee->getName() << "'!\n";
+ abort();
+ default:
+ cerr << "Error: Code generator does not support intrinsic function '"
+ << Callee->getName() << "'!\n";
+ abort();
+
+ // The setjmp/longjmp intrinsics should only exist in the code if it was
+ // never optimized (ie, right out of the CFE), or if it has been hacked on
+ // by the lowerinvoke pass. In both cases, the right thing to do is to
+ // convert the call to an explicit setjmp or longjmp call.
+ case Intrinsic::setjmp: {
+ static Constant *SetjmpFCache = 0;
+ Value *V = ReplaceCallWith("setjmp", CI, CI->op_begin()+1, CI->op_end(),
+ Type::Int32Ty, SetjmpFCache);
+ if (CI->getType() != Type::VoidTy)
+ CI->replaceAllUsesWith(V);
+ break;
+ }
+ case Intrinsic::sigsetjmp:
+ if (CI->getType() != Type::VoidTy)
+ CI->replaceAllUsesWith(Constant::getNullValue(CI->getType()));
+ break;
+
+ case Intrinsic::longjmp: {
+ static Constant *LongjmpFCache = 0;
+ ReplaceCallWith("longjmp", CI, CI->op_begin()+1, CI->op_end(),
+ Type::VoidTy, LongjmpFCache);
+ break;
+ }
+
+ case Intrinsic::siglongjmp: {
+ // Insert the call to abort
+ static Constant *AbortFCache = 0;
+ ReplaceCallWith("abort", CI, CI->op_end(), CI->op_end(),
+ Type::VoidTy, AbortFCache);
+ break;
+ }
+ case Intrinsic::ctpop:
+ CI->replaceAllUsesWith(LowerCTPOP(CI->getOperand(1), CI));
+ break;
+
+ case Intrinsic::bswap:
+ CI->replaceAllUsesWith(LowerBSWAP(CI->getOperand(1), CI));
+ break;
+
+ case Intrinsic::ctlz:
+ CI->replaceAllUsesWith(LowerCTLZ(CI->getOperand(1), CI));
+ break;
+
+ case Intrinsic::cttz: {
+ // cttz(x) -> ctpop(~X & (X-1))
+ Value *Src = CI->getOperand(1);
+ Value *NotSrc = BinaryOperator::createNot(Src, Src->getName()+".not", CI);
+ Value *SrcM1 = ConstantInt::get(Src->getType(), 1);
+ SrcM1 = BinaryOperator::createSub(Src, SrcM1, "", CI);
+ Src = LowerCTPOP(BinaryOperator::createAnd(NotSrc, SrcM1, "", CI), CI);
+ CI->replaceAllUsesWith(Src);
+ break;
+ }
+
+ case Intrinsic::part_select:
+ CI->replaceAllUsesWith(LowerPartSelect(CI));
+ break;
+
+ case Intrinsic::part_set:
+ CI->replaceAllUsesWith(LowerPartSet(CI));
+ break;
+
+ case Intrinsic::stacksave:
+ case Intrinsic::stackrestore: {
+ static bool Warned = false;
+ if (!Warned)
+ cerr << "WARNING: this target does not support the llvm.stack"
+ << (Callee->getIntrinsicID() == Intrinsic::stacksave ?
+ "save" : "restore") << " intrinsic.\n";
+ Warned = true;
+ if (Callee->getIntrinsicID() == Intrinsic::stacksave)
+ CI->replaceAllUsesWith(Constant::getNullValue(CI->getType()));
+ break;
+ }
+
+ case Intrinsic::returnaddress:
+ case Intrinsic::frameaddress:
+ cerr << "WARNING: this target does not support the llvm."
+ << (Callee->getIntrinsicID() == Intrinsic::returnaddress ?
+ "return" : "frame") << "address intrinsic.\n";
+ CI->replaceAllUsesWith(ConstantPointerNull::get(
+ cast<PointerType>(CI->getType())));
+ break;
+
+ case Intrinsic::prefetch:
+ break; // Simply strip out prefetches on unsupported architectures
+
+ case Intrinsic::pcmarker:
+ break; // Simply strip out pcmarker on unsupported architectures
+ case Intrinsic::readcyclecounter: {
+ cerr << "WARNING: this target does not support the llvm.readcyclecoun"
+ << "ter intrinsic. It is being lowered to a constant 0\n";
+ CI->replaceAllUsesWith(ConstantInt::get(Type::Int64Ty, 0));
+ break;
+ }
+
+ case Intrinsic::dbg_stoppoint:
+ case Intrinsic::dbg_region_start:
+ case Intrinsic::dbg_region_end:
+ case Intrinsic::dbg_func_start:
+ case Intrinsic::dbg_declare:
+ break; // Simply strip out debugging intrinsics
+
+ case Intrinsic::eh_exception:
+ case Intrinsic::eh_selector:
+ CI->replaceAllUsesWith(Constant::getNullValue(CI->getType()));
+ break;
+
+ case Intrinsic::eh_typeid_for:
+ // Return something different to eh_selector.
+ CI->replaceAllUsesWith(ConstantInt::get(CI->getType(), 1));
+ break;
+
+ case Intrinsic::var_annotation:
+ break; // Strip out annotate intrinsic
+
+ case Intrinsic::memcpy_i32:
+ case Intrinsic::memcpy_i64: {
+ static Constant *MemcpyFCache = 0;
+ Value *Size = CI->getOperand(3);
+ const Type *IntPtr = TD.getIntPtrType();
+ if (Size->getType()->getPrimitiveSizeInBits() <
+ IntPtr->getPrimitiveSizeInBits())
+ Size = new ZExtInst(Size, IntPtr, "", CI);
+ else if (Size->getType()->getPrimitiveSizeInBits() >
+ IntPtr->getPrimitiveSizeInBits())
+ Size = new TruncInst(Size, IntPtr, "", CI);
+ Value *Ops[3];
+ Ops[0] = CI->getOperand(1);
+ Ops[1] = CI->getOperand(2);
+ Ops[2] = Size;
+ ReplaceCallWith("memcpy", CI, Ops, Ops+3, CI->getOperand(1)->getType(),
+ MemcpyFCache);
+ break;
+ }
+ case Intrinsic::memmove_i32:
+ case Intrinsic::memmove_i64: {
+ static Constant *MemmoveFCache = 0;
+ Value *Size = CI->getOperand(3);
+ const Type *IntPtr = TD.getIntPtrType();
+ if (Size->getType()->getPrimitiveSizeInBits() <
+ IntPtr->getPrimitiveSizeInBits())
+ Size = new ZExtInst(Size, IntPtr, "", CI);
+ else if (Size->getType()->getPrimitiveSizeInBits() >
+ IntPtr->getPrimitiveSizeInBits())
+ Size = new TruncInst(Size, IntPtr, "", CI);
+ Value *Ops[3];
+ Ops[0] = CI->getOperand(1);
+ Ops[1] = CI->getOperand(2);
+ Ops[2] = Size;
+ ReplaceCallWith("memmove", CI, Ops, Ops+3, CI->getOperand(1)->getType(),
+ MemmoveFCache);
+ break;
+ }
+ case Intrinsic::memset_i32:
+ case Intrinsic::memset_i64: {
+ static Constant *MemsetFCache = 0;
+ Value *Size = CI->getOperand(3);
+ const Type *IntPtr = TD.getIntPtrType();
+ if (Size->getType()->getPrimitiveSizeInBits() <
+ IntPtr->getPrimitiveSizeInBits())
+ Size = new ZExtInst(Size, IntPtr, "", CI);
+ else if (Size->getType()->getPrimitiveSizeInBits() >
+ IntPtr->getPrimitiveSizeInBits())
+ Size = new TruncInst(Size, IntPtr, "", CI);
+ Value *Ops[3];
+ Ops[0] = CI->getOperand(1);
+ // Extend the amount to i32.
+ Ops[1] = new ZExtInst(CI->getOperand(2), Type::Int32Ty, "", CI);
+ Ops[2] = Size;
+ ReplaceCallWith("memset", CI, Ops, Ops+3, CI->getOperand(1)->getType(),
+ MemsetFCache);
+ break;
+ }
+ case Intrinsic::sqrt_f32: {
+ static Constant *sqrtfFCache = 0;
+ ReplaceCallWith("sqrtf", CI, CI->op_begin()+1, CI->op_end(),
+ Type::FloatTy, sqrtfFCache);
+ break;
+ }
+ case Intrinsic::sqrt_f64: {
+ static Constant *sqrtFCache = 0;
+ ReplaceCallWith("sqrt", CI, CI->op_begin()+1, CI->op_end(),
+ Type::DoubleTy, sqrtFCache);
+ break;
+ }
+ }
+
+ assert(CI->use_empty() &&
+ "Lowering should have eliminated any uses of the intrinsic call!");
+ CI->eraseFromParent();
+}
diff --git a/lib/CodeGen/LLVMTargetMachine.cpp b/lib/CodeGen/LLVMTargetMachine.cpp
new file mode 100644
index 0000000..b72704b
--- /dev/null
+++ b/lib/CodeGen/LLVMTargetMachine.cpp
@@ -0,0 +1,205 @@
+//===-- LLVMTargetMachine.cpp - Implement the LLVMTargetMachine class -----===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file was developed by Chris Lattner and is distributed under
+// the University of Illinois Open Source License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements the LLVMTargetMachine class.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Target/TargetMachine.h"
+#include "llvm/PassManager.h"
+#include "llvm/Pass.h"
+#include "llvm/Assembly/PrintModulePass.h"
+#include "llvm/Analysis/LoopPass.h"
+#include "llvm/CodeGen/Passes.h"
+#include "llvm/Target/TargetOptions.h"
+#include "llvm/Transforms/Scalar.h"
+#include "llvm/Support/CommandLine.h"
+using namespace llvm;
+
+static cl::opt<bool> PrintLSR("print-lsr-output", cl::Hidden,
+ cl::desc("Print LLVM IR produced by the loop-reduce pass"));
+static cl::opt<bool> PrintISelInput("print-isel-input", cl::Hidden,
+ cl::desc("Print LLVM IR input to isel pass"));
+
+FileModel::Model
+LLVMTargetMachine::addPassesToEmitFile(FunctionPassManager &PM,
+ std::ostream &Out,
+ CodeGenFileType FileType,
+ bool Fast) {
+ // Standard LLVM-Level Passes.
+
+ // Run loop strength reduction before anything else.
+ if (!Fast) {
+ PM.add(createLoopStrengthReducePass(getTargetLowering()));
+ if (PrintLSR)
+ PM.add(new PrintFunctionPass("\n\n*** Code after LSR *** \n", &cerr));
+ }
+
+ // FIXME: Implement efficient support for garbage collection intrinsics.
+ PM.add(createLowerGCPass());
+
+ if (!ExceptionHandling)
+ PM.add(createLowerInvokePass(getTargetLowering()));
+
+ // Make sure that no unreachable blocks are instruction selected.
+ PM.add(createUnreachableBlockEliminationPass());
+
+ if (!Fast)
+ PM.add(createCodeGenPreparePass(getTargetLowering()));
+
+ if (PrintISelInput)
+ PM.add(new PrintFunctionPass("\n\n*** Final LLVM Code input to ISel *** \n",
+ &cerr));
+
+ // Ask the target for an isel.
+ if (addInstSelector(PM, Fast))
+ return FileModel::Error;
+
+ // Print the instruction selected machine code...
+ if (PrintMachineCode)
+ PM.add(createMachineFunctionPrinterPass(cerr));
+
+ // Perform register allocation to convert to a concrete x86 representation
+ PM.add(createRegisterAllocator());
+
+ if (PrintMachineCode)
+ PM.add(createMachineFunctionPrinterPass(cerr));
+
+ // Run post-ra passes.
+ if (addPostRegAlloc(PM, Fast) && PrintMachineCode)
+ PM.add(createMachineFunctionPrinterPass(cerr));
+
+ // Insert prolog/epilog code. Eliminate abstract frame index references...
+ PM.add(createPrologEpilogCodeInserter());
+
+ // Second pass scheduler.
+ if (!Fast)
+ PM.add(createPostRAScheduler());
+
+ // Branch folding must be run after regalloc and prolog/epilog insertion.
+ if (!Fast)
+ PM.add(createBranchFoldingPass(getEnableTailMergeDefault()));
+
+ // Fold redundant debug labels.
+ PM.add(createDebugLabelFoldingPass());
+
+ if (PrintMachineCode) // Print the register-allocated code
+ PM.add(createMachineFunctionPrinterPass(cerr));
+
+ if (addPreEmitPass(PM, Fast) && PrintMachineCode)
+ PM.add(createMachineFunctionPrinterPass(cerr));
+
+ switch (FileType) {
+ default:
+ break;
+ case TargetMachine::AssemblyFile:
+ if (addAssemblyEmitter(PM, Fast, Out))
+ return FileModel::Error;
+ return FileModel::AsmFile;
+ case TargetMachine::ObjectFile:
+ if (getMachOWriterInfo())
+ return FileModel::MachOFile;
+ else if (getELFWriterInfo())
+ return FileModel::ElfFile;
+ }
+
+ return FileModel::Error;
+}
+
+/// addPassesToEmitFileFinish - If the passes to emit the specified file had to
+/// be split up (e.g., to add an object writer pass), this method can be used to
+/// finish up adding passes to emit the file, if necessary.
+bool LLVMTargetMachine::addPassesToEmitFileFinish(FunctionPassManager &PM,
+ MachineCodeEmitter *MCE,
+ bool Fast) {
+ if (MCE)
+ addSimpleCodeEmitter(PM, Fast, *MCE);
+
+ // Delete machine code for this function
+ PM.add(createMachineCodeDeleter());
+
+ return false; // success!
+}
+
+/// addPassesToEmitMachineCode - Add passes to the specified pass manager to
+/// get machine code emitted. This uses a MachineCodeEmitter object to handle
+/// actually outputting the machine code and resolving things like the address
+/// of functions. This method should returns true if machine code emission is
+/// not supported.
+///
+bool LLVMTargetMachine::addPassesToEmitMachineCode(FunctionPassManager &PM,
+ MachineCodeEmitter &MCE,
+ bool Fast) {
+ // Standard LLVM-Level Passes.
+
+ // Run loop strength reduction before anything else.
+ if (!Fast) {
+ PM.add(createLoopStrengthReducePass(getTargetLowering()));
+ if (PrintLSR)
+ PM.add(new PrintFunctionPass("\n\n*** Code after LSR *** \n", &cerr));
+ }
+
+ // FIXME: Implement efficient support for garbage collection intrinsics.
+ PM.add(createLowerGCPass());
+
+ // FIXME: Implement the invoke/unwind instructions!
+ PM.add(createLowerInvokePass(getTargetLowering()));
+
+ // Make sure that no unreachable blocks are instruction selected.
+ PM.add(createUnreachableBlockEliminationPass());
+
+ if (!Fast)
+ PM.add(createCodeGenPreparePass(getTargetLowering()));
+
+ if (PrintISelInput)
+ PM.add(new PrintFunctionPass("\n\n*** Final LLVM Code input to ISel *** \n",
+ &cerr));
+
+ // Ask the target for an isel.
+ if (addInstSelector(PM, Fast))
+ return true;
+
+ // Print the instruction selected machine code...
+ if (PrintMachineCode)
+ PM.add(createMachineFunctionPrinterPass(cerr));
+
+ // Perform register allocation to convert to a concrete x86 representation
+ PM.add(createRegisterAllocator());
+
+ if (PrintMachineCode)
+ PM.add(createMachineFunctionPrinterPass(cerr));
+
+ // Run post-ra passes.
+ if (addPostRegAlloc(PM, Fast) && PrintMachineCode)
+ PM.add(createMachineFunctionPrinterPass(cerr));
+
+ // Insert prolog/epilog code. Eliminate abstract frame index references...
+ PM.add(createPrologEpilogCodeInserter());
+
+ if (PrintMachineCode) // Print the register-allocated code
+ PM.add(createMachineFunctionPrinterPass(cerr));
+
+ // Second pass scheduler.
+ if (!Fast)
+ PM.add(createPostRAScheduler());
+
+ // Branch folding must be run after regalloc and prolog/epilog insertion.
+ if (!Fast)
+ PM.add(createBranchFoldingPass(getEnableTailMergeDefault()));
+
+ if (addPreEmitPass(PM, Fast) && PrintMachineCode)
+ PM.add(createMachineFunctionPrinterPass(cerr));
+
+ addCodeEmitter(PM, Fast, MCE);
+
+ // Delete machine code for this function
+ PM.add(createMachineCodeDeleter());
+
+ return false; // success!
+}
diff --git a/lib/CodeGen/LiveInterval.cpp b/lib/CodeGen/LiveInterval.cpp
new file mode 100644
index 0000000..45c1dd0
--- /dev/null
+++ b/lib/CodeGen/LiveInterval.cpp
@@ -0,0 +1,526 @@
+//===-- LiveInterval.cpp - Live Interval Representation -------------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file was developed by the LLVM research group and is distributed under
+// the University of Illinois Open Source License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements the LiveRange and LiveInterval classes. Given some
+// numbering of each the machine instructions an interval [i, j) is said to be a
+// live interval for register v if there is no instruction with number j' > j
+// such that v is live at j' abd there is no instruction with number i' < i such
+// that v is live at i'. In this implementation intervals can have holes,
+// i.e. an interval might look like [1,20), [50,65), [1000,1001). Each
+// individual range is represented as an instance of LiveRange, and the whole
+// interval is represented as an instance of LiveInterval.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/CodeGen/LiveInterval.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/Support/Streams.h"
+#include "llvm/Target/MRegisterInfo.h"
+#include <algorithm>
+#include <map>
+#include <ostream>
+using namespace llvm;
+
+// An example for liveAt():
+//
+// this = [1,4), liveAt(0) will return false. The instruction defining this
+// spans slots [0,3]. The interval belongs to an spilled definition of the
+// variable it represents. This is because slot 1 is used (def slot) and spans
+// up to slot 3 (store slot).
+//
+bool LiveInterval::liveAt(unsigned I) const {
+ Ranges::const_iterator r = std::upper_bound(ranges.begin(), ranges.end(), I);
+
+ if (r == ranges.begin())
+ return false;
+
+ --r;
+ return r->contains(I);
+}
+
+// overlaps - Return true if the intersection of the two live intervals is
+// not empty.
+//
+// An example for overlaps():
+//
+// 0: A = ...
+// 4: B = ...
+// 8: C = A + B ;; last use of A
+//
+// The live intervals should look like:
+//
+// A = [3, 11)
+// B = [7, x)
+// C = [11, y)
+//
+// A->overlaps(C) should return false since we want to be able to join
+// A and C.
+//
+bool LiveInterval::overlapsFrom(const LiveInterval& other,
+ const_iterator StartPos) const {
+ const_iterator i = begin();
+ const_iterator ie = end();
+ const_iterator j = StartPos;
+ const_iterator je = other.end();
+
+ assert((StartPos->start <= i->start || StartPos == other.begin()) &&
+ StartPos != other.end() && "Bogus start position hint!");
+
+ if (i->start < j->start) {
+ i = std::upper_bound(i, ie, j->start);
+ if (i != ranges.begin()) --i;
+ } else if (j->start < i->start) {
+ ++StartPos;
+ if (StartPos != other.end() && StartPos->start <= i->start) {
+ assert(StartPos < other.end() && i < end());
+ j = std::upper_bound(j, je, i->start);
+ if (j != other.ranges.begin()) --j;
+ }
+ } else {
+ return true;
+ }
+
+ if (j == je) return false;
+
+ while (i != ie) {
+ if (i->start > j->start) {
+ std::swap(i, j);
+ std::swap(ie, je);
+ }
+
+ if (i->end > j->start)
+ return true;
+ ++i;
+ }
+
+ return false;
+}
+
+/// extendIntervalEndTo - This method is used when we want to extend the range
+/// specified by I to end at the specified endpoint. To do this, we should
+/// merge and eliminate all ranges that this will overlap with. The iterator is
+/// not invalidated.
+void LiveInterval::extendIntervalEndTo(Ranges::iterator I, unsigned NewEnd) {
+ assert(I != ranges.end() && "Not a valid interval!");
+ unsigned ValId = I->ValId;
+
+ // Search for the first interval that we can't merge with.
+ Ranges::iterator MergeTo = next(I);
+ for (; MergeTo != ranges.end() && NewEnd >= MergeTo->end; ++MergeTo) {
+ assert(MergeTo->ValId == ValId && "Cannot merge with differing values!");
+ }
+
+ // If NewEnd was in the middle of an interval, make sure to get its endpoint.
+ I->end = std::max(NewEnd, prior(MergeTo)->end);
+
+ // Erase any dead ranges.
+ ranges.erase(next(I), MergeTo);
+
+ // If the newly formed range now touches the range after it and if they have
+ // the same value number, merge the two ranges into one range.
+ Ranges::iterator Next = next(I);
+ if (Next != ranges.end() && Next->start <= I->end && Next->ValId == ValId) {
+ I->end = Next->end;
+ ranges.erase(Next);
+ }
+}
+
+
+/// extendIntervalStartTo - This method is used when we want to extend the range
+/// specified by I to start at the specified endpoint. To do this, we should
+/// merge and eliminate all ranges that this will overlap with.
+LiveInterval::Ranges::iterator
+LiveInterval::extendIntervalStartTo(Ranges::iterator I, unsigned NewStart) {
+ assert(I != ranges.end() && "Not a valid interval!");
+ unsigned ValId = I->ValId;
+
+ // Search for the first interval that we can't merge with.
+ Ranges::iterator MergeTo = I;
+ do {
+ if (MergeTo == ranges.begin()) {
+ I->start = NewStart;
+ ranges.erase(MergeTo, I);
+ return I;
+ }
+ assert(MergeTo->ValId == ValId && "Cannot merge with differing values!");
+ --MergeTo;
+ } while (NewStart <= MergeTo->start);
+
+ // If we start in the middle of another interval, just delete a range and
+ // extend that interval.
+ if (MergeTo->end >= NewStart && MergeTo->ValId == ValId) {
+ MergeTo->end = I->end;
+ } else {
+ // Otherwise, extend the interval right after.
+ ++MergeTo;
+ MergeTo->start = NewStart;
+ MergeTo->end = I->end;
+ }
+
+ ranges.erase(next(MergeTo), next(I));
+ return MergeTo;
+}
+
+LiveInterval::iterator
+LiveInterval::addRangeFrom(LiveRange LR, iterator From) {
+ unsigned Start = LR.start, End = LR.end;
+ iterator it = std::upper_bound(From, ranges.end(), Start);
+
+ // If the inserted interval starts in the middle or right at the end of
+ // another interval, just extend that interval to contain the range of LR.
+ if (it != ranges.begin()) {
+ iterator B = prior(it);
+ if (LR.ValId == B->ValId) {
+ if (B->start <= Start && B->end >= Start) {
+ extendIntervalEndTo(B, End);
+ return B;
+ }
+ } else {
+ // Check to make sure that we are not overlapping two live ranges with
+ // different ValId's.
+ assert(B->end <= Start &&
+ "Cannot overlap two LiveRanges with differing ValID's"
+ " (did you def the same reg twice in a MachineInstr?)");
+ }
+ }
+
+ // Otherwise, if this range ends in the middle of, or right next to, another
+ // interval, merge it into that interval.
+ if (it != ranges.end())
+ if (LR.ValId == it->ValId) {
+ if (it->start <= End) {
+ it = extendIntervalStartTo(it, Start);
+
+ // If LR is a complete superset of an interval, we may need to grow its
+ // endpoint as well.
+ if (End > it->end)
+ extendIntervalEndTo(it, End);
+ return it;
+ }
+ } else {
+ // Check to make sure that we are not overlapping two live ranges with
+ // different ValId's.
+ assert(it->start >= End &&
+ "Cannot overlap two LiveRanges with differing ValID's");
+ }
+
+ // Otherwise, this is just a new range that doesn't interact with anything.
+ // Insert it.
+ return ranges.insert(it, LR);
+}
+
+
+/// removeRange - Remove the specified range from this interval. Note that
+/// the range must already be in this interval in its entirety.
+void LiveInterval::removeRange(unsigned Start, unsigned End) {
+ // Find the LiveRange containing this span.
+ Ranges::iterator I = std::upper_bound(ranges.begin(), ranges.end(), Start);
+ assert(I != ranges.begin() && "Range is not in interval!");
+ --I;
+ assert(I->contains(Start) && I->contains(End-1) &&
+ "Range is not entirely in interval!");
+
+ // If the span we are removing is at the start of the LiveRange, adjust it.
+ if (I->start == Start) {
+ if (I->end == End)
+ ranges.erase(I); // Removed the whole LiveRange.
+ else
+ I->start = End;
+ return;
+ }
+
+ // Otherwise if the span we are removing is at the end of the LiveRange,
+ // adjust the other way.
+ if (I->end == End) {
+ I->end = Start;
+ return;
+ }
+
+ // Otherwise, we are splitting the LiveRange into two pieces.
+ unsigned OldEnd = I->end;
+ I->end = Start; // Trim the old interval.
+
+ // Insert the new one.
+ ranges.insert(next(I), LiveRange(End, OldEnd, I->ValId));
+}
+
+/// getLiveRangeContaining - Return the live range that contains the
+/// specified index, or null if there is none.
+LiveInterval::const_iterator
+LiveInterval::FindLiveRangeContaining(unsigned Idx) const {
+ const_iterator It = std::upper_bound(begin(), end(), Idx);
+ if (It != ranges.begin()) {
+ --It;
+ if (It->contains(Idx))
+ return It;
+ }
+
+ return end();
+}
+
+LiveInterval::iterator
+LiveInterval::FindLiveRangeContaining(unsigned Idx) {
+ iterator It = std::upper_bound(begin(), end(), Idx);
+ if (It != begin()) {
+ --It;
+ if (It->contains(Idx))
+ return It;
+ }
+
+ return end();
+}
+
+/// join - Join two live intervals (this, and other) together. This applies
+/// mappings to the value numbers in the LHS/RHS intervals as specified. If
+/// the intervals are not joinable, this aborts.
+void LiveInterval::join(LiveInterval &Other, int *LHSValNoAssignments,
+ int *RHSValNoAssignments,
+ SmallVector<std::pair<unsigned,
+ unsigned>, 16> &NewValueNumberInfo) {
+
+ // Try to do the least amount of work possible. In particular, if there are
+ // more liverange chunks in the other set than there are in the 'this' set,
+ // swap sets to merge the fewest chunks in possible.
+ //
+ // Also, if one range is a physreg and one is a vreg, we always merge from the
+ // vreg into the physreg, which leaves the vreg intervals pristine.
+ if ((Other.ranges.size() > ranges.size() &&
+ MRegisterInfo::isVirtualRegister(reg)) ||
+ MRegisterInfo::isPhysicalRegister(Other.reg)) {
+ swap(Other);
+ std::swap(LHSValNoAssignments, RHSValNoAssignments);
+ }
+
+ // Determine if any of our live range values are mapped. This is uncommon, so
+ // we want to avoid the interval scan if not.
+ bool MustMapCurValNos = false;
+ for (unsigned i = 0, e = getNumValNums(); i != e; ++i) {
+ if (ValueNumberInfo[i].first == ~2U) continue; // tombstone value #
+ if (i != (unsigned)LHSValNoAssignments[i]) {
+ MustMapCurValNos = true;
+ break;
+ }
+ }
+
+ // If we have to apply a mapping to our base interval assignment, rewrite it
+ // now.
+ if (MustMapCurValNos) {
+ // Map the first live range.
+ iterator OutIt = begin();
+ OutIt->ValId = LHSValNoAssignments[OutIt->ValId];
+ ++OutIt;
+ for (iterator I = OutIt, E = end(); I != E; ++I) {
+ OutIt->ValId = LHSValNoAssignments[I->ValId];
+
+ // If this live range has the same value # as its immediate predecessor,
+ // and if they are neighbors, remove one LiveRange. This happens when we
+ // have [0,3:0)[4,7:1) and map 0/1 onto the same value #.
+ if (OutIt->ValId == (OutIt-1)->ValId && (OutIt-1)->end == OutIt->start) {
+ (OutIt-1)->end = OutIt->end;
+ } else {
+ if (I != OutIt) {
+ OutIt->start = I->start;
+ OutIt->end = I->end;
+ }
+
+ // Didn't merge, on to the next one.
+ ++OutIt;
+ }
+ }
+
+ // If we merge some live ranges, chop off the end.
+ ranges.erase(OutIt, end());
+ }
+
+ // Okay, now insert the RHS live ranges into the LHS.
+ iterator InsertPos = begin();
+ for (iterator I = Other.begin(), E = Other.end(); I != E; ++I) {
+ // Map the ValId in the other live range to the current live range.
+ I->ValId = RHSValNoAssignments[I->ValId];
+ InsertPos = addRangeFrom(*I, InsertPos);
+ }
+
+ ValueNumberInfo.clear();
+ ValueNumberInfo.append(NewValueNumberInfo.begin(), NewValueNumberInfo.end());
+ weight += Other.weight;
+ if (Other.preference && !preference)
+ preference = Other.preference;
+}
+
+/// MergeRangesInAsValue - Merge all of the intervals in RHS into this live
+/// interval as the specified value number. The LiveRanges in RHS are
+/// allowed to overlap with LiveRanges in the current interval, but only if
+/// the overlapping LiveRanges have the specified value number.
+void LiveInterval::MergeRangesInAsValue(const LiveInterval &RHS,
+ unsigned LHSValNo) {
+ // TODO: Make this more efficient.
+ iterator InsertPos = begin();
+ for (const_iterator I = RHS.begin(), E = RHS.end(); I != E; ++I) {
+ // Map the ValId in the other live range to the current live range.
+ LiveRange Tmp = *I;
+ Tmp.ValId = LHSValNo;
+ InsertPos = addRangeFrom(Tmp, InsertPos);
+ }
+}
+
+
+/// MergeInClobberRanges - For any live ranges that are not defined in the
+/// current interval, but are defined in the Clobbers interval, mark them
+/// used with an unknown definition value.
+void LiveInterval::MergeInClobberRanges(const LiveInterval &Clobbers) {
+ if (Clobbers.begin() == Clobbers.end()) return;
+
+ // Find a value # to use for the clobber ranges. If there is already a value#
+ // for unknown values, use it.
+ // FIXME: Use a single sentinal number for these!
+ unsigned ClobberValNo = getNextValue(~0U, 0);
+
+ iterator IP = begin();
+ for (const_iterator I = Clobbers.begin(), E = Clobbers.end(); I != E; ++I) {
+ unsigned Start = I->start, End = I->end;
+ IP = std::upper_bound(IP, end(), Start);
+
+ // If the start of this range overlaps with an existing liverange, trim it.
+ if (IP != begin() && IP[-1].end > Start) {
+ Start = IP[-1].end;
+ // Trimmed away the whole range?
+ if (Start >= End) continue;
+ }
+ // If the end of this range overlaps with an existing liverange, trim it.
+ if (IP != end() && End > IP->start) {
+ End = IP->start;
+ // If this trimmed away the whole range, ignore it.
+ if (Start == End) continue;
+ }
+
+ // Insert the clobber interval.
+ IP = addRangeFrom(LiveRange(Start, End, ClobberValNo), IP);
+ }
+}
+
+/// MergeValueNumberInto - This method is called when two value nubmers
+/// are found to be equivalent. This eliminates V1, replacing all
+/// LiveRanges with the V1 value number with the V2 value number. This can
+/// cause merging of V1/V2 values numbers and compaction of the value space.
+void LiveInterval::MergeValueNumberInto(unsigned V1, unsigned V2) {
+ assert(V1 != V2 && "Identical value#'s are always equivalent!");
+
+ // This code actually merges the (numerically) larger value number into the
+ // smaller value number, which is likely to allow us to compactify the value
+ // space. The only thing we have to be careful of is to preserve the
+ // instruction that defines the result value.
+
+ // Make sure V2 is smaller than V1.
+ if (V1 < V2) {
+ setValueNumberInfo(V1, getValNumInfo(V2));
+ std::swap(V1, V2);
+ }
+
+ // Merge V1 live ranges into V2.
+ for (iterator I = begin(); I != end(); ) {
+ iterator LR = I++;
+ if (LR->ValId != V1) continue; // Not a V1 LiveRange.
+
+ // Okay, we found a V1 live range. If it had a previous, touching, V2 live
+ // range, extend it.
+ if (LR != begin()) {
+ iterator Prev = LR-1;
+ if (Prev->ValId == V2 && Prev->end == LR->start) {
+ Prev->end = LR->end;
+
+ // Erase this live-range.
+ ranges.erase(LR);
+ I = Prev+1;
+ LR = Prev;
+ }
+ }
+
+ // Okay, now we have a V1 or V2 live range that is maximally merged forward.
+ // Ensure that it is a V2 live-range.
+ LR->ValId = V2;
+
+ // If we can merge it into later V2 live ranges, do so now. We ignore any
+ // following V1 live ranges, as they will be merged in subsequent iterations
+ // of the loop.
+ if (I != end()) {
+ if (I->start == LR->end && I->ValId == V2) {
+ LR->end = I->end;
+ ranges.erase(I);
+ I = LR+1;
+ }
+ }
+ }
+
+ // Now that V1 is dead, remove it. If it is the largest value number, just
+ // nuke it (and any other deleted values neighboring it), otherwise mark it as
+ // ~1U so it can be nuked later.
+ if (V1 == getNumValNums()-1) {
+ do {
+ ValueNumberInfo.pop_back();
+ } while (ValueNumberInfo.back().first == ~1U);
+ } else {
+ ValueNumberInfo[V1].first = ~1U;
+ }
+}
+
+unsigned LiveInterval::getSize() const {
+ unsigned Sum = 0;
+ for (const_iterator I = begin(), E = end(); I != E; ++I)
+ Sum += I->end - I->start;
+ return Sum;
+}
+
+std::ostream& llvm::operator<<(std::ostream& os, const LiveRange &LR) {
+ return os << '[' << LR.start << ',' << LR.end << ':' << LR.ValId << ")";
+}
+
+void LiveRange::dump() const {
+ cerr << *this << "\n";
+}
+
+void LiveInterval::print(std::ostream &OS, const MRegisterInfo *MRI) const {
+ if (MRI && MRegisterInfo::isPhysicalRegister(reg))
+ OS << MRI->getName(reg);
+ else
+ OS << "%reg" << reg;
+
+ OS << ',' << weight;
+
+ if (empty())
+ OS << "EMPTY";
+ else {
+ OS << " = ";
+ for (LiveInterval::Ranges::const_iterator I = ranges.begin(),
+ E = ranges.end(); I != E; ++I)
+ OS << *I;
+ }
+
+ // Print value number info.
+ if (getNumValNums()) {
+ OS << " ";
+ for (unsigned i = 0; i != getNumValNums(); ++i) {
+ if (i) OS << " ";
+ OS << i << "@";
+ if (ValueNumberInfo[i].first == ~0U) {
+ OS << "?";
+ } else {
+ OS << ValueNumberInfo[i].first;
+ }
+ }
+ }
+}
+
+void LiveInterval::dump() const {
+ cerr << *this << "\n";
+}
+
+
+void LiveRange::print(std::ostream &os) const {
+ os << *this;
+}
diff --git a/lib/CodeGen/LiveIntervalAnalysis.cpp b/lib/CodeGen/LiveIntervalAnalysis.cpp
new file mode 100644
index 0000000..369493f
--- /dev/null
+++ b/lib/CodeGen/LiveIntervalAnalysis.cpp
@@ -0,0 +1,692 @@
+//===-- LiveIntervalAnalysis.cpp - Live Interval Analysis -----------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file was developed by the LLVM research group and is distributed under
+// the University of Illinois Open Source License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements the LiveInterval analysis pass which is used
+// by the Linear Scan Register allocator. This pass linearizes the
+// basic blocks of the function in DFS order and uses the
+// LiveVariables pass to conservatively compute live intervals for
+// each virtual and physical register.
+//
+//===----------------------------------------------------------------------===//
+
+#define DEBUG_TYPE "liveintervals"
+#include "llvm/CodeGen/LiveIntervalAnalysis.h"
+#include "VirtRegMap.h"
+#include "llvm/Value.h"
+#include "llvm/Analysis/LoopInfo.h"
+#include "llvm/CodeGen/LiveVariables.h"
+#include "llvm/CodeGen/MachineFrameInfo.h"
+#include "llvm/CodeGen/MachineInstr.h"
+#include "llvm/CodeGen/Passes.h"
+#include "llvm/CodeGen/SSARegMap.h"
+#include "llvm/Target/MRegisterInfo.h"
+#include "llvm/Target/TargetInstrInfo.h"
+#include "llvm/Target/TargetMachine.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/ADT/SmallSet.h"
+#include "llvm/ADT/Statistic.h"
+#include "llvm/ADT/STLExtras.h"
+#include <algorithm>
+#include <cmath>
+using namespace llvm;
+
+STATISTIC(numIntervals, "Number of original intervals");
+STATISTIC(numIntervalsAfter, "Number of intervals after coalescing");
+STATISTIC(numFolded , "Number of loads/stores folded into instructions");
+
+char LiveIntervals::ID = 0;
+namespace {
+ RegisterPass<LiveIntervals> X("liveintervals", "Live Interval Analysis");
+}
+
+void LiveIntervals::getAnalysisUsage(AnalysisUsage &AU) const {
+ AU.addPreserved<LiveVariables>();
+ AU.addRequired<LiveVariables>();
+ AU.addPreservedID(PHIEliminationID);
+ AU.addRequiredID(PHIEliminationID);
+ AU.addRequiredID(TwoAddressInstructionPassID);
+ AU.addRequired<LoopInfo>();
+ MachineFunctionPass::getAnalysisUsage(AU);
+}
+
+void LiveIntervals::releaseMemory() {
+ mi2iMap_.clear();
+ i2miMap_.clear();
+ r2iMap_.clear();
+}
+
+/// runOnMachineFunction - Register allocate the whole function
+///
+bool LiveIntervals::runOnMachineFunction(MachineFunction &fn) {
+ mf_ = &fn;
+ tm_ = &fn.getTarget();
+ mri_ = tm_->getRegisterInfo();
+ tii_ = tm_->getInstrInfo();
+ lv_ = &getAnalysis<LiveVariables>();
+ allocatableRegs_ = mri_->getAllocatableSet(fn);
+
+ // Number MachineInstrs and MachineBasicBlocks.
+ // Initialize MBB indexes to a sentinal.
+ MBB2IdxMap.resize(mf_->getNumBlockIDs(), ~0U);
+
+ unsigned MIIndex = 0;
+ for (MachineFunction::iterator MBB = mf_->begin(), E = mf_->end();
+ MBB != E; ++MBB) {
+ // Set the MBB2IdxMap entry for this MBB.
+ MBB2IdxMap[MBB->getNumber()] = MIIndex;
+
+ for (MachineBasicBlock::iterator I = MBB->begin(), E = MBB->end();
+ I != E; ++I) {
+ bool inserted = mi2iMap_.insert(std::make_pair(I, MIIndex)).second;
+ assert(inserted && "multiple MachineInstr -> index mappings");
+ i2miMap_.push_back(I);
+ MIIndex += InstrSlots::NUM;
+ }
+ }
+
+ computeIntervals();
+
+ numIntervals += getNumIntervals();
+
+ DOUT << "********** INTERVALS **********\n";
+ for (iterator I = begin(), E = end(); I != E; ++I) {
+ I->second.print(DOUT, mri_);
+ DOUT << "\n";
+ }
+
+ numIntervalsAfter += getNumIntervals();
+ DEBUG(dump());
+ return true;
+}
+
+/// print - Implement the dump method.
+void LiveIntervals::print(std::ostream &O, const Module* ) const {
+ O << "********** INTERVALS **********\n";
+ for (const_iterator I = begin(), E = end(); I != E; ++I) {
+ I->second.print(DOUT, mri_);
+ DOUT << "\n";
+ }
+
+ O << "********** MACHINEINSTRS **********\n";
+ for (MachineFunction::iterator mbbi = mf_->begin(), mbbe = mf_->end();
+ mbbi != mbbe; ++mbbi) {
+ O << ((Value*)mbbi->getBasicBlock())->getName() << ":\n";
+ for (MachineBasicBlock::iterator mii = mbbi->begin(),
+ mie = mbbi->end(); mii != mie; ++mii) {
+ O << getInstructionIndex(mii) << '\t' << *mii;
+ }
+ }
+}
+
+// Not called?
+/// CreateNewLiveInterval - Create a new live interval with the given live
+/// ranges. The new live interval will have an infinite spill weight.
+LiveInterval&
+LiveIntervals::CreateNewLiveInterval(const LiveInterval *LI,
+ const std::vector<LiveRange> &LRs) {
+ const TargetRegisterClass *RC = mf_->getSSARegMap()->getRegClass(LI->reg);
+
+ // Create a new virtual register for the spill interval.
+ unsigned NewVReg = mf_->getSSARegMap()->createVirtualRegister(RC);
+
+ // Replace the old virtual registers in the machine operands with the shiny
+ // new one.
+ for (std::vector<LiveRange>::const_iterator
+ I = LRs.begin(), E = LRs.end(); I != E; ++I) {
+ unsigned Index = getBaseIndex(I->start);
+ unsigned End = getBaseIndex(I->end - 1) + InstrSlots::NUM;
+
+ for (; Index != End; Index += InstrSlots::NUM) {
+ // Skip deleted instructions
+ while (Index != End && !getInstructionFromIndex(Index))
+ Index += InstrSlots::NUM;
+
+ if (Index == End) break;
+
+ MachineInstr *MI = getInstructionFromIndex(Index);
+
+ for (unsigned J = 0, e = MI->getNumOperands(); J != e; ++J) {
+ MachineOperand &MOp = MI->getOperand(J);
+ if (MOp.isRegister() && MOp.getReg() == LI->reg)
+ MOp.setReg(NewVReg);
+ }
+ }
+ }
+
+ LiveInterval &NewLI = getOrCreateInterval(NewVReg);
+
+ // The spill weight is now infinity as it cannot be spilled again
+ NewLI.weight = float(HUGE_VAL);
+
+ for (std::vector<LiveRange>::const_iterator
+ I = LRs.begin(), E = LRs.end(); I != E; ++I) {
+ DOUT << " Adding live range " << *I << " to new interval\n";
+ NewLI.addRange(*I);
+ }
+
+ DOUT << "Created new live interval " << NewLI << "\n";
+ return NewLI;
+}
+
+std::vector<LiveInterval*> LiveIntervals::
+addIntervalsForSpills(const LiveInterval &li, VirtRegMap &vrm, int slot) {
+ // since this is called after the analysis is done we don't know if
+ // LiveVariables is available
+ lv_ = getAnalysisToUpdate<LiveVariables>();
+
+ std::vector<LiveInterval*> added;
+
+ assert(li.weight != HUGE_VALF &&
+ "attempt to spill already spilled interval!");
+
+ DOUT << "\t\t\t\tadding intervals for spills for interval: ";
+ li.print(DOUT, mri_);
+ DOUT << '\n';
+
+ const TargetRegisterClass* rc = mf_->getSSARegMap()->getRegClass(li.reg);
+
+ for (LiveInterval::Ranges::const_iterator
+ i = li.ranges.begin(), e = li.ranges.end(); i != e; ++i) {
+ unsigned index = getBaseIndex(i->start);
+ unsigned end = getBaseIndex(i->end-1) + InstrSlots::NUM;
+ for (; index != end; index += InstrSlots::NUM) {
+ // skip deleted instructions
+ while (index != end && !getInstructionFromIndex(index))
+ index += InstrSlots::NUM;
+ if (index == end) break;
+
+ MachineInstr *MI = getInstructionFromIndex(index);
+
+ RestartInstruction:
+ for (unsigned i = 0; i != MI->getNumOperands(); ++i) {
+ MachineOperand& mop = MI->getOperand(i);
+ if (mop.isRegister() && mop.getReg() == li.reg) {
+ MachineInstr *fmi = li.remat ? NULL
+ : mri_->foldMemoryOperand(MI, i, slot);
+ if (fmi) {
+ // Attempt to fold the memory reference into the instruction. If we
+ // can do this, we don't need to insert spill code.
+ if (lv_)
+ lv_->instructionChanged(MI, fmi);
+ MachineBasicBlock &MBB = *MI->getParent();
+ vrm.virtFolded(li.reg, MI, i, fmi);
+ mi2iMap_.erase(MI);
+ i2miMap_[index/InstrSlots::NUM] = fmi;
+ mi2iMap_[fmi] = index;
+ MI = MBB.insert(MBB.erase(MI), fmi);
+ ++numFolded;
+ // Folding the load/store can completely change the instruction in
+ // unpredictable ways, rescan it from the beginning.
+ goto RestartInstruction;
+ } else {
+ // Create a new virtual register for the spill interval.
+ unsigned NewVReg = mf_->getSSARegMap()->createVirtualRegister(rc);
+
+ // Scan all of the operands of this instruction rewriting operands
+ // to use NewVReg instead of li.reg as appropriate. We do this for
+ // two reasons:
+ //
+ // 1. If the instr reads the same spilled vreg multiple times, we
+ // want to reuse the NewVReg.
+ // 2. If the instr is a two-addr instruction, we are required to
+ // keep the src/dst regs pinned.
+ //
+ // Keep track of whether we replace a use and/or def so that we can
+ // create the spill interval with the appropriate range.
+ mop.setReg(NewVReg);
+
+ bool HasUse = mop.isUse();
+ bool HasDef = mop.isDef();
+ for (unsigned j = i+1, e = MI->getNumOperands(); j != e; ++j) {
+ if (MI->getOperand(j).isReg() &&
+ MI->getOperand(j).getReg() == li.reg) {
+ MI->getOperand(j).setReg(NewVReg);
+ HasUse |= MI->getOperand(j).isUse();
+ HasDef |= MI->getOperand(j).isDef();
+ }
+ }
+
+ // create a new register for this spill
+ vrm.grow();
+ if (li.remat)
+ vrm.setVirtIsReMaterialized(NewVReg, li.remat);
+ vrm.assignVirt2StackSlot(NewVReg, slot);
+ LiveInterval &nI = getOrCreateInterval(NewVReg);
+ nI.remat = li.remat;
+ assert(nI.empty());
+
+ // the spill weight is now infinity as it
+ // cannot be spilled again
+ nI.weight = HUGE_VALF;
+
+ if (HasUse) {
+ LiveRange LR(getLoadIndex(index), getUseIndex(index),
+ nI.getNextValue(~0U, 0));
+ DOUT << " +" << LR;
+ nI.addRange(LR);
+ }
+ if (HasDef) {
+ LiveRange LR(getDefIndex(index), getStoreIndex(index),
+ nI.getNextValue(~0U, 0));
+ DOUT << " +" << LR;
+ nI.addRange(LR);
+ }
+
+ added.push_back(&nI);
+
+ // update live variables if it is available
+ if (lv_)
+ lv_->addVirtualRegisterKilled(NewVReg, MI);
+
+ DOUT << "\t\t\t\tadded new interval: ";
+ nI.print(DOUT, mri_);
+ DOUT << '\n';
+ }
+ }
+ }
+ }
+ }
+
+ return added;
+}
+
+void LiveIntervals::printRegName(unsigned reg) const {
+ if (MRegisterInfo::isPhysicalRegister(reg))
+ cerr << mri_->getName(reg);
+ else
+ cerr << "%reg" << reg;
+}
+
+/// isReDefinedByTwoAddr - Returns true if the Reg re-definition is due to
+/// two addr elimination.
+static bool isReDefinedByTwoAddr(MachineInstr *MI, unsigned Reg,
+ const TargetInstrInfo *TII) {
+ for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
+ MachineOperand &MO1 = MI->getOperand(i);
+ if (MO1.isRegister() && MO1.isDef() && MO1.getReg() == Reg) {
+ for (unsigned j = i+1; j < e; ++j) {
+ MachineOperand &MO2 = MI->getOperand(j);
+ if (MO2.isRegister() && MO2.isUse() && MO2.getReg() == Reg &&
+ MI->getInstrDescriptor()->
+ getOperandConstraint(j, TOI::TIED_TO) == (int)i)
+ return true;
+ }
+ }
+ }
+ return false;
+}
+
+void LiveIntervals::handleVirtualRegisterDef(MachineBasicBlock *mbb,
+ MachineBasicBlock::iterator mi,
+ unsigned MIIdx,
+ LiveInterval &interval) {
+ DOUT << "\t\tregister: "; DEBUG(printRegName(interval.reg));
+ LiveVariables::VarInfo& vi = lv_->getVarInfo(interval.reg);
+
+ // Virtual registers may be defined multiple times (due to phi
+ // elimination and 2-addr elimination). Much of what we do only has to be
+ // done once for the vreg. We use an empty interval to detect the first
+ // time we see a vreg.
+ if (interval.empty()) {
+ // Remember if the definition can be rematerialized. All load's from fixed
+ // stack slots are re-materializable. The target may permit other
+ // instructions to be re-materialized as well.
+ int FrameIdx = 0;
+ if (vi.DefInst &&
+ (tii_->isTriviallyReMaterializable(vi.DefInst) ||
+ (tii_->isLoadFromStackSlot(vi.DefInst, FrameIdx) &&
+ mf_->getFrameInfo()->isFixedObjectIndex(FrameIdx))))
+ interval.remat = vi.DefInst;
+
+ // Get the Idx of the defining instructions.
+ unsigned defIndex = getDefIndex(MIIdx);
+
+ unsigned ValNum;
+ unsigned SrcReg, DstReg;
+ if (!tii_->isMoveInstr(*mi, SrcReg, DstReg))
+ ValNum = interval.getNextValue(~0U, 0);
+ else
+ ValNum = interval.getNextValue(defIndex, SrcReg);
+
+ assert(ValNum == 0 && "First value in interval is not 0?");
+ ValNum = 0; // Clue in the optimizer.
+
+ // Loop over all of the blocks that the vreg is defined in. There are
+ // two cases we have to handle here. The most common case is a vreg
+ // whose lifetime is contained within a basic block. In this case there
+ // will be a single kill, in MBB, which comes after the definition.
+ if (vi.Kills.size() == 1 && vi.Kills[0]->getParent() == mbb) {
+ // FIXME: what about dead vars?
+ unsigned killIdx;
+ if (vi.Kills[0] != mi)
+ killIdx = getUseIndex(getInstructionIndex(vi.Kills[0]))+1;
+ else
+ killIdx = defIndex+1;
+
+ // If the kill happens after the definition, we have an intra-block
+ // live range.
+ if (killIdx > defIndex) {
+ assert(vi.AliveBlocks.none() &&
+ "Shouldn't be alive across any blocks!");
+ LiveRange LR(defIndex, killIdx, ValNum);
+ interval.addRange(LR);
+ DOUT << " +" << LR << "\n";
+ return;
+ }
+ }
+
+ // The other case we handle is when a virtual register lives to the end
+ // of the defining block, potentially live across some blocks, then is
+ // live into some number of blocks, but gets killed. Start by adding a
+ // range that goes from this definition to the end of the defining block.
+ LiveRange NewLR(defIndex,
+ getInstructionIndex(&mbb->back()) + InstrSlots::NUM,
+ ValNum);
+ DOUT << " +" << NewLR;
+ interval.addRange(NewLR);
+
+ // Iterate over all of the blocks that the variable is completely
+ // live in, adding [insrtIndex(begin), instrIndex(end)+4) to the
+ // live interval.
+ for (unsigned i = 0, e = vi.AliveBlocks.size(); i != e; ++i) {
+ if (vi.AliveBlocks[i]) {
+ MachineBasicBlock *MBB = mf_->getBlockNumbered(i);
+ if (!MBB->empty()) {
+ LiveRange LR(getMBBStartIdx(i),
+ getInstructionIndex(&MBB->back()) + InstrSlots::NUM,
+ ValNum);
+ interval.addRange(LR);
+ DOUT << " +" << LR;
+ }
+ }
+ }
+
+ // Finally, this virtual register is live from the start of any killing
+ // block to the 'use' slot of the killing instruction.
+ for (unsigned i = 0, e = vi.Kills.size(); i != e; ++i) {
+ MachineInstr *Kill = vi.Kills[i];
+ LiveRange LR(getMBBStartIdx(Kill->getParent()),
+ getUseIndex(getInstructionIndex(Kill))+1,
+ ValNum);
+ interval.addRange(LR);
+ DOUT << " +" << LR;
+ }
+
+ } else {
+ // Can no longer safely assume definition is rematerializable.
+ interval.remat = NULL;
+
+ // If this is the second time we see a virtual register definition, it
+ // must be due to phi elimination or two addr elimination. If this is
+ // the result of two address elimination, then the vreg is one of the
+ // def-and-use register operand.
+ if (isReDefinedByTwoAddr(mi, interval.reg, tii_)) {
+ // If this is a two-address definition, then we have already processed
+ // the live range. The only problem is that we didn't realize there
+ // are actually two values in the live interval. Because of this we
+ // need to take the LiveRegion that defines this register and split it
+ // into two values.
+ unsigned DefIndex = getDefIndex(getInstructionIndex(vi.DefInst));
+ unsigned RedefIndex = getDefIndex(MIIdx);
+
+ // Delete the initial value, which should be short and continuous,
+ // because the 2-addr copy must be in the same MBB as the redef.
+ interval.removeRange(DefIndex, RedefIndex);
+
+ // Two-address vregs should always only be redefined once. This means
+ // that at this point, there should be exactly one value number in it.
+ assert(interval.containsOneValue() && "Unexpected 2-addr liveint!");
+
+ // The new value number (#1) is defined by the instruction we claimed
+ // defined value #0.
+ unsigned ValNo = interval.getNextValue(0, 0);
+ interval.setValueNumberInfo(1, interval.getValNumInfo(0));
+
+ // Value#0 is now defined by the 2-addr instruction.
+ interval.setValueNumberInfo(0, std::make_pair(~0U, 0U));
+
+ // Add the new live interval which replaces the range for the input copy.
+ LiveRange LR(DefIndex, RedefIndex, ValNo);
+ DOUT << " replace range with " << LR;
+ interval.addRange(LR);
+
+ // If this redefinition is dead, we need to add a dummy unit live
+ // range covering the def slot.
+ if (lv_->RegisterDefIsDead(mi, interval.reg))
+ interval.addRange(LiveRange(RedefIndex, RedefIndex+1, 0));
+
+ DOUT << " RESULT: ";
+ interval.print(DOUT, mri_);
+
+ } else {
+ // Otherwise, this must be because of phi elimination. If this is the
+ // first redefinition of the vreg that we have seen, go back and change
+ // the live range in the PHI block to be a different value number.
+ if (interval.containsOneValue()) {
+ assert(vi.Kills.size() == 1 &&
+ "PHI elimination vreg should have one kill, the PHI itself!");
+
+ // Remove the old range that we now know has an incorrect number.
+ MachineInstr *Killer = vi.Kills[0];
+ unsigned Start = getMBBStartIdx(Killer->getParent());
+ unsigned End = getUseIndex(getInstructionIndex(Killer))+1;
+ DOUT << " Removing [" << Start << "," << End << "] from: ";
+ interval.print(DOUT, mri_); DOUT << "\n";
+ interval.removeRange(Start, End);
+ DOUT << " RESULT: "; interval.print(DOUT, mri_);
+
+ // Replace the interval with one of a NEW value number. Note that this
+ // value number isn't actually defined by an instruction, weird huh? :)
+ LiveRange LR(Start, End, interval.getNextValue(~0U, 0));
+ DOUT << " replace range with " << LR;
+ interval.addRange(LR);
+ DOUT << " RESULT: "; interval.print(DOUT, mri_);
+ }
+
+ // In the case of PHI elimination, each variable definition is only
+ // live until the end of the block. We've already taken care of the
+ // rest of the live range.
+ unsigned defIndex = getDefIndex(MIIdx);
+
+ unsigned ValNum;
+ unsigned SrcReg, DstReg;
+ if (!tii_->isMoveInstr(*mi, SrcReg, DstReg))
+ ValNum = interval.getNextValue(~0U, 0);
+ else
+ ValNum = interval.getNextValue(defIndex, SrcReg);
+
+ LiveRange LR(defIndex,
+ getInstructionIndex(&mbb->back()) + InstrSlots::NUM, ValNum);
+ interval.addRange(LR);
+ DOUT << " +" << LR;
+ }
+ }
+
+ DOUT << '\n';
+}
+
+void LiveIntervals::handlePhysicalRegisterDef(MachineBasicBlock *MBB,
+ MachineBasicBlock::iterator mi,
+ unsigned MIIdx,
+ LiveInterval &interval,
+ unsigned SrcReg) {
+ // A physical register cannot be live across basic block, so its
+ // lifetime must end somewhere in its defining basic block.
+ DOUT << "\t\tregister: "; DEBUG(printRegName(interval.reg));
+
+ unsigned baseIndex = MIIdx;
+ unsigned start = getDefIndex(baseIndex);
+ unsigned end = start;
+
+ // If it is not used after definition, it is considered dead at
+ // the instruction defining it. Hence its interval is:
+ // [defSlot(def), defSlot(def)+1)
+ if (lv_->RegisterDefIsDead(mi, interval.reg)) {
+ DOUT << " dead";
+ end = getDefIndex(start) + 1;
+ goto exit;
+ }
+
+ // If it is not dead on definition, it must be killed by a
+ // subsequent instruction. Hence its interval is:
+ // [defSlot(def), useSlot(kill)+1)
+ while (++mi != MBB->end()) {
+ baseIndex += InstrSlots::NUM;
+ if (lv_->KillsRegister(mi, interval.reg)) {
+ DOUT << " killed";
+ end = getUseIndex(baseIndex) + 1;
+ goto exit;
+ } else if (lv_->ModifiesRegister(mi, interval.reg)) {
+ // Another instruction redefines the register before it is ever read.
+ // Then the register is essentially dead at the instruction that defines
+ // it. Hence its interval is:
+ // [defSlot(def), defSlot(def)+1)
+ DOUT << " dead";
+ end = getDefIndex(start) + 1;
+ goto exit;
+ }
+ }
+
+ // The only case we should have a dead physreg here without a killing or
+ // instruction where we know it's dead is if it is live-in to the function
+ // and never used.
+ assert(!SrcReg && "physreg was not killed in defining block!");
+ end = getDefIndex(start) + 1; // It's dead.
+
+exit:
+ assert(start < end && "did not find end of interval?");
+
+ // Already exists? Extend old live interval.
+ LiveInterval::iterator OldLR = interval.FindLiveRangeContaining(start);
+ unsigned Id = (OldLR != interval.end())
+ ? OldLR->ValId
+ : interval.getNextValue(SrcReg != 0 ? start : ~0U, SrcReg);
+ LiveRange LR(start, end, Id);
+ interval.addRange(LR);
+ DOUT << " +" << LR << '\n';
+}
+
+void LiveIntervals::handleRegisterDef(MachineBasicBlock *MBB,
+ MachineBasicBlock::iterator MI,
+ unsigned MIIdx,
+ unsigned reg) {
+ if (MRegisterInfo::isVirtualRegister(reg))
+ handleVirtualRegisterDef(MBB, MI, MIIdx, getOrCreateInterval(reg));
+ else if (allocatableRegs_[reg]) {
+ unsigned SrcReg, DstReg;
+ if (!tii_->isMoveInstr(*MI, SrcReg, DstReg))
+ SrcReg = 0;
+ handlePhysicalRegisterDef(MBB, MI, MIIdx, getOrCreateInterval(reg), SrcReg);
+ // Def of a register also defines its sub-registers.
+ for (const unsigned* AS = mri_->getSubRegisters(reg); *AS; ++AS)
+ // Avoid processing some defs more than once.
+ if (!MI->findRegisterDefOperand(*AS))
+ handlePhysicalRegisterDef(MBB, MI, MIIdx, getOrCreateInterval(*AS), 0);
+ }
+}
+
+void LiveIntervals::handleLiveInRegister(MachineBasicBlock *MBB,
+ unsigned MIIdx,
+ LiveInterval &interval, bool isAlias) {
+ DOUT << "\t\tlivein register: "; DEBUG(printRegName(interval.reg));
+
+ // Look for kills, if it reaches a def before it's killed, then it shouldn't
+ // be considered a livein.
+ MachineBasicBlock::iterator mi = MBB->begin();
+ unsigned baseIndex = MIIdx;
+ unsigned start = baseIndex;
+ unsigned end = start;
+ while (mi != MBB->end()) {
+ if (lv_->KillsRegister(mi, interval.reg)) {
+ DOUT << " killed";
+ end = getUseIndex(baseIndex) + 1;
+ goto exit;
+ } else if (lv_->ModifiesRegister(mi, interval.reg)) {
+ // Another instruction redefines the register before it is ever read.
+ // Then the register is essentially dead at the instruction that defines
+ // it. Hence its interval is:
+ // [defSlot(def), defSlot(def)+1)
+ DOUT << " dead";
+ end = getDefIndex(start) + 1;
+ goto exit;
+ }
+
+ baseIndex += InstrSlots::NUM;
+ ++mi;
+ }
+
+exit:
+ // Live-in register might not be used at all.
+ if (end == MIIdx) {
+ if (isAlias) {
+ DOUT << " dead";
+ end = getDefIndex(MIIdx) + 1;
+ } else {
+ DOUT << " live through";
+ end = baseIndex;
+ }
+ }
+
+ LiveRange LR(start, end, interval.getNextValue(~0U, 0));
+ DOUT << " +" << LR << '\n';
+ interval.addRange(LR);
+}
+
+/// computeIntervals - computes the live intervals for virtual
+/// registers. for some ordering of the machine instructions [1,N] a
+/// live interval is an interval [i, j) where 1 <= i <= j < N for
+/// which a variable is live
+void LiveIntervals::computeIntervals() {
+ DOUT << "********** COMPUTING LIVE INTERVALS **********\n"
+ << "********** Function: "
+ << ((Value*)mf_->getFunction())->getName() << '\n';
+ // Track the index of the current machine instr.
+ unsigned MIIndex = 0;
+ for (MachineFunction::iterator MBBI = mf_->begin(), E = mf_->end();
+ MBBI != E; ++MBBI) {
+ MachineBasicBlock *MBB = MBBI;
+ DOUT << ((Value*)MBB->getBasicBlock())->getName() << ":\n";
+
+ MachineBasicBlock::iterator MI = MBB->begin(), miEnd = MBB->end();
+
+ if (MBB->livein_begin() != MBB->livein_end()) {
+ // Create intervals for live-ins to this BB first.
+ for (MachineBasicBlock::const_livein_iterator LI = MBB->livein_begin(),
+ LE = MBB->livein_end(); LI != LE; ++LI) {
+ handleLiveInRegister(MBB, MIIndex, getOrCreateInterval(*LI));
+ // Multiple live-ins can alias the same register.
+ for (const unsigned* AS = mri_->getSubRegisters(*LI); *AS; ++AS)
+ if (!hasInterval(*AS))
+ handleLiveInRegister(MBB, MIIndex, getOrCreateInterval(*AS),
+ true);
+ }
+ }
+
+ for (; MI != miEnd; ++MI) {
+ DOUT << MIIndex << "\t" << *MI;
+
+ // Handle defs.
+ for (int i = MI->getNumOperands() - 1; i >= 0; --i) {
+ MachineOperand &MO = MI->getOperand(i);
+ // handle register defs - build intervals
+ if (MO.isRegister() && MO.getReg() && MO.isDef())
+ handleRegisterDef(MBB, MI, MIIndex, MO.getReg());
+ }
+
+ MIIndex += InstrSlots::NUM;
+ }
+ }
+}
+
+LiveInterval LiveIntervals::createInterval(unsigned reg) {
+ float Weight = MRegisterInfo::isPhysicalRegister(reg) ?
+ HUGE_VALF : 0.0F;
+ return LiveInterval(reg, Weight);
+}
diff --git a/lib/CodeGen/LiveVariables.cpp b/lib/CodeGen/LiveVariables.cpp
new file mode 100644
index 0000000..504b607
--- /dev/null
+++ b/lib/CodeGen/LiveVariables.cpp
@@ -0,0 +1,643 @@
+//===-- LiveVariables.cpp - Live Variable Analysis for Machine Code -------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file was developed by the LLVM research group and is distributed under
+// the University of Illinois Open Source License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements the LiveVariable analysis pass. For each machine
+// instruction in the function, this pass calculates the set of registers that
+// are immediately dead after the instruction (i.e., the instruction calculates
+// the value, but it is never used) and the set of registers that are used by
+// the instruction, but are never used after the instruction (i.e., they are
+// killed).
+//
+// This class computes live variables using are sparse implementation based on
+// the machine code SSA form. This class computes live variable information for
+// each virtual and _register allocatable_ physical register in a function. It
+// uses the dominance properties of SSA form to efficiently compute live
+// variables for virtual registers, and assumes that physical registers are only
+// live within a single basic block (allowing it to do a single local analysis
+// to resolve physical register lifetimes in each basic block). If a physical
+// register is not register allocatable, it is not tracked. This is useful for
+// things like the stack pointer and condition codes.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/CodeGen/LiveVariables.h"
+#include "llvm/CodeGen/MachineInstr.h"
+#include "llvm/Target/MRegisterInfo.h"
+#include "llvm/Target/TargetInstrInfo.h"
+#include "llvm/Target/TargetMachine.h"
+#include "llvm/ADT/DepthFirstIterator.h"
+#include "llvm/ADT/SmallPtrSet.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/Config/alloca.h"
+#include <algorithm>
+using namespace llvm;
+
+char LiveVariables::ID = 0;
+static RegisterPass<LiveVariables> X("livevars", "Live Variable Analysis");
+
+void LiveVariables::VarInfo::dump() const {
+ cerr << "Register Defined by: ";
+ if (DefInst)
+ cerr << *DefInst;
+ else
+ cerr << "<null>\n";
+ cerr << " Alive in blocks: ";
+ for (unsigned i = 0, e = AliveBlocks.size(); i != e; ++i)
+ if (AliveBlocks[i]) cerr << i << ", ";
+ cerr << "\n Killed by:";
+ if (Kills.empty())
+ cerr << " No instructions.\n";
+ else {
+ for (unsigned i = 0, e = Kills.size(); i != e; ++i)
+ cerr << "\n #" << i << ": " << *Kills[i];
+ cerr << "\n";
+ }
+}
+
+LiveVariables::VarInfo &LiveVariables::getVarInfo(unsigned RegIdx) {
+ assert(MRegisterInfo::isVirtualRegister(RegIdx) &&
+ "getVarInfo: not a virtual register!");
+ RegIdx -= MRegisterInfo::FirstVirtualRegister;
+ if (RegIdx >= VirtRegInfo.size()) {
+ if (RegIdx >= 2*VirtRegInfo.size())
+ VirtRegInfo.resize(RegIdx*2);
+ else
+ VirtRegInfo.resize(2*VirtRegInfo.size());
+ }
+ VarInfo &VI = VirtRegInfo[RegIdx];
+ VI.AliveBlocks.resize(MF->getNumBlockIDs());
+ return VI;
+}
+
+bool LiveVariables::KillsRegister(MachineInstr *MI, unsigned Reg) const {
+ for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
+ MachineOperand &MO = MI->getOperand(i);
+ if (MO.isReg() && MO.isKill()) {
+ if ((MO.getReg() == Reg) ||
+ (MRegisterInfo::isPhysicalRegister(MO.getReg()) &&
+ MRegisterInfo::isPhysicalRegister(Reg) &&
+ RegInfo->isSubRegister(MO.getReg(), Reg)))
+ return true;
+ }
+ }
+ return false;
+}
+
+bool LiveVariables::RegisterDefIsDead(MachineInstr *MI, unsigned Reg) const {
+ for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
+ MachineOperand &MO = MI->getOperand(i);
+ if (MO.isReg() && MO.isDead()) {
+ if ((MO.getReg() == Reg) ||
+ (MRegisterInfo::isPhysicalRegister(MO.getReg()) &&
+ MRegisterInfo::isPhysicalRegister(Reg) &&
+ RegInfo->isSubRegister(MO.getReg(), Reg)))
+ return true;
+ }
+ }
+ return false;
+}
+
+bool LiveVariables::ModifiesRegister(MachineInstr *MI, unsigned Reg) const {
+ for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
+ MachineOperand &MO = MI->getOperand(i);
+ if (MO.isReg() && MO.isDef() && MO.getReg() == Reg)
+ return true;
+ }
+ return false;
+}
+
+void LiveVariables::MarkVirtRegAliveInBlock(VarInfo &VRInfo,
+ MachineBasicBlock *MBB,
+ std::vector<MachineBasicBlock*> &WorkList) {
+ unsigned BBNum = MBB->getNumber();
+
+ // Check to see if this basic block is one of the killing blocks. If so,
+ // remove it...
+ for (unsigned i = 0, e = VRInfo.Kills.size(); i != e; ++i)
+ if (VRInfo.Kills[i]->getParent() == MBB) {
+ VRInfo.Kills.erase(VRInfo.Kills.begin()+i); // Erase entry
+ break;
+ }
+
+ if (MBB == VRInfo.DefInst->getParent()) return; // Terminate recursion
+
+ if (VRInfo.AliveBlocks[BBNum])
+ return; // We already know the block is live
+
+ // Mark the variable known alive in this bb
+ VRInfo.AliveBlocks[BBNum] = true;
+
+ for (MachineBasicBlock::const_pred_reverse_iterator PI = MBB->pred_rbegin(),
+ E = MBB->pred_rend(); PI != E; ++PI)
+ WorkList.push_back(*PI);
+}
+
+void LiveVariables::MarkVirtRegAliveInBlock(VarInfo &VRInfo,
+ MachineBasicBlock *MBB) {
+ std::vector<MachineBasicBlock*> WorkList;
+ MarkVirtRegAliveInBlock(VRInfo, MBB, WorkList);
+ while (!WorkList.empty()) {
+ MachineBasicBlock *Pred = WorkList.back();
+ WorkList.pop_back();
+ MarkVirtRegAliveInBlock(VRInfo, Pred, WorkList);
+ }
+}
+
+
+void LiveVariables::HandleVirtRegUse(VarInfo &VRInfo, MachineBasicBlock *MBB,
+ MachineInstr *MI) {
+ assert(VRInfo.DefInst && "Register use before def!");
+
+ VRInfo.NumUses++;
+
+ // Check to see if this basic block is already a kill block...
+ if (!VRInfo.Kills.empty() && VRInfo.Kills.back()->getParent() == MBB) {
+ // Yes, this register is killed in this basic block already. Increase the
+ // live range by updating the kill instruction.
+ VRInfo.Kills.back() = MI;
+ return;
+ }
+
+#ifndef NDEBUG
+ for (unsigned i = 0, e = VRInfo.Kills.size(); i != e; ++i)
+ assert(VRInfo.Kills[i]->getParent() != MBB && "entry should be at end!");
+#endif
+
+ assert(MBB != VRInfo.DefInst->getParent() &&
+ "Should have kill for defblock!");
+
+ // Add a new kill entry for this basic block.
+ // If this virtual register is already marked as alive in this basic block,
+ // that means it is alive in at least one of the successor block, it's not
+ // a kill.
+ if (!VRInfo.AliveBlocks[MBB->getNumber()])
+ VRInfo.Kills.push_back(MI);
+
+ // Update all dominating blocks to mark them known live.
+ for (MachineBasicBlock::const_pred_iterator PI = MBB->pred_begin(),
+ E = MBB->pred_end(); PI != E; ++PI)
+ MarkVirtRegAliveInBlock(VRInfo, *PI);
+}
+
+bool LiveVariables::addRegisterKilled(unsigned IncomingReg, MachineInstr *MI,
+ bool AddIfNotFound) {
+ bool Found = false;
+ for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
+ MachineOperand &MO = MI->getOperand(i);
+ if (MO.isReg() && MO.isUse()) {
+ unsigned Reg = MO.getReg();
+ if (!Reg)
+ continue;
+ if (Reg == IncomingReg) {
+ MO.setIsKill();
+ Found = true;
+ break;
+ } else if (MRegisterInfo::isPhysicalRegister(Reg) &&
+ MRegisterInfo::isPhysicalRegister(IncomingReg) &&
+ RegInfo->isSuperRegister(IncomingReg, Reg) &&
+ MO.isKill())
+ // A super-register kill already exists.
+ return true;
+ }
+ }
+
+ // If not found, this means an alias of one of the operand is killed. Add a
+ // new implicit operand if required.
+ if (!Found && AddIfNotFound) {
+ MI->addRegOperand(IncomingReg, false/*IsDef*/,true/*IsImp*/,true/*IsKill*/);
+ return true;
+ }
+ return Found;
+}
+
+bool LiveVariables::addRegisterDead(unsigned IncomingReg, MachineInstr *MI,
+ bool AddIfNotFound) {
+ bool Found = false;
+ for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
+ MachineOperand &MO = MI->getOperand(i);
+ if (MO.isReg() && MO.isDef()) {
+ unsigned Reg = MO.getReg();
+ if (!Reg)
+ continue;
+ if (Reg == IncomingReg) {
+ MO.setIsDead();
+ Found = true;
+ break;
+ } else if (MRegisterInfo::isPhysicalRegister(Reg) &&
+ MRegisterInfo::isPhysicalRegister(IncomingReg) &&
+ RegInfo->isSuperRegister(IncomingReg, Reg) &&
+ MO.isDead())
+ // There exists a super-register that's marked dead.
+ return true;
+ }
+ }
+
+ // If not found, this means an alias of one of the operand is dead. Add a
+ // new implicit operand.
+ if (!Found && AddIfNotFound) {
+ MI->addRegOperand(IncomingReg, true/*IsDef*/,true/*IsImp*/,false/*IsKill*/,
+ true/*IsDead*/);
+ return true;
+ }
+ return Found;
+}
+
+void LiveVariables::HandlePhysRegUse(unsigned Reg, MachineInstr *MI) {
+ // There is a now a proper use, forget about the last partial use.
+ PhysRegPartUse[Reg] = NULL;
+
+ // Turn previous partial def's into read/mod/write.
+ for (unsigned i = 0, e = PhysRegPartDef[Reg].size(); i != e; ++i) {
+ MachineInstr *Def = PhysRegPartDef[Reg][i];
+ // First one is just a def. This means the use is reading some undef bits.
+ if (i != 0)
+ Def->addRegOperand(Reg, false/*IsDef*/,true/*IsImp*/,true/*IsKill*/);
+ Def->addRegOperand(Reg, true/*IsDef*/,true/*IsImp*/);
+ }
+ PhysRegPartDef[Reg].clear();
+
+ // There was an earlier def of a super-register. Add implicit def to that MI.
+ // A: EAX = ...
+ // B: = AX
+ // Add implicit def to A.
+ if (PhysRegInfo[Reg] && !PhysRegUsed[Reg]) {
+ MachineInstr *Def = PhysRegInfo[Reg];
+ if (!Def->findRegisterDefOperand(Reg))
+ Def->addRegOperand(Reg, true/*IsDef*/,true/*IsImp*/);
+ }
+
+ PhysRegInfo[Reg] = MI;
+ PhysRegUsed[Reg] = true;
+
+ for (const unsigned *SubRegs = RegInfo->getSubRegisters(Reg);
+ unsigned SubReg = *SubRegs; ++SubRegs) {
+ PhysRegInfo[SubReg] = MI;
+ PhysRegUsed[SubReg] = true;
+ }
+
+ // Remember the partial uses.
+ for (const unsigned *SuperRegs = RegInfo->getSuperRegisters(Reg);
+ unsigned SuperReg = *SuperRegs; ++SuperRegs)
+ PhysRegPartUse[SuperReg] = MI;
+}
+
+bool LiveVariables::HandlePhysRegKill(unsigned Reg, MachineInstr *RefMI,
+ SmallSet<unsigned, 4> &SubKills) {
+ for (const unsigned *SubRegs = RegInfo->getImmediateSubRegisters(Reg);
+ unsigned SubReg = *SubRegs; ++SubRegs) {
+ MachineInstr *LastRef = PhysRegInfo[SubReg];
+ if (LastRef != RefMI)
+ SubKills.insert(SubReg);
+ else if (!HandlePhysRegKill(SubReg, RefMI, SubKills))
+ SubKills.insert(SubReg);
+ }
+
+ if (*RegInfo->getImmediateSubRegisters(Reg) == 0) {
+ // No sub-registers, just check if reg is killed by RefMI.
+ if (PhysRegInfo[Reg] == RefMI)
+ return true;
+ } else if (SubKills.empty())
+ // None of the sub-registers are killed elsewhere...
+ return true;
+ return false;
+}
+
+void LiveVariables::addRegisterKills(unsigned Reg, MachineInstr *MI,
+ SmallSet<unsigned, 4> &SubKills) {
+ if (SubKills.count(Reg) == 0)
+ addRegisterKilled(Reg, MI, true);
+ else {
+ for (const unsigned *SubRegs = RegInfo->getImmediateSubRegisters(Reg);
+ unsigned SubReg = *SubRegs; ++SubRegs)
+ addRegisterKills(SubReg, MI, SubKills);
+ }
+}
+
+bool LiveVariables::HandlePhysRegKill(unsigned Reg, MachineInstr *RefMI) {
+ SmallSet<unsigned, 4> SubKills;
+ if (HandlePhysRegKill(Reg, RefMI, SubKills)) {
+ addRegisterKilled(Reg, RefMI);
+ return true;
+ } else {
+ // Some sub-registers are killed by another MI.
+ for (const unsigned *SubRegs = RegInfo->getImmediateSubRegisters(Reg);
+ unsigned SubReg = *SubRegs; ++SubRegs)
+ addRegisterKills(SubReg, RefMI, SubKills);
+ return false;
+ }
+}
+
+void LiveVariables::HandlePhysRegDef(unsigned Reg, MachineInstr *MI) {
+ // Does this kill a previous version of this register?
+ if (MachineInstr *LastRef = PhysRegInfo[Reg]) {
+ if (PhysRegUsed[Reg]) {
+ if (!HandlePhysRegKill(Reg, LastRef)) {
+ if (PhysRegPartUse[Reg])
+ addRegisterKilled(Reg, PhysRegPartUse[Reg], true);
+ }
+ } else if (PhysRegPartUse[Reg])
+ // Add implicit use / kill to last use of a sub-register.
+ addRegisterKilled(Reg, PhysRegPartUse[Reg], true);
+ else
+ addRegisterDead(Reg, LastRef);
+ }
+
+ for (const unsigned *SubRegs = RegInfo->getSubRegisters(Reg);
+ unsigned SubReg = *SubRegs; ++SubRegs) {
+ if (MachineInstr *LastRef = PhysRegInfo[SubReg]) {
+ if (PhysRegUsed[SubReg]) {
+ if (!HandlePhysRegKill(SubReg, LastRef)) {
+ if (PhysRegPartUse[SubReg])
+ addRegisterKilled(SubReg, PhysRegPartUse[SubReg], true);
+ }
+ } else if (PhysRegPartUse[SubReg])
+ // Add implicit use / kill to last use of a sub-register.
+ addRegisterKilled(SubReg, PhysRegPartUse[SubReg], true);
+ else
+ addRegisterDead(SubReg, LastRef);
+ }
+ }
+
+ if (MI) {
+ for (const unsigned *SuperRegs = RegInfo->getSuperRegisters(Reg);
+ unsigned SuperReg = *SuperRegs; ++SuperRegs) {
+ if (PhysRegInfo[SuperReg]) {
+ // The larger register is previously defined. Now a smaller part is
+ // being re-defined. Treat it as read/mod/write.
+ // EAX =
+ // AX = EAX<imp-use,kill>, EAX<imp-def>
+ MI->addRegOperand(SuperReg, false/*IsDef*/,true/*IsImp*/,true/*IsKill*/);
+ MI->addRegOperand(SuperReg, true/*IsDef*/,true/*IsImp*/);
+ PhysRegInfo[SuperReg] = MI;
+ PhysRegUsed[SuperReg] = false;
+ PhysRegPartUse[SuperReg] = NULL;
+ } else {
+ // Remember this partial def.
+ PhysRegPartDef[SuperReg].push_back(MI);
+ }
+ }
+
+ PhysRegInfo[Reg] = MI;
+ PhysRegUsed[Reg] = false;
+ PhysRegPartUse[Reg] = NULL;
+ for (const unsigned *SubRegs = RegInfo->getSubRegisters(Reg);
+ unsigned SubReg = *SubRegs; ++SubRegs) {
+ PhysRegInfo[SubReg] = MI;
+ PhysRegUsed[SubReg] = false;
+ PhysRegPartUse[SubReg] = NULL;
+ }
+ }
+}
+
+bool LiveVariables::runOnMachineFunction(MachineFunction &mf) {
+ MF = &mf;
+ const TargetInstrInfo &TII = *MF->getTarget().getInstrInfo();
+ RegInfo = MF->getTarget().getRegisterInfo();
+ assert(RegInfo && "Target doesn't have register information?");
+
+ ReservedRegisters = RegInfo->getReservedRegs(mf);
+
+ unsigned NumRegs = RegInfo->getNumRegs();
+ PhysRegInfo = new MachineInstr*[NumRegs];
+ PhysRegUsed = new bool[NumRegs];
+ PhysRegPartUse = new MachineInstr*[NumRegs];
+ PhysRegPartDef = new SmallVector<MachineInstr*,4>[NumRegs];
+ PHIVarInfo = new SmallVector<unsigned, 4>[MF->getNumBlockIDs()];
+ std::fill(PhysRegInfo, PhysRegInfo + NumRegs, (MachineInstr*)0);
+ std::fill(PhysRegUsed, PhysRegUsed + NumRegs, false);
+ std::fill(PhysRegPartUse, PhysRegPartUse + NumRegs, (MachineInstr*)0);
+
+ /// Get some space for a respectable number of registers...
+ VirtRegInfo.resize(64);
+
+ analyzePHINodes(mf);
+
+ // Calculate live variable information in depth first order on the CFG of the
+ // function. This guarantees that we will see the definition of a virtual
+ // register before its uses due to dominance properties of SSA (except for PHI
+ // nodes, which are treated as a special case).
+ //
+ MachineBasicBlock *Entry = MF->begin();
+ SmallPtrSet<MachineBasicBlock*,16> Visited;
+ for (df_ext_iterator<MachineBasicBlock*, SmallPtrSet<MachineBasicBlock*,16> >
+ DFI = df_ext_begin(Entry, Visited), E = df_ext_end(Entry, Visited);
+ DFI != E; ++DFI) {
+ MachineBasicBlock *MBB = *DFI;
+
+ // Mark live-in registers as live-in.
+ for (MachineBasicBlock::const_livein_iterator II = MBB->livein_begin(),
+ EE = MBB->livein_end(); II != EE; ++II) {
+ assert(MRegisterInfo::isPhysicalRegister(*II) &&
+ "Cannot have a live-in virtual register!");
+ HandlePhysRegDef(*II, 0);
+ }
+
+ // Loop over all of the instructions, processing them.
+ for (MachineBasicBlock::iterator I = MBB->begin(), E = MBB->end();
+ I != E; ++I) {
+ MachineInstr *MI = I;
+
+ // Process all of the operands of the instruction...
+ unsigned NumOperandsToProcess = MI->getNumOperands();
+
+ // Unless it is a PHI node. In this case, ONLY process the DEF, not any
+ // of the uses. They will be handled in other basic blocks.
+ if (MI->getOpcode() == TargetInstrInfo::PHI)
+ NumOperandsToProcess = 1;
+
+ // Process all uses...
+ for (unsigned i = 0; i != NumOperandsToProcess; ++i) {
+ MachineOperand &MO = MI->getOperand(i);
+ if (MO.isRegister() && MO.isUse() && MO.getReg()) {
+ if (MRegisterInfo::isVirtualRegister(MO.getReg())){
+ HandleVirtRegUse(getVarInfo(MO.getReg()), MBB, MI);
+ } else if (MRegisterInfo::isPhysicalRegister(MO.getReg()) &&
+ !ReservedRegisters[MO.getReg()]) {
+ HandlePhysRegUse(MO.getReg(), MI);
+ }
+ }
+ }
+
+ // Process all defs...
+ for (unsigned i = 0; i != NumOperandsToProcess; ++i) {
+ MachineOperand &MO = MI->getOperand(i);
+ if (MO.isRegister() && MO.isDef() && MO.getReg()) {
+ if (MRegisterInfo::isVirtualRegister(MO.getReg())) {
+ VarInfo &VRInfo = getVarInfo(MO.getReg());
+
+ assert(VRInfo.DefInst == 0 && "Variable multiply defined!");
+ VRInfo.DefInst = MI;
+ // Defaults to dead
+ VRInfo.Kills.push_back(MI);
+ } else if (MRegisterInfo::isPhysicalRegister(MO.getReg()) &&
+ !ReservedRegisters[MO.getReg()]) {
+ HandlePhysRegDef(MO.getReg(), MI);
+ }
+ }
+ }
+ }
+
+ // Handle any virtual assignments from PHI nodes which might be at the
+ // bottom of this basic block. We check all of our successor blocks to see
+ // if they have PHI nodes, and if so, we simulate an assignment at the end
+ // of the current block.
+ if (!PHIVarInfo[MBB->getNumber()].empty()) {
+ SmallVector<unsigned, 4>& VarInfoVec = PHIVarInfo[MBB->getNumber()];
+
+ for (SmallVector<unsigned, 4>::iterator I = VarInfoVec.begin(),
+ E = VarInfoVec.end(); I != E; ++I) {
+ VarInfo& VRInfo = getVarInfo(*I);
+ assert(VRInfo.DefInst && "Register use before def (or no def)!");
+
+ // Only mark it alive only in the block we are representing.
+ MarkVirtRegAliveInBlock(VRInfo, MBB);
+ }
+ }
+
+ // Finally, if the last instruction in the block is a return, make sure to mark
+ // it as using all of the live-out values in the function.
+ if (!MBB->empty() && TII.isReturn(MBB->back().getOpcode())) {
+ MachineInstr *Ret = &MBB->back();
+ for (MachineFunction::liveout_iterator I = MF->liveout_begin(),
+ E = MF->liveout_end(); I != E; ++I) {
+ assert(MRegisterInfo::isPhysicalRegister(*I) &&
+ "Cannot have a live-in virtual register!");
+ HandlePhysRegUse(*I, Ret);
+ // Add live-out registers as implicit uses.
+ if (Ret->findRegisterUseOperandIdx(*I) == -1)
+ Ret->addRegOperand(*I, false, true);
+ }
+ }
+
+ // Loop over PhysRegInfo, killing any registers that are available at the
+ // end of the basic block. This also resets the PhysRegInfo map.
+ for (unsigned i = 0; i != NumRegs; ++i)
+ if (PhysRegInfo[i])
+ HandlePhysRegDef(i, 0);
+
+ // Clear some states between BB's. These are purely local information.
+ for (unsigned i = 0; i != NumRegs; ++i)
+ PhysRegPartDef[i].clear();
+ std::fill(PhysRegInfo, PhysRegInfo + NumRegs, (MachineInstr*)0);
+ std::fill(PhysRegUsed, PhysRegUsed + NumRegs, false);
+ std::fill(PhysRegPartUse, PhysRegPartUse + NumRegs, (MachineInstr*)0);
+ }
+
+ // Convert and transfer the dead / killed information we have gathered into
+ // VirtRegInfo onto MI's.
+ //
+ for (unsigned i = 0, e1 = VirtRegInfo.size(); i != e1; ++i)
+ for (unsigned j = 0, e2 = VirtRegInfo[i].Kills.size(); j != e2; ++j) {
+ if (VirtRegInfo[i].Kills[j] == VirtRegInfo[i].DefInst)
+ addRegisterDead(i + MRegisterInfo::FirstVirtualRegister,
+ VirtRegInfo[i].Kills[j]);
+ else
+ addRegisterKilled(i + MRegisterInfo::FirstVirtualRegister,
+ VirtRegInfo[i].Kills[j]);
+ }
+
+ // Check to make sure there are no unreachable blocks in the MC CFG for the
+ // function. If so, it is due to a bug in the instruction selector or some
+ // other part of the code generator if this happens.
+#ifndef NDEBUG
+ for(MachineFunction::iterator i = MF->begin(), e = MF->end(); i != e; ++i)
+ assert(Visited.count(&*i) != 0 && "unreachable basic block found");
+#endif
+
+ delete[] PhysRegInfo;
+ delete[] PhysRegUsed;
+ delete[] PhysRegPartUse;
+ delete[] PhysRegPartDef;
+ delete[] PHIVarInfo;
+
+ return false;
+}
+
+/// instructionChanged - When the address of an instruction changes, this
+/// method should be called so that live variables can update its internal
+/// data structures. This removes the records for OldMI, transfering them to
+/// the records for NewMI.
+void LiveVariables::instructionChanged(MachineInstr *OldMI,
+ MachineInstr *NewMI) {
+ // If the instruction defines any virtual registers, update the VarInfo,
+ // kill and dead information for the instruction.
+ for (unsigned i = 0, e = OldMI->getNumOperands(); i != e; ++i) {
+ MachineOperand &MO = OldMI->getOperand(i);
+ if (MO.isRegister() && MO.getReg() &&
+ MRegisterInfo::isVirtualRegister(MO.getReg())) {
+ unsigned Reg = MO.getReg();
+ VarInfo &VI = getVarInfo(Reg);
+ if (MO.isDef()) {
+ if (MO.isDead()) {
+ MO.unsetIsDead();
+ addVirtualRegisterDead(Reg, NewMI);
+ }
+ // Update the defining instruction.
+ if (VI.DefInst == OldMI)
+ VI.DefInst = NewMI;
+ }
+ if (MO.isUse()) {
+ if (MO.isKill()) {
+ MO.unsetIsKill();
+ addVirtualRegisterKilled(Reg, NewMI);
+ }
+ // If this is a kill of the value, update the VI kills list.
+ if (VI.removeKill(OldMI))
+ VI.Kills.push_back(NewMI); // Yes, there was a kill of it
+ }
+ }
+ }
+}
+
+/// removeVirtualRegistersKilled - Remove all killed info for the specified
+/// instruction.
+void LiveVariables::removeVirtualRegistersKilled(MachineInstr *MI) {
+ for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
+ MachineOperand &MO = MI->getOperand(i);
+ if (MO.isReg() && MO.isKill()) {
+ MO.unsetIsKill();
+ unsigned Reg = MO.getReg();
+ if (MRegisterInfo::isVirtualRegister(Reg)) {
+ bool removed = getVarInfo(Reg).removeKill(MI);
+ assert(removed && "kill not in register's VarInfo?");
+ }
+ }
+ }
+}
+
+/// removeVirtualRegistersDead - Remove all of the dead registers for the
+/// specified instruction from the live variable information.
+void LiveVariables::removeVirtualRegistersDead(MachineInstr *MI) {
+ for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
+ MachineOperand &MO = MI->getOperand(i);
+ if (MO.isReg() && MO.isDead()) {
+ MO.unsetIsDead();
+ unsigned Reg = MO.getReg();
+ if (MRegisterInfo::isVirtualRegister(Reg)) {
+ bool removed = getVarInfo(Reg).removeKill(MI);
+ assert(removed && "kill not in register's VarInfo?");
+ }
+ }
+ }
+}
+
+/// analyzePHINodes - Gather information about the PHI nodes in here. In
+/// particular, we want to map the variable information of a virtual
+/// register which is used in a PHI node. We map that to the BB the vreg is
+/// coming from.
+///
+void LiveVariables::analyzePHINodes(const MachineFunction& Fn) {
+ for (MachineFunction::const_iterator I = Fn.begin(), E = Fn.end();
+ I != E; ++I)
+ for (MachineBasicBlock::const_iterator BBI = I->begin(), BBE = I->end();
+ BBI != BBE && BBI->getOpcode() == TargetInstrInfo::PHI; ++BBI)
+ for (unsigned i = 1, e = BBI->getNumOperands(); i != e; i += 2)
+ PHIVarInfo[BBI->getOperand(i + 1).getMachineBasicBlock()->getNumber()].
+ push_back(BBI->getOperand(i).getReg());
+}
diff --git a/lib/CodeGen/MachOWriter.cpp b/lib/CodeGen/MachOWriter.cpp
new file mode 100644
index 0000000..36060e1
--- /dev/null
+++ b/lib/CodeGen/MachOWriter.cpp
@@ -0,0 +1,945 @@
+//===-- MachOWriter.cpp - Target-independent Mach-O Writer code -----------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file was developed by Nate Begeman and is distributed under the
+// University of Illinois Open Source License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements the target-independent Mach-O writer. This file writes
+// out the Mach-O file in the following order:
+//
+// #1 FatHeader (universal-only)
+// #2 FatArch (universal-only, 1 per universal arch)
+// Per arch:
+// #3 Header
+// #4 Load Commands
+// #5 Sections
+// #6 Relocations
+// #7 Symbols
+// #8 Strings
+//
+//===----------------------------------------------------------------------===//
+
+#include "MachOWriter.h"
+#include "llvm/Constants.h"
+#include "llvm/DerivedTypes.h"
+#include "llvm/Module.h"
+#include "llvm/PassManager.h"
+#include "llvm/CodeGen/FileWriters.h"
+#include "llvm/CodeGen/MachineCodeEmitter.h"
+#include "llvm/CodeGen/MachineConstantPool.h"
+#include "llvm/CodeGen/MachineJumpTableInfo.h"
+#include "llvm/Target/TargetAsmInfo.h"
+#include "llvm/Target/TargetJITInfo.h"
+#include "llvm/Support/Mangler.h"
+#include "llvm/Support/MathExtras.h"
+#include "llvm/Support/OutputBuffer.h"
+#include "llvm/Support/Streams.h"
+#include <algorithm>
+using namespace llvm;
+
+/// AddMachOWriter - Concrete function to add the Mach-O writer to the function
+/// pass manager.
+MachineCodeEmitter *llvm::AddMachOWriter(FunctionPassManager &FPM,
+ std::ostream &O,
+ TargetMachine &TM) {
+ MachOWriter *MOW = new MachOWriter(O, TM);
+ FPM.add(MOW);
+ return &MOW->getMachineCodeEmitter();
+}
+
+//===----------------------------------------------------------------------===//
+// MachOCodeEmitter Implementation
+//===----------------------------------------------------------------------===//
+
+namespace llvm {
+ /// MachOCodeEmitter - This class is used by the MachOWriter to emit the code
+ /// for functions to the Mach-O file.
+ class MachOCodeEmitter : public MachineCodeEmitter {
+ MachOWriter &MOW;
+
+ /// Target machine description.
+ TargetMachine &TM;
+
+ /// is64Bit/isLittleEndian - This information is inferred from the target
+ /// machine directly, indicating what header values and flags to set.
+ bool is64Bit, isLittleEndian;
+
+ /// Relocations - These are the relocations that the function needs, as
+ /// emitted.
+ std::vector<MachineRelocation> Relocations;
+
+ /// CPLocations - This is a map of constant pool indices to offsets from the
+ /// start of the section for that constant pool index.
+ std::vector<intptr_t> CPLocations;
+
+ /// CPSections - This is a map of constant pool indices to the MachOSection
+ /// containing the constant pool entry for that index.
+ std::vector<unsigned> CPSections;
+
+ /// JTLocations - This is a map of jump table indices to offsets from the
+ /// start of the section for that jump table index.
+ std::vector<intptr_t> JTLocations;
+
+ /// MBBLocations - This vector is a mapping from MBB ID's to their address.
+ /// It is filled in by the StartMachineBasicBlock callback and queried by
+ /// the getMachineBasicBlockAddress callback.
+ std::vector<intptr_t> MBBLocations;
+
+ public:
+ MachOCodeEmitter(MachOWriter &mow) : MOW(mow), TM(MOW.TM) {
+ is64Bit = TM.getTargetData()->getPointerSizeInBits() == 64;
+ isLittleEndian = TM.getTargetData()->isLittleEndian();
+ }
+
+ virtual void startFunction(MachineFunction &MF);
+ virtual bool finishFunction(MachineFunction &MF);
+
+ virtual void addRelocation(const MachineRelocation &MR) {
+ Relocations.push_back(MR);
+ }
+
+ void emitConstantPool(MachineConstantPool *MCP);
+ void emitJumpTables(MachineJumpTableInfo *MJTI);
+
+ virtual intptr_t getConstantPoolEntryAddress(unsigned Index) const {
+ assert(CPLocations.size() > Index && "CP not emitted!");
+ return CPLocations[Index];
+ }
+ virtual intptr_t getJumpTableEntryAddress(unsigned Index) const {
+ assert(JTLocations.size() > Index && "JT not emitted!");
+ return JTLocations[Index];
+ }
+
+ virtual void StartMachineBasicBlock(MachineBasicBlock *MBB) {
+ if (MBBLocations.size() <= (unsigned)MBB->getNumber())
+ MBBLocations.resize((MBB->getNumber()+1)*2);
+ MBBLocations[MBB->getNumber()] = getCurrentPCOffset();
+ }
+
+ virtual intptr_t getMachineBasicBlockAddress(MachineBasicBlock *MBB) const {
+ assert(MBBLocations.size() > (unsigned)MBB->getNumber() &&
+ MBBLocations[MBB->getNumber()] && "MBB not emitted!");
+ return MBBLocations[MBB->getNumber()];
+ }
+
+ /// JIT SPECIFIC FUNCTIONS - DO NOT IMPLEMENT THESE HERE!
+ virtual void startFunctionStub(unsigned StubSize, unsigned Alignment = 1) {
+ assert(0 && "JIT specific function called!");
+ abort();
+ }
+ virtual void *finishFunctionStub(const Function *F) {
+ assert(0 && "JIT specific function called!");
+ abort();
+ return 0;
+ }
+ };
+}
+
+/// startFunction - This callback is invoked when a new machine function is
+/// about to be emitted.
+void MachOCodeEmitter::startFunction(MachineFunction &MF) {
+ const TargetData *TD = TM.getTargetData();
+ const Function *F = MF.getFunction();
+
+ // Align the output buffer to the appropriate alignment, power of 2.
+ unsigned FnAlign = F->getAlignment();
+ unsigned TDAlign = TD->getPrefTypeAlignment(F->getType());
+ unsigned Align = Log2_32(std::max(FnAlign, TDAlign));
+ assert(!(Align & (Align-1)) && "Alignment is not a power of two!");
+
+ // Get the Mach-O Section that this function belongs in.
+ MachOWriter::MachOSection *MOS = MOW.getTextSection();
+
+ // FIXME: better memory management
+ MOS->SectionData.reserve(4096);
+ BufferBegin = &MOS->SectionData[0];
+ BufferEnd = BufferBegin + MOS->SectionData.capacity();
+
+ // Upgrade the section alignment if required.
+ if (MOS->align < Align) MOS->align = Align;
+
+ // Round the size up to the correct alignment for starting the new function.
+ if ((MOS->size & ((1 << Align) - 1)) != 0) {
+ MOS->size += (1 << Align);
+ MOS->size &= ~((1 << Align) - 1);
+ }
+
+ // FIXME: Using MOS->size directly here instead of calculating it from the
+ // output buffer size (impossible because the code emitter deals only in raw
+ // bytes) forces us to manually synchronize size and write padding zero bytes
+ // to the output buffer for all non-text sections. For text sections, we do
+ // not synchonize the output buffer, and we just blow up if anyone tries to
+ // write non-code to it. An assert should probably be added to
+ // AddSymbolToSection to prevent calling it on the text section.
+ CurBufferPtr = BufferBegin + MOS->size;
+
+ // Clear per-function data structures.
+ CPLocations.clear();
+ CPSections.clear();
+ JTLocations.clear();
+ MBBLocations.clear();
+}
+
+/// finishFunction - This callback is invoked after the function is completely
+/// finished.
+bool MachOCodeEmitter::finishFunction(MachineFunction &MF) {
+ // Get the Mach-O Section that this function belongs in.
+ MachOWriter::MachOSection *MOS = MOW.getTextSection();
+
+ // Get a symbol for the function to add to the symbol table
+ // FIXME: it seems like we should call something like AddSymbolToSection
+ // in startFunction rather than changing the section size and symbol n_value
+ // here.
+ const GlobalValue *FuncV = MF.getFunction();
+ MachOSym FnSym(FuncV, MOW.Mang->getValueName(FuncV), MOS->Index, TM);
+ FnSym.n_value = MOS->size;
+ MOS->size = CurBufferPtr - BufferBegin;
+
+ // Emit constant pool to appropriate section(s)
+ emitConstantPool(MF.getConstantPool());
+
+ // Emit jump tables to appropriate section
+ emitJumpTables(MF.getJumpTableInfo());
+
+ // If we have emitted any relocations to function-specific objects such as
+ // basic blocks, constant pools entries, or jump tables, record their
+ // addresses now so that we can rewrite them with the correct addresses
+ // later.
+ for (unsigned i = 0, e = Relocations.size(); i != e; ++i) {
+ MachineRelocation &MR = Relocations[i];
+ intptr_t Addr;
+
+ if (MR.isBasicBlock()) {
+ Addr = getMachineBasicBlockAddress(MR.getBasicBlock());
+ MR.setConstantVal(MOS->Index);
+ MR.setResultPointer((void*)Addr);
+ } else if (MR.isJumpTableIndex()) {
+ Addr = getJumpTableEntryAddress(MR.getJumpTableIndex());
+ MR.setConstantVal(MOW.getJumpTableSection()->Index);
+ MR.setResultPointer((void*)Addr);
+ } else if (MR.isConstantPoolIndex()) {
+ Addr = getConstantPoolEntryAddress(MR.getConstantPoolIndex());
+ MR.setConstantVal(CPSections[MR.getConstantPoolIndex()]);
+ MR.setResultPointer((void*)Addr);
+ } else if (MR.isGlobalValue()) {
+ // FIXME: This should be a set or something that uniques
+ MOW.PendingGlobals.push_back(MR.getGlobalValue());
+ } else {
+ assert(0 && "Unhandled relocation type");
+ }
+ MOS->Relocations.push_back(MR);
+ }
+ Relocations.clear();
+
+ // Finally, add it to the symtab.
+ MOW.SymbolTable.push_back(FnSym);
+ return false;
+}
+
+/// emitConstantPool - For each constant pool entry, figure out which section
+/// the constant should live in, allocate space for it, and emit it to the
+/// Section data buffer.
+void MachOCodeEmitter::emitConstantPool(MachineConstantPool *MCP) {
+ const std::vector<MachineConstantPoolEntry> &CP = MCP->getConstants();
+ if (CP.empty()) return;
+
+ // FIXME: handle PIC codegen
+ bool isPIC = TM.getRelocationModel() == Reloc::PIC_;
+ assert(!isPIC && "PIC codegen not yet handled for mach-o jump tables!");
+
+ // Although there is no strict necessity that I am aware of, we will do what
+ // gcc for OS X does and put each constant pool entry in a section of constant
+ // objects of a certain size. That means that float constants go in the
+ // literal4 section, and double objects go in literal8, etc.
+ //
+ // FIXME: revisit this decision if we ever do the "stick everything into one
+ // "giant object for PIC" optimization.
+ for (unsigned i = 0, e = CP.size(); i != e; ++i) {
+ const Type *Ty = CP[i].getType();
+ unsigned Size = TM.getTargetData()->getTypeSize(Ty);
+
+ MachOWriter::MachOSection *Sec = MOW.getConstSection(CP[i].Val.ConstVal);
+ OutputBuffer SecDataOut(Sec->SectionData, is64Bit, isLittleEndian);
+
+ CPLocations.push_back(Sec->SectionData.size());
+ CPSections.push_back(Sec->Index);
+
+ // FIXME: remove when we have unified size + output buffer
+ Sec->size += Size;
+
+ // Allocate space in the section for the global.
+ // FIXME: need alignment?
+ // FIXME: share between here and AddSymbolToSection?
+ for (unsigned j = 0; j < Size; ++j)
+ SecDataOut.outbyte(0);
+
+ MOW.InitMem(CP[i].Val.ConstVal, &Sec->SectionData[0], CPLocations[i],
+ TM.getTargetData(), Sec->Relocations);
+ }
+}
+
+/// emitJumpTables - Emit all the jump tables for a given jump table info
+/// record to the appropriate section.
+void MachOCodeEmitter::emitJumpTables(MachineJumpTableInfo *MJTI) {
+ const std::vector<MachineJumpTableEntry> &JT = MJTI->getJumpTables();
+ if (JT.empty()) return;
+
+ // FIXME: handle PIC codegen
+ bool isPIC = TM.getRelocationModel() == Reloc::PIC_;
+ assert(!isPIC && "PIC codegen not yet handled for mach-o jump tables!");
+
+ MachOWriter::MachOSection *Sec = MOW.getJumpTableSection();
+ unsigned TextSecIndex = MOW.getTextSection()->Index;
+ OutputBuffer SecDataOut(Sec->SectionData, is64Bit, isLittleEndian);
+
+ for (unsigned i = 0, e = JT.size(); i != e; ++i) {
+ // For each jump table, record its offset from the start of the section,
+ // reserve space for the relocations to the MBBs, and add the relocations.
+ const std::vector<MachineBasicBlock*> &MBBs = JT[i].MBBs;
+ JTLocations.push_back(Sec->SectionData.size());
+ for (unsigned mi = 0, me = MBBs.size(); mi != me; ++mi) {
+ MachineRelocation MR(MOW.GetJTRelocation(Sec->SectionData.size(),
+ MBBs[mi]));
+ MR.setResultPointer((void *)JTLocations[i]);
+ MR.setConstantVal(TextSecIndex);
+ Sec->Relocations.push_back(MR);
+ SecDataOut.outaddr(0);
+ }
+ }
+ // FIXME: remove when we have unified size + output buffer
+ Sec->size = Sec->SectionData.size();
+}
+
+//===----------------------------------------------------------------------===//
+// MachOWriter Implementation
+//===----------------------------------------------------------------------===//
+
+char MachOWriter::ID = 0;
+MachOWriter::MachOWriter(std::ostream &o, TargetMachine &tm)
+ : MachineFunctionPass((intptr_t)&ID), O(o), TM(tm) {
+ is64Bit = TM.getTargetData()->getPointerSizeInBits() == 64;
+ isLittleEndian = TM.getTargetData()->isLittleEndian();
+
+ // Create the machine code emitter object for this target.
+ MCE = new MachOCodeEmitter(*this);
+}
+
+MachOWriter::~MachOWriter() {
+ delete MCE;
+}
+
+void MachOWriter::AddSymbolToSection(MachOSection *Sec, GlobalVariable *GV) {
+ const Type *Ty = GV->getType()->getElementType();
+ unsigned Size = TM.getTargetData()->getTypeSize(Ty);
+ unsigned Align = GV->getAlignment();
+ if (Align == 0)
+ Align = TM.getTargetData()->getPrefTypeAlignment(Ty);
+
+ // Reserve space in the .bss section for this symbol while maintaining the
+ // desired section alignment, which must be at least as much as required by
+ // this symbol.
+ OutputBuffer SecDataOut(Sec->SectionData, is64Bit, isLittleEndian);
+
+ if (Align) {
+ uint64_t OrigSize = Sec->size;
+ Align = Log2_32(Align);
+ Sec->align = std::max(unsigned(Sec->align), Align);
+ Sec->size = (Sec->size + Align - 1) & ~(Align-1);
+
+ // Add alignment padding to buffer as well.
+ // FIXME: remove when we have unified size + output buffer
+ unsigned AlignedSize = Sec->size - OrigSize;
+ for (unsigned i = 0; i < AlignedSize; ++i)
+ SecDataOut.outbyte(0);
+ }
+ // Globals without external linkage apparently do not go in the symbol table.
+ if (GV->getLinkage() != GlobalValue::InternalLinkage) {
+ MachOSym Sym(GV, Mang->getValueName(GV), Sec->Index, TM);
+ Sym.n_value = Sec->size;
+ SymbolTable.push_back(Sym);
+ }
+
+ // Record the offset of the symbol, and then allocate space for it.
+ // FIXME: remove when we have unified size + output buffer
+ Sec->size += Size;
+
+ // Now that we know what section the GlovalVariable is going to be emitted
+ // into, update our mappings.
+ // FIXME: We may also need to update this when outputting non-GlobalVariable
+ // GlobalValues such as functions.
+ GVSection[GV] = Sec;
+ GVOffset[GV] = Sec->SectionData.size();
+
+ // Allocate space in the section for the global.
+ for (unsigned i = 0; i < Size; ++i)
+ SecDataOut.outbyte(0);
+}
+
+void MachOWriter::EmitGlobal(GlobalVariable *GV) {
+ const Type *Ty = GV->getType()->getElementType();
+ unsigned Size = TM.getTargetData()->getTypeSize(Ty);
+ bool NoInit = !GV->hasInitializer();
+
+ // If this global has a zero initializer, it is part of the .bss or common
+ // section.
+ if (NoInit || GV->getInitializer()->isNullValue()) {
+ // If this global is part of the common block, add it now. Variables are
+ // part of the common block if they are zero initialized and allowed to be
+ // merged with other symbols.
+ if (NoInit || GV->hasLinkOnceLinkage() || GV->hasWeakLinkage()) {
+ MachOSym ExtOrCommonSym(GV, Mang->getValueName(GV), MachOSym::NO_SECT,TM);
+ // For undefined (N_UNDF) external (N_EXT) types, n_value is the size in
+ // bytes of the symbol.
+ ExtOrCommonSym.n_value = Size;
+ SymbolTable.push_back(ExtOrCommonSym);
+ // Remember that we've seen this symbol
+ GVOffset[GV] = Size;
+ return;
+ }
+ // Otherwise, this symbol is part of the .bss section.
+ MachOSection *BSS = getBSSSection();
+ AddSymbolToSection(BSS, GV);
+ return;
+ }
+
+ // Scalar read-only data goes in a literal section if the scalar is 4, 8, or
+ // 16 bytes, or a cstring. Other read only data goes into a regular const
+ // section. Read-write data goes in the data section.
+ MachOSection *Sec = GV->isConstant() ? getConstSection(GV->getInitializer()) :
+ getDataSection();
+ AddSymbolToSection(Sec, GV);
+ InitMem(GV->getInitializer(), &Sec->SectionData[0], GVOffset[GV],
+ TM.getTargetData(), Sec->Relocations);
+}
+
+
+bool MachOWriter::runOnMachineFunction(MachineFunction &MF) {
+ // Nothing to do here, this is all done through the MCE object.
+ return false;
+}
+
+bool MachOWriter::doInitialization(Module &M) {
+ // Set the magic value, now that we know the pointer size and endianness
+ Header.setMagic(isLittleEndian, is64Bit);
+
+ // Set the file type
+ // FIXME: this only works for object files, we do not support the creation
+ // of dynamic libraries or executables at this time.
+ Header.filetype = MachOHeader::MH_OBJECT;
+
+ Mang = new Mangler(M);
+ return false;
+}
+
+/// doFinalization - Now that the module has been completely processed, emit
+/// the Mach-O file to 'O'.
+bool MachOWriter::doFinalization(Module &M) {
+ // FIXME: we don't handle debug info yet, we should probably do that.
+
+ // Okay, the.text section has been completed, build the .data, .bss, and
+ // "common" sections next.
+ for (Module::global_iterator I = M.global_begin(), E = M.global_end();
+ I != E; ++I)
+ EmitGlobal(I);
+
+ // Emit the header and load commands.
+ EmitHeaderAndLoadCommands();
+
+ // Emit the various sections and their relocation info.
+ EmitSections();
+
+ // Write the symbol table and the string table to the end of the file.
+ O.write((char*)&SymT[0], SymT.size());
+ O.write((char*)&StrT[0], StrT.size());
+
+ // We are done with the abstract symbols.
+ SectionList.clear();
+ SymbolTable.clear();
+ DynamicSymbolTable.clear();
+
+ // Release the name mangler object.
+ delete Mang; Mang = 0;
+ return false;
+}
+
+void MachOWriter::EmitHeaderAndLoadCommands() {
+ // Step #0: Fill in the segment load command size, since we need it to figure
+ // out the rest of the header fields
+ MachOSegment SEG("", is64Bit);
+ SEG.nsects = SectionList.size();
+ SEG.cmdsize = SEG.cmdSize(is64Bit) +
+ SEG.nsects * SectionList[0]->cmdSize(is64Bit);
+
+ // Step #1: calculate the number of load commands. We always have at least
+ // one, for the LC_SEGMENT load command, plus two for the normal
+ // and dynamic symbol tables, if there are any symbols.
+ Header.ncmds = SymbolTable.empty() ? 1 : 3;
+
+ // Step #2: calculate the size of the load commands
+ Header.sizeofcmds = SEG.cmdsize;
+ if (!SymbolTable.empty())
+ Header.sizeofcmds += SymTab.cmdsize + DySymTab.cmdsize;
+
+ // Step #3: write the header to the file
+ // Local alias to shortenify coming code.
+ DataBuffer &FH = Header.HeaderData;
+ OutputBuffer FHOut(FH, is64Bit, isLittleEndian);
+
+ FHOut.outword(Header.magic);
+ FHOut.outword(TM.getMachOWriterInfo()->getCPUType());
+ FHOut.outword(TM.getMachOWriterInfo()->getCPUSubType());
+ FHOut.outword(Header.filetype);
+ FHOut.outword(Header.ncmds);
+ FHOut.outword(Header.sizeofcmds);
+ FHOut.outword(Header.flags);
+ if (is64Bit)
+ FHOut.outword(Header.reserved);
+
+ // Step #4: Finish filling in the segment load command and write it out
+ for (std::vector<MachOSection*>::iterator I = SectionList.begin(),
+ E = SectionList.end(); I != E; ++I)
+ SEG.filesize += (*I)->size;
+
+ SEG.vmsize = SEG.filesize;
+ SEG.fileoff = Header.cmdSize(is64Bit) + Header.sizeofcmds;
+
+ FHOut.outword(SEG.cmd);
+ FHOut.outword(SEG.cmdsize);
+ FHOut.outstring(SEG.segname, 16);
+ FHOut.outaddr(SEG.vmaddr);
+ FHOut.outaddr(SEG.vmsize);
+ FHOut.outaddr(SEG.fileoff);
+ FHOut.outaddr(SEG.filesize);
+ FHOut.outword(SEG.maxprot);
+ FHOut.outword(SEG.initprot);
+ FHOut.outword(SEG.nsects);
+ FHOut.outword(SEG.flags);
+
+ // Step #5: Finish filling in the fields of the MachOSections
+ uint64_t currentAddr = 0;
+ for (std::vector<MachOSection*>::iterator I = SectionList.begin(),
+ E = SectionList.end(); I != E; ++I) {
+ MachOSection *MOS = *I;
+ MOS->addr = currentAddr;
+ MOS->offset = currentAddr + SEG.fileoff;
+
+ // FIXME: do we need to do something with alignment here?
+ currentAddr += MOS->size;
+ }
+
+ // Step #6: Emit the symbol table to temporary buffers, so that we know the
+ // size of the string table when we write the next load command. This also
+ // sorts and assigns indices to each of the symbols, which is necessary for
+ // emitting relocations to externally-defined objects.
+ BufferSymbolAndStringTable();
+
+ // Step #7: Calculate the number of relocations for each section and write out
+ // the section commands for each section
+ currentAddr += SEG.fileoff;
+ for (std::vector<MachOSection*>::iterator I = SectionList.begin(),
+ E = SectionList.end(); I != E; ++I) {
+ MachOSection *MOS = *I;
+ // Convert the relocations to target-specific relocations, and fill in the
+ // relocation offset for this section.
+ CalculateRelocations(*MOS);
+ MOS->reloff = MOS->nreloc ? currentAddr : 0;
+ currentAddr += MOS->nreloc * 8;
+
+ // write the finalized section command to the output buffer
+ FHOut.outstring(MOS->sectname, 16);
+ FHOut.outstring(MOS->segname, 16);
+ FHOut.outaddr(MOS->addr);
+ FHOut.outaddr(MOS->size);
+ FHOut.outword(MOS->offset);
+ FHOut.outword(MOS->align);
+ FHOut.outword(MOS->reloff);
+ FHOut.outword(MOS->nreloc);
+ FHOut.outword(MOS->flags);
+ FHOut.outword(MOS->reserved1);
+ FHOut.outword(MOS->reserved2);
+ if (is64Bit)
+ FHOut.outword(MOS->reserved3);
+ }
+
+ // Step #8: Emit LC_SYMTAB/LC_DYSYMTAB load commands
+ SymTab.symoff = currentAddr;
+ SymTab.nsyms = SymbolTable.size();
+ SymTab.stroff = SymTab.symoff + SymT.size();
+ SymTab.strsize = StrT.size();
+ FHOut.outword(SymTab.cmd);
+ FHOut.outword(SymTab.cmdsize);
+ FHOut.outword(SymTab.symoff);
+ FHOut.outword(SymTab.nsyms);
+ FHOut.outword(SymTab.stroff);
+ FHOut.outword(SymTab.strsize);
+
+ // FIXME: set DySymTab fields appropriately
+ // We should probably just update these in BufferSymbolAndStringTable since
+ // thats where we're partitioning up the different kinds of symbols.
+ FHOut.outword(DySymTab.cmd);
+ FHOut.outword(DySymTab.cmdsize);
+ FHOut.outword(DySymTab.ilocalsym);
+ FHOut.outword(DySymTab.nlocalsym);
+ FHOut.outword(DySymTab.iextdefsym);
+ FHOut.outword(DySymTab.nextdefsym);
+ FHOut.outword(DySymTab.iundefsym);
+ FHOut.outword(DySymTab.nundefsym);
+ FHOut.outword(DySymTab.tocoff);
+ FHOut.outword(DySymTab.ntoc);
+ FHOut.outword(DySymTab.modtaboff);
+ FHOut.outword(DySymTab.nmodtab);
+ FHOut.outword(DySymTab.extrefsymoff);
+ FHOut.outword(DySymTab.nextrefsyms);
+ FHOut.outword(DySymTab.indirectsymoff);
+ FHOut.outword(DySymTab.nindirectsyms);
+ FHOut.outword(DySymTab.extreloff);
+ FHOut.outword(DySymTab.nextrel);
+ FHOut.outword(DySymTab.locreloff);
+ FHOut.outword(DySymTab.nlocrel);
+
+ O.write((char*)&FH[0], FH.size());
+}
+
+/// EmitSections - Now that we have constructed the file header and load
+/// commands, emit the data for each section to the file.
+void MachOWriter::EmitSections() {
+ for (std::vector<MachOSection*>::iterator I = SectionList.begin(),
+ E = SectionList.end(); I != E; ++I)
+ // Emit the contents of each section
+ O.write((char*)&(*I)->SectionData[0], (*I)->size);
+ for (std::vector<MachOSection*>::iterator I = SectionList.begin(),
+ E = SectionList.end(); I != E; ++I)
+ // Emit the relocation entry data for each section.
+ O.write((char*)&(*I)->RelocBuffer[0], (*I)->RelocBuffer.size());
+}
+
+/// PartitionByLocal - Simple boolean predicate that returns true if Sym is
+/// a local symbol rather than an external symbol.
+bool MachOWriter::PartitionByLocal(const MachOSym &Sym) {
+ return (Sym.n_type & (MachOSym::N_EXT | MachOSym::N_PEXT)) == 0;
+}
+
+/// PartitionByDefined - Simple boolean predicate that returns true if Sym is
+/// defined in this module.
+bool MachOWriter::PartitionByDefined(const MachOSym &Sym) {
+ // FIXME: Do N_ABS or N_INDR count as defined?
+ return (Sym.n_type & MachOSym::N_SECT) == MachOSym::N_SECT;
+}
+
+/// BufferSymbolAndStringTable - Sort the symbols we encountered and assign them
+/// each a string table index so that they appear in the correct order in the
+/// output file.
+void MachOWriter::BufferSymbolAndStringTable() {
+ // The order of the symbol table is:
+ // 1. local symbols
+ // 2. defined external symbols (sorted by name)
+ // 3. undefined external symbols (sorted by name)
+
+ // Before sorting the symbols, check the PendingGlobals for any undefined
+ // globals that need to be put in the symbol table.
+ for (std::vector<GlobalValue*>::iterator I = PendingGlobals.begin(),
+ E = PendingGlobals.end(); I != E; ++I) {
+ if (GVOffset[*I] == 0 && GVSection[*I] == 0) {
+ MachOSym UndfSym(*I, Mang->getValueName(*I), MachOSym::NO_SECT, TM);
+ SymbolTable.push_back(UndfSym);
+ GVOffset[*I] = -1;
+ }
+ }
+
+ // Sort the symbols by name, so that when we partition the symbols by scope
+ // of definition, we won't have to sort by name within each partition.
+ std::sort(SymbolTable.begin(), SymbolTable.end(), MachOSymCmp());
+
+ // Parition the symbol table entries so that all local symbols come before
+ // all symbols with external linkage. { 1 | 2 3 }
+ std::partition(SymbolTable.begin(), SymbolTable.end(), PartitionByLocal);
+
+ // Advance iterator to beginning of external symbols and partition so that
+ // all external symbols defined in this module come before all external
+ // symbols defined elsewhere. { 1 | 2 | 3 }
+ for (std::vector<MachOSym>::iterator I = SymbolTable.begin(),
+ E = SymbolTable.end(); I != E; ++I) {
+ if (!PartitionByLocal(*I)) {
+ std::partition(I, E, PartitionByDefined);
+ break;
+ }
+ }
+
+ // Calculate the starting index for each of the local, extern defined, and
+ // undefined symbols, as well as the number of each to put in the LC_DYSYMTAB
+ // load command.
+ for (std::vector<MachOSym>::iterator I = SymbolTable.begin(),
+ E = SymbolTable.end(); I != E; ++I) {
+ if (PartitionByLocal(*I)) {
+ ++DySymTab.nlocalsym;
+ ++DySymTab.iextdefsym;
+ ++DySymTab.iundefsym;
+ } else if (PartitionByDefined(*I)) {
+ ++DySymTab.nextdefsym;
+ ++DySymTab.iundefsym;
+ } else {
+ ++DySymTab.nundefsym;
+ }
+ }
+
+ // Write out a leading zero byte when emitting string table, for n_strx == 0
+ // which means an empty string.
+ OutputBuffer StrTOut(StrT, is64Bit, isLittleEndian);
+ StrTOut.outbyte(0);
+
+ // The order of the string table is:
+ // 1. strings for external symbols
+ // 2. strings for local symbols
+ // Since this is the opposite order from the symbol table, which we have just
+ // sorted, we can walk the symbol table backwards to output the string table.
+ for (std::vector<MachOSym>::reverse_iterator I = SymbolTable.rbegin(),
+ E = SymbolTable.rend(); I != E; ++I) {
+ if (I->GVName == "") {
+ I->n_strx = 0;
+ } else {
+ I->n_strx = StrT.size();
+ StrTOut.outstring(I->GVName, I->GVName.length()+1);
+ }
+ }
+
+ OutputBuffer SymTOut(SymT, is64Bit, isLittleEndian);
+
+ unsigned index = 0;
+ for (std::vector<MachOSym>::iterator I = SymbolTable.begin(),
+ E = SymbolTable.end(); I != E; ++I, ++index) {
+ // Add the section base address to the section offset in the n_value field
+ // to calculate the full address.
+ // FIXME: handle symbols where the n_value field is not the address
+ GlobalValue *GV = const_cast<GlobalValue*>(I->GV);
+ if (GV && GVSection[GV])
+ I->n_value += GVSection[GV]->addr;
+ if (GV && (GVOffset[GV] == -1))
+ GVOffset[GV] = index;
+
+ // Emit nlist to buffer
+ SymTOut.outword(I->n_strx);
+ SymTOut.outbyte(I->n_type);
+ SymTOut.outbyte(I->n_sect);
+ SymTOut.outhalf(I->n_desc);
+ SymTOut.outaddr(I->n_value);
+ }
+}
+
+/// CalculateRelocations - For each MachineRelocation in the current section,
+/// calculate the index of the section containing the object to be relocated,
+/// and the offset into that section. From this information, create the
+/// appropriate target-specific MachORelocation type and add buffer it to be
+/// written out after we are finished writing out sections.
+void MachOWriter::CalculateRelocations(MachOSection &MOS) {
+ for (unsigned i = 0, e = MOS.Relocations.size(); i != e; ++i) {
+ MachineRelocation &MR = MOS.Relocations[i];
+ unsigned TargetSection = MR.getConstantVal();
+ unsigned TargetAddr = 0;
+ unsigned TargetIndex = 0;
+
+ // This is a scattered relocation entry if it points to a global value with
+ // a non-zero offset.
+ bool Scattered = false;
+ bool Extern = false;
+
+ // Since we may not have seen the GlobalValue we were interested in yet at
+ // the time we emitted the relocation for it, fix it up now so that it
+ // points to the offset into the correct section.
+ if (MR.isGlobalValue()) {
+ GlobalValue *GV = MR.getGlobalValue();
+ MachOSection *MOSPtr = GVSection[GV];
+ intptr_t Offset = GVOffset[GV];
+
+ // If we have never seen the global before, it must be to a symbol
+ // defined in another module (N_UNDF).
+ if (!MOSPtr) {
+ // FIXME: need to append stub suffix
+ Extern = true;
+ TargetAddr = 0;
+ TargetIndex = GVOffset[GV];
+ } else {
+ Scattered = TargetSection != 0;
+ TargetSection = MOSPtr->Index;
+ }
+ MR.setResultPointer((void*)Offset);
+ }
+
+ // If the symbol is locally defined, pass in the address of the section and
+ // the section index to the code which will generate the target relocation.
+ if (!Extern) {
+ MachOSection &To = *SectionList[TargetSection - 1];
+ TargetAddr = To.addr;
+ TargetIndex = To.Index;
+ }
+
+ OutputBuffer RelocOut(MOS.RelocBuffer, is64Bit, isLittleEndian);
+ OutputBuffer SecOut(MOS.SectionData, is64Bit, isLittleEndian);
+
+ MOS.nreloc += GetTargetRelocation(MR, MOS.Index, TargetAddr, TargetIndex,
+ RelocOut, SecOut, Scattered, Extern);
+ }
+}
+
+// InitMem - Write the value of a Constant to the specified memory location,
+// converting it into bytes and relocations.
+void MachOWriter::InitMem(const Constant *C, void *Addr, intptr_t Offset,
+ const TargetData *TD,
+ std::vector<MachineRelocation> &MRs) {
+ typedef std::pair<const Constant*, intptr_t> CPair;
+ std::vector<CPair> WorkList;
+
+ WorkList.push_back(CPair(C,(intptr_t)Addr + Offset));
+
+ intptr_t ScatteredOffset = 0;
+
+ while (!WorkList.empty()) {
+ const Constant *PC = WorkList.back().first;
+ intptr_t PA = WorkList.back().second;
+ WorkList.pop_back();
+
+ if (isa<UndefValue>(PC)) {
+ continue;
+ } else if (const ConstantVector *CP = dyn_cast<ConstantVector>(PC)) {
+ unsigned ElementSize = TD->getTypeSize(CP->getType()->getElementType());
+ for (unsigned i = 0, e = CP->getNumOperands(); i != e; ++i)
+ WorkList.push_back(CPair(CP->getOperand(i), PA+i*ElementSize));
+ } else if (const ConstantExpr *CE = dyn_cast<ConstantExpr>(PC)) {
+ //
+ // FIXME: Handle ConstantExpression. See EE::getConstantValue()
+ //
+ switch (CE->getOpcode()) {
+ case Instruction::GetElementPtr: {
+ SmallVector<Value*, 8> Indices(CE->op_begin()+1, CE->op_end());
+ ScatteredOffset = TD->getIndexedOffset(CE->getOperand(0)->getType(),
+ &Indices[0], Indices.size());
+ WorkList.push_back(CPair(CE->getOperand(0), PA));
+ break;
+ }
+ case Instruction::Add:
+ default:
+ cerr << "ConstantExpr not handled as global var init: " << *CE << "\n";
+ abort();
+ break;
+ }
+ } else if (PC->getType()->isFirstClassType()) {
+ unsigned char *ptr = (unsigned char *)PA;
+ switch (PC->getType()->getTypeID()) {
+ case Type::IntegerTyID: {
+ unsigned NumBits = cast<IntegerType>(PC->getType())->getBitWidth();
+ uint64_t val = cast<ConstantInt>(PC)->getZExtValue();
+ if (NumBits <= 8)
+ ptr[0] = val;
+ else if (NumBits <= 16) {
+ if (TD->isBigEndian())
+ val = ByteSwap_16(val);
+ ptr[0] = val;
+ ptr[1] = val >> 8;
+ } else if (NumBits <= 32) {
+ if (TD->isBigEndian())
+ val = ByteSwap_32(val);
+ ptr[0] = val;
+ ptr[1] = val >> 8;
+ ptr[2] = val >> 16;
+ ptr[3] = val >> 24;
+ } else if (NumBits <= 64) {
+ if (TD->isBigEndian())
+ val = ByteSwap_64(val);
+ ptr[0] = val;
+ ptr[1] = val >> 8;
+ ptr[2] = val >> 16;
+ ptr[3] = val >> 24;
+ ptr[4] = val >> 32;
+ ptr[5] = val >> 40;
+ ptr[6] = val >> 48;
+ ptr[7] = val >> 56;
+ } else {
+ assert(0 && "Not implemented: bit widths > 64");
+ }
+ break;
+ }
+ case Type::FloatTyID: {
+ uint64_t val = FloatToBits(cast<ConstantFP>(PC)->getValue());
+ if (TD->isBigEndian())
+ val = ByteSwap_32(val);
+ ptr[0] = val;
+ ptr[1] = val >> 8;
+ ptr[2] = val >> 16;
+ ptr[3] = val >> 24;
+ break;
+ }
+ case Type::DoubleTyID: {
+ uint64_t val = DoubleToBits(cast<ConstantFP>(PC)->getValue());
+ if (TD->isBigEndian())
+ val = ByteSwap_64(val);
+ ptr[0] = val;
+ ptr[1] = val >> 8;
+ ptr[2] = val >> 16;
+ ptr[3] = val >> 24;
+ ptr[4] = val >> 32;
+ ptr[5] = val >> 40;
+ ptr[6] = val >> 48;
+ ptr[7] = val >> 56;
+ break;
+ }
+ case Type::PointerTyID:
+ if (isa<ConstantPointerNull>(PC))
+ memset(ptr, 0, TD->getPointerSize());
+ else if (const GlobalValue* GV = dyn_cast<GlobalValue>(PC)) {
+ // FIXME: what about function stubs?
+ MRs.push_back(MachineRelocation::getGV(PA-(intptr_t)Addr,
+ MachineRelocation::VANILLA,
+ const_cast<GlobalValue*>(GV),
+ ScatteredOffset));
+ ScatteredOffset = 0;
+ } else
+ assert(0 && "Unknown constant pointer type!");
+ break;
+ default:
+ cerr << "ERROR: Constant unimp for type: " << *PC->getType() << "\n";
+ abort();
+ }
+ } else if (isa<ConstantAggregateZero>(PC)) {
+ memset((void*)PA, 0, (size_t)TD->getTypeSize(PC->getType()));
+ } else if (const ConstantArray *CPA = dyn_cast<ConstantArray>(PC)) {
+ unsigned ElementSize = TD->getTypeSize(CPA->getType()->getElementType());
+ for (unsigned i = 0, e = CPA->getNumOperands(); i != e; ++i)
+ WorkList.push_back(CPair(CPA->getOperand(i), PA+i*ElementSize));
+ } else if (const ConstantStruct *CPS = dyn_cast<ConstantStruct>(PC)) {
+ const StructLayout *SL =
+ TD->getStructLayout(cast<StructType>(CPS->getType()));
+ for (unsigned i = 0, e = CPS->getNumOperands(); i != e; ++i)
+ WorkList.push_back(CPair(CPS->getOperand(i),
+ PA+SL->getElementOffset(i)));
+ } else {
+ cerr << "Bad Type: " << *PC->getType() << "\n";
+ assert(0 && "Unknown constant type to initialize memory with!");
+ }
+ }
+}
+
+MachOSym::MachOSym(const GlobalValue *gv, std::string name, uint8_t sect,
+ TargetMachine &TM) :
+ GV(gv), n_strx(0), n_type(sect == NO_SECT ? N_UNDF : N_SECT), n_sect(sect),
+ n_desc(0), n_value(0) {
+
+ const TargetAsmInfo *TAI = TM.getTargetAsmInfo();
+
+ switch (GV->getLinkage()) {
+ default:
+ assert(0 && "Unexpected linkage type!");
+ break;
+ case GlobalValue::WeakLinkage:
+ case GlobalValue::LinkOnceLinkage:
+ assert(!isa<Function>(gv) && "Unexpected linkage type for Function!");
+ case GlobalValue::ExternalLinkage:
+ GVName = TAI->getGlobalPrefix() + name;
+ n_type |= GV->hasHiddenVisibility() ? N_PEXT : N_EXT;
+ break;
+ case GlobalValue::InternalLinkage:
+ GVName = TAI->getGlobalPrefix() + name;
+ break;
+ }
+}
diff --git a/lib/CodeGen/MachOWriter.h b/lib/CodeGen/MachOWriter.h
new file mode 100644
index 0000000..6d88832
--- /dev/null
+++ b/lib/CodeGen/MachOWriter.h
@@ -0,0 +1,627 @@
+//=== MachOWriter.h - Target-independent Mach-O writer support --*- C++ -*-===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file was developed by Nate Begeman and is distributed under the
+// University of Illinois Open Source License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines the MachOWriter class.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MACHOWRITER_H
+#define MACHOWRITER_H
+
+#include "llvm/Constants.h"
+#include "llvm/DerivedTypes.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/MachineRelocation.h"
+#include "llvm/Target/TargetData.h"
+#include "llvm/Target/TargetMachine.h"
+#include "llvm/Target/TargetMachOWriterInfo.h"
+
+namespace llvm {
+ class GlobalVariable;
+ class Mangler;
+ class MachineCodeEmitter;
+ class MachOCodeEmitter;
+ class OutputBuffer;
+
+ /// MachOSym - This struct contains information about each symbol that is
+ /// added to logical symbol table for the module. This is eventually
+ /// turned into a real symbol table in the file.
+ struct MachOSym {
+ const GlobalValue *GV; // The global value this corresponds to.
+ std::string GVName; // The mangled name of the global value.
+ uint32_t n_strx; // index into the string table
+ uint8_t n_type; // type flag
+ uint8_t n_sect; // section number or NO_SECT
+ int16_t n_desc; // see <mach-o/stab.h>
+ uint64_t n_value; // value for this symbol (or stab offset)
+
+ // Constants for the n_sect field
+ // see <mach-o/nlist.h>
+ enum { NO_SECT = 0 }; // symbol is not in any section
+
+ // Constants for the n_type field
+ // see <mach-o/nlist.h>
+ enum { N_UNDF = 0x0, // undefined, n_sect == NO_SECT
+ N_ABS = 0x2, // absolute, n_sect == NO_SECT
+ N_SECT = 0xe, // defined in section number n_sect
+ N_PBUD = 0xc, // prebound undefined (defined in a dylib)
+ N_INDR = 0xa // indirect
+ };
+ // The following bits are OR'd into the types above. For example, a type
+ // of 0x0f would be an external N_SECT symbol (0x0e | 0x01).
+ enum { N_EXT = 0x01, // external symbol bit
+ N_PEXT = 0x10 // private external symbol bit
+ };
+
+ // Constants for the n_desc field
+ // see <mach-o/loader.h>
+ enum { REFERENCE_FLAG_UNDEFINED_NON_LAZY = 0,
+ REFERENCE_FLAG_UNDEFINED_LAZY = 1,
+ REFERENCE_FLAG_DEFINED = 2,
+ REFERENCE_FLAG_PRIVATE_DEFINED = 3,
+ REFERENCE_FLAG_PRIVATE_UNDEFINED_NON_LAZY = 4,
+ REFERENCE_FLAG_PRIVATE_UNDEFINED_LAZY = 5
+ };
+ enum { N_NO_DEAD_STRIP = 0x0020, // symbol is not to be dead stripped
+ N_WEAK_REF = 0x0040, // symbol is weak referenced
+ N_WEAK_DEF = 0x0080 // coalesced symbol is a weak definition
+ };
+
+ MachOSym(const GlobalValue *gv, std::string name, uint8_t sect,
+ TargetMachine &TM);
+ };
+
+ /// MachOWriter - This class implements the common target-independent code for
+ /// writing Mach-O files. Targets should derive a class from this to
+ /// parameterize the output format.
+ ///
+ class MachOWriter : public MachineFunctionPass {
+ friend class MachOCodeEmitter;
+ public:
+ static char ID;
+ MachineCodeEmitter &getMachineCodeEmitter() const {
+ return *(MachineCodeEmitter*)MCE;
+ }
+
+ MachOWriter(std::ostream &O, TargetMachine &TM);
+ virtual ~MachOWriter();
+
+ virtual const char *getPassName() const {
+ return "Mach-O Writer";
+ }
+
+ typedef std::vector<unsigned char> DataBuffer;
+ protected:
+ /// Output stream to send the resultant object file to.
+ ///
+ std::ostream &O;
+
+ /// Target machine description.
+ ///
+ TargetMachine &TM;
+
+ /// Mang - The object used to perform name mangling for this module.
+ ///
+ Mangler *Mang;
+
+ /// MCE - The MachineCodeEmitter object that we are exposing to emit machine
+ /// code for functions to the .o file.
+ MachOCodeEmitter *MCE;
+
+ /// is64Bit/isLittleEndian - This information is inferred from the target
+ /// machine directly, indicating what header values and flags to set.
+ bool is64Bit, isLittleEndian;
+
+ /// doInitialization - Emit the file header and all of the global variables
+ /// for the module to the Mach-O file.
+ bool doInitialization(Module &M);
+
+ bool runOnMachineFunction(MachineFunction &MF);
+
+ /// doFinalization - Now that the module has been completely processed, emit
+ /// the Mach-O file to 'O'.
+ bool doFinalization(Module &M);
+
+ /// MachOHeader - This struct contains the header information about a
+ /// specific architecture type/subtype pair that is emitted to the file.
+ struct MachOHeader {
+ uint32_t magic; // mach magic number identifier
+ uint32_t filetype; // type of file
+ uint32_t ncmds; // number of load commands
+ uint32_t sizeofcmds; // the size of all the load commands
+ uint32_t flags; // flags
+ uint32_t reserved; // 64-bit only
+
+ /// HeaderData - The actual data for the header which we are building
+ /// up for emission to the file.
+ DataBuffer HeaderData;
+
+ // Constants for the filetype field
+ // see <mach-o/loader.h> for additional info on the various types
+ enum { MH_OBJECT = 1, // relocatable object file
+ MH_EXECUTE = 2, // demand paged executable file
+ MH_FVMLIB = 3, // fixed VM shared library file
+ MH_CORE = 4, // core file
+ MH_PRELOAD = 5, // preloaded executable file
+ MH_DYLIB = 6, // dynamically bound shared library
+ MH_DYLINKER = 7, // dynamic link editor
+ MH_BUNDLE = 8, // dynamically bound bundle file
+ MH_DYLIB_STUB = 9, // shared library stub for static linking only
+ MH_DSYM = 10 // companion file wiht only debug sections
+ };
+
+ // Constants for the flags field
+ enum { MH_NOUNDEFS = 1 << 0,
+ // the object file has no undefined references
+ MH_INCRLINK = 1 << 1,
+ // the object file is the output of an incremental link against
+ // a base file and cannot be link edited again
+ MH_DYLDLINK = 1 << 2,
+ // the object file is input for the dynamic linker and cannot be
+ // statically link edited again.
+ MH_BINDATLOAD = 1 << 3,
+ // the object file's undefined references are bound by the
+ // dynamic linker when loaded.
+ MH_PREBOUND = 1 << 4,
+ // the file has its dynamic undefined references prebound
+ MH_SPLIT_SEGS = 1 << 5,
+ // the file has its read-only and read-write segments split
+ // see <mach/shared_memory_server.h>
+ MH_LAZY_INIT = 1 << 6,
+ // the shared library init routine is to be run lazily via
+ // catching memory faults to its writable segments (obsolete)
+ MH_TWOLEVEL = 1 << 7,
+ // the image is using two-level namespace bindings
+ MH_FORCE_FLAT = 1 << 8,
+ // the executable is forcing all images to use flat namespace
+ // bindings.
+ MH_NOMULTIDEFS = 1 << 8,
+ // this umbrella guarantees no multiple definitions of symbols
+ // in its sub-images so the two-level namespace hints can
+ // always be used.
+ MH_NOFIXPREBINDING = 1 << 10,
+ // do not have dyld notify the prebidning agent about this
+ // executable.
+ MH_PREBINDABLE = 1 << 11,
+ // the binary is not prebound but can have its prebinding
+ // redone. only used when MH_PREBOUND is not set.
+ MH_ALLMODSBOUND = 1 << 12,
+ // indicates that this binary binds to all two-level namespace
+ // modules of its dependent libraries. Only used when
+ // MH_PREBINDABLE and MH_TWOLEVEL are both set.
+ MH_SUBSECTIONS_VIA_SYMBOLS = 1 << 13,
+ // safe to divide up the sections into sub-sections via symbols
+ // for dead code stripping.
+ MH_CANONICAL = 1 << 14,
+ // the binary has been canonicalized via the unprebind operation
+ MH_WEAK_DEFINES = 1 << 15,
+ // the final linked image contains external weak symbols
+ MH_BINDS_TO_WEAK = 1 << 16,
+ // the final linked image uses weak symbols
+ MH_ALLOW_STACK_EXECUTION = 1 << 17
+ // When this bit is set, all stacks in the task will be given
+ // stack execution privilege. Only used in MH_EXECUTE filetype
+ };
+
+ MachOHeader() : magic(0), filetype(0), ncmds(0), sizeofcmds(0), flags(0),
+ reserved(0) { }
+
+ /// cmdSize - This routine returns the size of the MachOSection as written
+ /// to disk, depending on whether the destination is a 64 bit Mach-O file.
+ unsigned cmdSize(bool is64Bit) const {
+ if (is64Bit)
+ return 8 * sizeof(uint32_t);
+ else
+ return 7 * sizeof(uint32_t);
+ }
+
+ /// setMagic - This routine sets the appropriate value for the 'magic'
+ /// field based on pointer size and endianness.
+ void setMagic(bool isLittleEndian, bool is64Bit) {
+ if (isLittleEndian)
+ if (is64Bit) magic = 0xcffaedfe;
+ else magic = 0xcefaedfe;
+ else
+ if (is64Bit) magic = 0xfeedfacf;
+ else magic = 0xfeedface;
+ }
+ };
+
+ /// Header - An instance of MachOHeader that we will update while we build
+ /// the file, and then emit during finalization.
+ MachOHeader Header;
+
+ /// MachOSegment - This struct contains the necessary information to
+ /// emit the load commands for each section in the file.
+ struct MachOSegment {
+ uint32_t cmd; // LC_SEGMENT or LC_SEGMENT_64
+ uint32_t cmdsize; // Total size of this struct and section commands
+ std::string segname; // segment name
+ uint64_t vmaddr; // address of this segment
+ uint64_t vmsize; // size of this segment, may be larger than filesize
+ uint64_t fileoff; // offset in file
+ uint64_t filesize; // amount to read from file
+ uint32_t maxprot; // maximum VM protection
+ uint32_t initprot; // initial VM protection
+ uint32_t nsects; // number of sections in this segment
+ uint32_t flags; // flags
+
+ // The following constants are getting pulled in by one of the
+ // system headers, which creates a neat clash with the enum.
+#if !defined(VM_PROT_NONE)
+#define VM_PROT_NONE 0x00
+#endif
+#if !defined(VM_PROT_READ)
+#define VM_PROT_READ 0x01
+#endif
+#if !defined(VM_PROT_WRITE)
+#define VM_PROT_WRITE 0x02
+#endif
+#if !defined(VM_PROT_EXECUTE)
+#define VM_PROT_EXECUTE 0x04
+#endif
+#if !defined(VM_PROT_ALL)
+#define VM_PROT_ALL 0x07
+#endif
+
+ // Constants for the vm protection fields
+ // see <mach-o/vm_prot.h>
+ enum { SEG_VM_PROT_NONE = VM_PROT_NONE,
+ SEG_VM_PROT_READ = VM_PROT_READ, // read permission
+ SEG_VM_PROT_WRITE = VM_PROT_WRITE, // write permission
+ SEG_VM_PROT_EXECUTE = VM_PROT_EXECUTE,
+ SEG_VM_PROT_ALL = VM_PROT_ALL
+ };
+
+ // Constants for the cmd field
+ // see <mach-o/loader.h>
+ enum { LC_SEGMENT = 0x01, // segment of this file to be mapped
+ LC_SEGMENT_64 = 0x19 // 64-bit segment of this file to be mapped
+ };
+
+ /// cmdSize - This routine returns the size of the MachOSection as written
+ /// to disk, depending on whether the destination is a 64 bit Mach-O file.
+ unsigned cmdSize(bool is64Bit) const {
+ if (is64Bit)
+ return 6 * sizeof(uint32_t) + 4 * sizeof(uint64_t) + 16;
+ else
+ return 10 * sizeof(uint32_t) + 16; // addresses only 32 bits
+ }
+
+ MachOSegment(const std::string &seg, bool is64Bit)
+ : cmd(is64Bit ? LC_SEGMENT_64 : LC_SEGMENT), cmdsize(0), segname(seg),
+ vmaddr(0), vmsize(0), fileoff(0), filesize(0), maxprot(VM_PROT_ALL),
+ initprot(VM_PROT_ALL), nsects(0), flags(0) { }
+ };
+
+ /// MachOSection - This struct contains information about each section in a
+ /// particular segment that is emitted to the file. This is eventually
+ /// turned into the SectionCommand in the load command for a particlar
+ /// segment.
+ struct MachOSection {
+ std::string sectname; // name of this section,
+ std::string segname; // segment this section goes in
+ uint64_t addr; // memory address of this section
+ uint64_t size; // size in bytes of this section
+ uint32_t offset; // file offset of this section
+ uint32_t align; // section alignment (power of 2)
+ uint32_t reloff; // file offset of relocation entries
+ uint32_t nreloc; // number of relocation entries
+ uint32_t flags; // flags (section type and attributes)
+ uint32_t reserved1; // reserved (for offset or index)
+ uint32_t reserved2; // reserved (for count or sizeof)
+ uint32_t reserved3; // reserved (64 bit only)
+
+ /// A unique number for this section, which will be used to match symbols
+ /// to the correct section.
+ uint32_t Index;
+
+ /// SectionData - The actual data for this section which we are building
+ /// up for emission to the file.
+ DataBuffer SectionData;
+
+ /// RelocBuffer - A buffer to hold the mach-o relocations before we write
+ /// them out at the appropriate location in the file.
+ DataBuffer RelocBuffer;
+
+ /// Relocations - The relocations that we have encountered so far in this
+ /// section that we will need to convert to MachORelocation entries when
+ /// the file is written.
+ std::vector<MachineRelocation> Relocations;
+
+ // Constants for the section types (low 8 bits of flags field)
+ // see <mach-o/loader.h>
+ enum { S_REGULAR = 0,
+ // regular section
+ S_ZEROFILL = 1,
+ // zero fill on demand section
+ S_CSTRING_LITERALS = 2,
+ // section with only literal C strings
+ S_4BYTE_LITERALS = 3,
+ // section with only 4 byte literals
+ S_8BYTE_LITERALS = 4,
+ // section with only 8 byte literals
+ S_LITERAL_POINTERS = 5,
+ // section with only pointers to literals
+ S_NON_LAZY_SYMBOL_POINTERS = 6,
+ // section with only non-lazy symbol pointers
+ S_LAZY_SYMBOL_POINTERS = 7,
+ // section with only lazy symbol pointers
+ S_SYMBOL_STUBS = 8,
+ // section with only symbol stubs
+ // byte size of stub in the reserved2 field
+ S_MOD_INIT_FUNC_POINTERS = 9,
+ // section with only function pointers for initialization
+ S_MOD_TERM_FUNC_POINTERS = 10,
+ // section with only function pointers for termination
+ S_COALESCED = 11,
+ // section contains symbols that are coalesced
+ S_GB_ZEROFILL = 12,
+ // zero fill on demand section (that can be larger than 4GB)
+ S_INTERPOSING = 13,
+ // section with only pairs of function pointers for interposing
+ S_16BYTE_LITERALS = 14
+ // section with only 16 byte literals
+ };
+
+ // Constants for the section flags (high 24 bits of flags field)
+ // see <mach-o/loader.h>
+ enum { S_ATTR_PURE_INSTRUCTIONS = 1 << 31,
+ // section contains only true machine instructions
+ S_ATTR_NO_TOC = 1 << 30,
+ // section contains coalesced symbols that are not to be in a
+ // ranlib table of contents
+ S_ATTR_STRIP_STATIC_SYMS = 1 << 29,
+ // ok to strip static symbols in this section in files with the
+ // MY_DYLDLINK flag
+ S_ATTR_NO_DEAD_STRIP = 1 << 28,
+ // no dead stripping
+ S_ATTR_LIVE_SUPPORT = 1 << 27,
+ // blocks are live if they reference live blocks
+ S_ATTR_SELF_MODIFYING_CODE = 1 << 26,
+ // used with i386 code stubs written on by dyld
+ S_ATTR_DEBUG = 1 << 25,
+ // a debug section
+ S_ATTR_SOME_INSTRUCTIONS = 1 << 10,
+ // section contains some machine instructions
+ S_ATTR_EXT_RELOC = 1 << 9,
+ // section has external relocation entries
+ S_ATTR_LOC_RELOC = 1 << 8
+ // section has local relocation entries
+ };
+
+ /// cmdSize - This routine returns the size of the MachOSection as written
+ /// to disk, depending on whether the destination is a 64 bit Mach-O file.
+ unsigned cmdSize(bool is64Bit) const {
+ if (is64Bit)
+ return 7 * sizeof(uint32_t) + 2 * sizeof(uint64_t) + 32;
+ else
+ return 9 * sizeof(uint32_t) + 32; // addresses only 32 bits
+ }
+
+ MachOSection(const std::string &seg, const std::string §)
+ : sectname(sect), segname(seg), addr(0), size(0), offset(0), align(2),
+ reloff(0), nreloc(0), flags(0), reserved1(0), reserved2(0),
+ reserved3(0) { }
+ };
+
+ private:
+
+ /// SectionList - This is the list of sections that we have emitted to the
+ /// file. Once the file has been completely built, the segment load command
+ /// SectionCommands are constructed from this info.
+ std::vector<MachOSection*> SectionList;
+
+ /// SectionLookup - This is a mapping from section name to SectionList entry
+ std::map<std::string, MachOSection*> SectionLookup;
+
+ /// GVSection - This is a mapping from a GlobalValue to a MachOSection,
+ /// to aid in emitting relocations.
+ std::map<GlobalValue*, MachOSection*> GVSection;
+
+ /// GVOffset - This is a mapping from a GlobalValue to an offset from the
+ /// start of the section in which the GV resides, to aid in emitting
+ /// relocations.
+ std::map<GlobalValue*, intptr_t> GVOffset;
+
+ /// getSection - Return the section with the specified name, creating a new
+ /// section if one does not already exist.
+ MachOSection *getSection(const std::string &seg, const std::string §,
+ unsigned Flags = 0) {
+ MachOSection *MOS = SectionLookup[seg+sect];
+ if (MOS) return MOS;
+
+ MOS = new MachOSection(seg, sect);
+ SectionList.push_back(MOS);
+ MOS->Index = SectionList.size();
+ MOS->flags = MachOSection::S_REGULAR | Flags;
+ SectionLookup[seg+sect] = MOS;
+ return MOS;
+ }
+ MachOSection *getTextSection(bool isCode = true) {
+ if (isCode)
+ return getSection("__TEXT", "__text",
+ MachOSection::S_ATTR_PURE_INSTRUCTIONS |
+ MachOSection::S_ATTR_SOME_INSTRUCTIONS);
+ else
+ return getSection("__TEXT", "__text");
+ }
+ MachOSection *getBSSSection() {
+ return getSection("__DATA", "__bss", MachOSection::S_ZEROFILL);
+ }
+ MachOSection *getDataSection() {
+ return getSection("__DATA", "__data");
+ }
+ MachOSection *getConstSection(Constant *C) {
+ const ConstantArray *CVA = dyn_cast<ConstantArray>(C);
+ if (CVA && CVA->isCString())
+ return getSection("__TEXT", "__cstring",
+ MachOSection::S_CSTRING_LITERALS);
+
+ const Type *Ty = C->getType();
+ if (Ty->isPrimitiveType() || Ty->isInteger()) {
+ unsigned Size = TM.getTargetData()->getTypeSize(Ty);
+ switch(Size) {
+ default: break; // Fall through to __TEXT,__const
+ case 4:
+ return getSection("__TEXT", "__literal4",
+ MachOSection::S_4BYTE_LITERALS);
+ case 8:
+ return getSection("__TEXT", "__literal8",
+ MachOSection::S_8BYTE_LITERALS);
+ case 16:
+ return getSection("__TEXT", "__literal16",
+ MachOSection::S_16BYTE_LITERALS);
+ }
+ }
+ return getSection("__TEXT", "__const");
+ }
+ MachOSection *getJumpTableSection() {
+ if (TM.getRelocationModel() == Reloc::PIC_)
+ return getTextSection(false);
+ else
+ return getSection("__TEXT", "__const");
+ }
+
+ /// MachOSymTab - This struct contains information about the offsets and
+ /// size of symbol table information.
+ /// segment.
+ struct MachOSymTab {
+ uint32_t cmd; // LC_SYMTAB
+ uint32_t cmdsize; // sizeof( MachOSymTab )
+ uint32_t symoff; // symbol table offset
+ uint32_t nsyms; // number of symbol table entries
+ uint32_t stroff; // string table offset
+ uint32_t strsize; // string table size in bytes
+
+ // Constants for the cmd field
+ // see <mach-o/loader.h>
+ enum { LC_SYMTAB = 0x02 // link-edit stab symbol table info
+ };
+
+ MachOSymTab() : cmd(LC_SYMTAB), cmdsize(6 * sizeof(uint32_t)), symoff(0),
+ nsyms(0), stroff(0), strsize(0) { }
+ };
+
+ /// MachOSymTab - This struct contains information about the offsets and
+ /// size of symbol table information.
+ /// segment.
+ struct MachODySymTab {
+ uint32_t cmd; // LC_DYSYMTAB
+ uint32_t cmdsize; // sizeof( MachODySymTab )
+ uint32_t ilocalsym; // index to local symbols
+ uint32_t nlocalsym; // number of local symbols
+ uint32_t iextdefsym; // index to externally defined symbols
+ uint32_t nextdefsym; // number of externally defined symbols
+ uint32_t iundefsym; // index to undefined symbols
+ uint32_t nundefsym; // number of undefined symbols
+ uint32_t tocoff; // file offset to table of contents
+ uint32_t ntoc; // number of entries in table of contents
+ uint32_t modtaboff; // file offset to module table
+ uint32_t nmodtab; // number of module table entries
+ uint32_t extrefsymoff; // offset to referenced symbol table
+ uint32_t nextrefsyms; // number of referenced symbol table entries
+ uint32_t indirectsymoff; // file offset to the indirect symbol table
+ uint32_t nindirectsyms; // number of indirect symbol table entries
+ uint32_t extreloff; // offset to external relocation entries
+ uint32_t nextrel; // number of external relocation entries
+ uint32_t locreloff; // offset to local relocation entries
+ uint32_t nlocrel; // number of local relocation entries
+
+ // Constants for the cmd field
+ // see <mach-o/loader.h>
+ enum { LC_DYSYMTAB = 0x0B // dynamic link-edit symbol table info
+ };
+
+ MachODySymTab() : cmd(LC_DYSYMTAB), cmdsize(20 * sizeof(uint32_t)),
+ ilocalsym(0), nlocalsym(0), iextdefsym(0), nextdefsym(0),
+ iundefsym(0), nundefsym(0), tocoff(0), ntoc(0), modtaboff(0),
+ nmodtab(0), extrefsymoff(0), nextrefsyms(0), indirectsymoff(0),
+ nindirectsyms(0), extreloff(0), nextrel(0), locreloff(0), nlocrel(0) { }
+ };
+
+ /// SymTab - The "stab" style symbol table information
+ MachOSymTab SymTab;
+ /// DySymTab - symbol table info for the dynamic link editor
+ MachODySymTab DySymTab;
+
+ struct MachOSymCmp {
+ // FIXME: this does not appear to be sorting 'f' after 'F'
+ bool operator()(const MachOSym &LHS, const MachOSym &RHS) {
+ return LHS.GVName < RHS.GVName;
+ }
+ };
+
+ /// PartitionByLocal - Simple boolean predicate that returns true if Sym is
+ /// a local symbol rather than an external symbol.
+ static bool PartitionByLocal(const MachOSym &Sym);
+
+ /// PartitionByDefined - Simple boolean predicate that returns true if Sym
+ /// is defined in this module.
+ static bool PartitionByDefined(const MachOSym &Sym);
+
+ protected:
+
+ /// SymbolTable - This is the list of symbols we have emitted to the file.
+ /// This actually gets rearranged before emission to the file (to put the
+ /// local symbols first in the list).
+ std::vector<MachOSym> SymbolTable;
+
+ /// SymT - A buffer to hold the symbol table before we write it out at the
+ /// appropriate location in the file.
+ DataBuffer SymT;
+
+ /// StrT - A buffer to hold the string table before we write it out at the
+ /// appropriate location in the file.
+ DataBuffer StrT;
+
+ /// PendingSyms - This is a list of externally defined symbols that we have
+ /// been asked to emit, but have not seen a reference to. When a reference
+ /// is seen, the symbol will move from this list to the SymbolTable.
+ std::vector<GlobalValue*> PendingGlobals;
+
+ /// DynamicSymbolTable - This is just a vector of indices into
+ /// SymbolTable to aid in emitting the DYSYMTAB load command.
+ std::vector<unsigned> DynamicSymbolTable;
+
+ static void InitMem(const Constant *C, void *Addr, intptr_t Offset,
+ const TargetData *TD,
+ std::vector<MachineRelocation> &MRs);
+
+ private:
+ void AddSymbolToSection(MachOSection *MOS, GlobalVariable *GV);
+ void EmitGlobal(GlobalVariable *GV);
+ void EmitHeaderAndLoadCommands();
+ void EmitSections();
+ void BufferSymbolAndStringTable();
+ void CalculateRelocations(MachOSection &MOS);
+
+ MachineRelocation GetJTRelocation(unsigned Offset,
+ MachineBasicBlock *MBB) const {
+ return TM.getMachOWriterInfo()->GetJTRelocation(Offset, MBB);
+ }
+
+ /// GetTargetRelocation - Returns the number of relocations.
+ unsigned GetTargetRelocation(MachineRelocation &MR,
+ unsigned FromIdx,
+ unsigned ToAddr,
+ unsigned ToIndex,
+ OutputBuffer &RelocOut,
+ OutputBuffer &SecOut,
+ bool Scattered,
+ bool Extern) {
+ return TM.getMachOWriterInfo()->GetTargetRelocation(MR, FromIdx, ToAddr,
+ ToIndex, RelocOut,
+ SecOut, Scattered,
+ Extern);
+ }
+ };
+}
+
+#endif
diff --git a/lib/CodeGen/MachineBasicBlock.cpp b/lib/CodeGen/MachineBasicBlock.cpp
new file mode 100644
index 0000000..ba428c5
--- /dev/null
+++ b/lib/CodeGen/MachineBasicBlock.cpp
@@ -0,0 +1,287 @@
+//===-- llvm/CodeGen/MachineBasicBlock.cpp ----------------------*- C++ -*-===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file was developed by the LLVM research group and is distributed under
+// the University of Illinois Open Source License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// Collect the sequence of machine instructions for a basic block.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/CodeGen/MachineBasicBlock.h"
+#include "llvm/BasicBlock.h"
+#include "llvm/CodeGen/MachineFunction.h"
+#include "llvm/CodeGen/MachineInstr.h"
+#include "llvm/Target/MRegisterInfo.h"
+#include "llvm/Target/TargetData.h"
+#include "llvm/Target/TargetInstrInfo.h"
+#include "llvm/Target/TargetMachine.h"
+#include "llvm/Support/LeakDetector.h"
+#include <algorithm>
+using namespace llvm;
+
+MachineBasicBlock::~MachineBasicBlock() {
+ LeakDetector::removeGarbageObject(this);
+}
+
+std::ostream& llvm::operator<<(std::ostream &OS, const MachineBasicBlock &MBB) {
+ MBB.print(OS);
+ return OS;
+}
+
+// MBBs start out as #-1. When a MBB is added to a MachineFunction, it
+// gets the next available unique MBB number. If it is removed from a
+// MachineFunction, it goes back to being #-1.
+void ilist_traits<MachineBasicBlock>::addNodeToList(MachineBasicBlock* N) {
+ assert(N->Parent == 0 && "machine instruction already in a basic block");
+ N->Parent = Parent;
+ N->Number = Parent->addToMBBNumbering(N);
+ LeakDetector::removeGarbageObject(N);
+}
+
+void ilist_traits<MachineBasicBlock>::removeNodeFromList(MachineBasicBlock* N) {
+ assert(N->Parent != 0 && "machine instruction not in a basic block");
+ N->Parent->removeFromMBBNumbering(N->Number);
+ N->Number = -1;
+ N->Parent = 0;
+ LeakDetector::addGarbageObject(N);
+}
+
+
+MachineInstr* ilist_traits<MachineInstr>::createSentinel() {
+ MachineInstr* dummy = new MachineInstr();
+ LeakDetector::removeGarbageObject(dummy);
+ return dummy;
+}
+
+void ilist_traits<MachineInstr>::addNodeToList(MachineInstr* N) {
+ assert(N->parent == 0 && "machine instruction already in a basic block");
+ N->parent = parent;
+ LeakDetector::removeGarbageObject(N);
+}
+
+void ilist_traits<MachineInstr>::removeNodeFromList(MachineInstr* N) {
+ assert(N->parent != 0 && "machine instruction not in a basic block");
+ N->parent = 0;
+ LeakDetector::addGarbageObject(N);
+}
+
+void ilist_traits<MachineInstr>::transferNodesFromList(
+ iplist<MachineInstr, ilist_traits<MachineInstr> >& fromList,
+ ilist_iterator<MachineInstr> first,
+ ilist_iterator<MachineInstr> last) {
+ if (parent != fromList.parent)
+ for (; first != last; ++first)
+ first->parent = parent;
+}
+
+MachineBasicBlock::iterator MachineBasicBlock::getFirstTerminator() {
+ const TargetInstrInfo& TII = *getParent()->getTarget().getInstrInfo();
+ iterator I = end();
+ while (I != begin() && TII.isTerminatorInstr((--I)->getOpcode()));
+ if (I != end() && !TII.isTerminatorInstr(I->getOpcode())) ++I;
+ return I;
+}
+
+void MachineBasicBlock::dump() const {
+ print(*cerr.stream());
+}
+
+static inline void OutputReg(std::ostream &os, unsigned RegNo,
+ const MRegisterInfo *MRI = 0) {
+ if (!RegNo || MRegisterInfo::isPhysicalRegister(RegNo)) {
+ if (MRI)
+ os << " %" << MRI->get(RegNo).Name;
+ else
+ os << " %mreg(" << RegNo << ")";
+ } else
+ os << " %reg" << RegNo;
+}
+
+void MachineBasicBlock::print(std::ostream &OS) const {
+ const MachineFunction *MF = getParent();
+ if(!MF) {
+ OS << "Can't print out MachineBasicBlock because parent MachineFunction"
+ << " is null\n";
+ return;
+ }
+
+ const BasicBlock *LBB = getBasicBlock();
+ OS << "\n";
+ if (LBB) OS << LBB->getName() << ": ";
+ OS << (const void*)this
+ << ", LLVM BB @" << (const void*) LBB << ", ID#" << getNumber();
+ if (isLandingPad()) OS << ", EH LANDING PAD";
+ OS << ":\n";
+
+ const MRegisterInfo *MRI = MF->getTarget().getRegisterInfo();
+ if (livein_begin() != livein_end()) {
+ OS << "Live Ins:";
+ for (const_livein_iterator I = livein_begin(),E = livein_end(); I != E; ++I)
+ OutputReg(OS, *I, MRI);
+ OS << "\n";
+ }
+ // Print the preds of this block according to the CFG.
+ if (!pred_empty()) {
+ OS << " Predecessors according to CFG:";
+ for (const_pred_iterator PI = pred_begin(), E = pred_end(); PI != E; ++PI)
+ OS << " " << *PI << " (#" << (*PI)->getNumber() << ")";
+ OS << "\n";
+ }
+
+ for (const_iterator I = begin(); I != end(); ++I) {
+ OS << "\t";
+ I->print(OS, &getParent()->getTarget());
+ }
+
+ // Print the successors of this block according to the CFG.
+ if (!succ_empty()) {
+ OS << " Successors according to CFG:";
+ for (const_succ_iterator SI = succ_begin(), E = succ_end(); SI != E; ++SI)
+ OS << " " << *SI << " (#" << (*SI)->getNumber() << ")";
+ OS << "\n";
+ }
+}
+
+void MachineBasicBlock::removeLiveIn(unsigned Reg) {
+ livein_iterator I = std::find(livein_begin(), livein_end(), Reg);
+ assert(I != livein_end() && "Not a live in!");
+ LiveIns.erase(I);
+}
+
+void MachineBasicBlock::moveBefore(MachineBasicBlock *NewAfter) {
+ MachineFunction::BasicBlockListType &BBList =getParent()->getBasicBlockList();
+ getParent()->getBasicBlockList().splice(NewAfter, BBList, this);
+}
+
+void MachineBasicBlock::moveAfter(MachineBasicBlock *NewBefore) {
+ MachineFunction::BasicBlockListType &BBList =getParent()->getBasicBlockList();
+ MachineFunction::iterator BBI = NewBefore;
+ getParent()->getBasicBlockList().splice(++BBI, BBList, this);
+}
+
+
+void MachineBasicBlock::addSuccessor(MachineBasicBlock *succ) {
+ Successors.push_back(succ);
+ succ->addPredecessor(this);
+}
+
+void MachineBasicBlock::removeSuccessor(MachineBasicBlock *succ) {
+ succ->removePredecessor(this);
+ succ_iterator I = std::find(Successors.begin(), Successors.end(), succ);
+ assert(I != Successors.end() && "Not a current successor!");
+ Successors.erase(I);
+}
+
+MachineBasicBlock::succ_iterator MachineBasicBlock::removeSuccessor(succ_iterator I) {
+ assert(I != Successors.end() && "Not a current successor!");
+ (*I)->removePredecessor(this);
+ return(Successors.erase(I));
+}
+
+void MachineBasicBlock::addPredecessor(MachineBasicBlock *pred) {
+ Predecessors.push_back(pred);
+}
+
+void MachineBasicBlock::removePredecessor(MachineBasicBlock *pred) {
+ std::vector<MachineBasicBlock *>::iterator I =
+ std::find(Predecessors.begin(), Predecessors.end(), pred);
+ assert(I != Predecessors.end() && "Pred is not a predecessor of this block!");
+ Predecessors.erase(I);
+}
+
+bool MachineBasicBlock::isSuccessor(MachineBasicBlock *MBB) const {
+ std::vector<MachineBasicBlock *>::const_iterator I =
+ std::find(Successors.begin(), Successors.end(), MBB);
+ return I != Successors.end();
+}
+
+/// ReplaceUsesOfBlockWith - Given a machine basic block that branched to
+/// 'Old', change the code and CFG so that it branches to 'New' instead.
+void MachineBasicBlock::ReplaceUsesOfBlockWith(MachineBasicBlock *Old,
+ MachineBasicBlock *New) {
+ assert(Old != New && "Cannot replace self with self!");
+
+ MachineBasicBlock::iterator I = end();
+ while (I != begin()) {
+ --I;
+ if (!(I->getInstrDescriptor()->Flags & M_TERMINATOR_FLAG)) break;
+
+ // Scan the operands of this machine instruction, replacing any uses of Old
+ // with New.
+ for (unsigned i = 0, e = I->getNumOperands(); i != e; ++i)
+ if (I->getOperand(i).isMachineBasicBlock() &&
+ I->getOperand(i).getMachineBasicBlock() == Old)
+ I->getOperand(i).setMachineBasicBlock(New);
+ }
+
+ // Update the successor information. If New was already a successor, just
+ // remove the link to Old instead of creating another one. PR 1444.
+ removeSuccessor(Old);
+ if (!isSuccessor(New))
+ addSuccessor(New);
+}
+
+/// CorrectExtraCFGEdges - Various pieces of code can cause excess edges in the
+/// CFG to be inserted. If we have proven that MBB can only branch to DestA and
+/// DestB, remove any other MBB successors from the CFG. DestA and DestB can
+/// be null.
+/// Besides DestA and DestB, retain other edges leading to LandingPads (currently
+/// there can be only one; we don't check or require that here).
+/// Note it is possible that DestA and/or DestB are LandingPads.
+bool MachineBasicBlock::CorrectExtraCFGEdges(MachineBasicBlock *DestA,
+ MachineBasicBlock *DestB,
+ bool isCond) {
+ bool MadeChange = false;
+ bool AddedFallThrough = false;
+
+ MachineBasicBlock *FallThru = getNext();
+
+ // If this block ends with a conditional branch that falls through to its
+ // successor, set DestB as the successor.
+ if (isCond) {
+ if (DestB == 0 && FallThru != getParent()->end()) {
+ DestB = FallThru;
+ AddedFallThrough = true;
+ }
+ } else {
+ // If this is an unconditional branch with no explicit dest, it must just be
+ // a fallthrough into DestB.
+ if (DestA == 0 && FallThru != getParent()->end()) {
+ DestA = FallThru;
+ AddedFallThrough = true;
+ }
+ }
+
+ MachineBasicBlock::succ_iterator SI = succ_begin();
+ MachineBasicBlock *OrigDestA = DestA, *OrigDestB = DestB;
+ while (SI != succ_end()) {
+ if (*SI == DestA && DestA == DestB) {
+ DestA = DestB = 0;
+ ++SI;
+ } else if (*SI == DestA) {
+ DestA = 0;
+ ++SI;
+ } else if (*SI == DestB) {
+ DestB = 0;
+ ++SI;
+ } else if ((*SI)->isLandingPad() &&
+ *SI!=OrigDestA && *SI!=OrigDestB) {
+ ++SI;
+ } else {
+ // Otherwise, this is a superfluous edge, remove it.
+ SI = removeSuccessor(SI);
+ MadeChange = true;
+ }
+ }
+ if (!AddedFallThrough) {
+ assert(DestA == 0 && DestB == 0 &&
+ "MachineCFG is missing edges!");
+ } else if (isCond) {
+ assert(DestA == 0 && "MachineCFG is missing edges!");
+ }
+ return MadeChange;
+}
diff --git a/lib/CodeGen/MachineFunction.cpp b/lib/CodeGen/MachineFunction.cpp
new file mode 100644
index 0000000..c762ae5
--- /dev/null
+++ b/lib/CodeGen/MachineFunction.cpp
@@ -0,0 +1,483 @@
+//===-- MachineFunction.cpp -----------------------------------------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file was developed by the LLVM research group and is distributed under
+// the University of Illinois Open Source License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// Collect native machine code information for a function. This allows
+// target-specific information about the generated code to be stored with each
+// function.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/DerivedTypes.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/MachineInstr.h"
+#include "llvm/CodeGen/SSARegMap.h"
+#include "llvm/CodeGen/MachineFrameInfo.h"
+#include "llvm/CodeGen/MachineConstantPool.h"
+#include "llvm/CodeGen/MachineJumpTableInfo.h"
+#include "llvm/CodeGen/Passes.h"
+#include "llvm/Target/TargetData.h"
+#include "llvm/Target/TargetLowering.h"
+#include "llvm/Target/TargetMachine.h"
+#include "llvm/Target/TargetFrameInfo.h"
+#include "llvm/Function.h"
+#include "llvm/Instructions.h"
+#include "llvm/Support/Compiler.h"
+#include "llvm/Support/GraphWriter.h"
+#include "llvm/Support/LeakDetector.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/Config/config.h"
+#include <fstream>
+#include <sstream>
+using namespace llvm;
+
+static AnnotationID MF_AID(
+ AnnotationManager::getID("CodeGen::MachineCodeForFunction"));
+
+// Out of line virtual function to home classes.
+void MachineFunctionPass::virtfn() {}
+
+namespace {
+ struct VISIBILITY_HIDDEN Printer : public MachineFunctionPass {
+ static char ID;
+
+ std::ostream *OS;
+ const std::string Banner;
+
+ Printer (std::ostream *_OS, const std::string &_Banner)
+ : MachineFunctionPass((intptr_t)&ID), OS (_OS), Banner (_Banner) { }
+
+ const char *getPassName() const { return "MachineFunction Printer"; }
+
+ virtual void getAnalysisUsage(AnalysisUsage &AU) const {
+ AU.setPreservesAll();
+ }
+
+ bool runOnMachineFunction(MachineFunction &MF) {
+ (*OS) << Banner;
+ MF.print (*OS);
+ return false;
+ }
+ };
+ char Printer::ID = 0;
+}
+
+/// Returns a newly-created MachineFunction Printer pass. The default output
+/// stream is std::cerr; the default banner is empty.
+///
+FunctionPass *llvm::createMachineFunctionPrinterPass(std::ostream *OS,
+ const std::string &Banner){
+ return new Printer(OS, Banner);
+}
+
+namespace {
+ struct VISIBILITY_HIDDEN Deleter : public MachineFunctionPass {
+ static char ID;
+ Deleter() : MachineFunctionPass((intptr_t)&ID) {}
+
+ const char *getPassName() const { return "Machine Code Deleter"; }
+
+ bool runOnMachineFunction(MachineFunction &MF) {
+ // Delete the annotation from the function now.
+ MachineFunction::destruct(MF.getFunction());
+ return true;
+ }
+ };
+ char Deleter::ID = 0;
+}
+
+/// MachineCodeDeletion Pass - This pass deletes all of the machine code for
+/// the current function, which should happen after the function has been
+/// emitted to a .s file or to memory.
+FunctionPass *llvm::createMachineCodeDeleter() {
+ return new Deleter();
+}
+
+
+
+//===---------------------------------------------------------------------===//
+// MachineFunction implementation
+//===---------------------------------------------------------------------===//
+
+MachineBasicBlock* ilist_traits<MachineBasicBlock>::createSentinel() {
+ MachineBasicBlock* dummy = new MachineBasicBlock();
+ LeakDetector::removeGarbageObject(dummy);
+ return dummy;
+}
+
+void ilist_traits<MachineBasicBlock>::transferNodesFromList(
+ iplist<MachineBasicBlock, ilist_traits<MachineBasicBlock> >& toList,
+ ilist_iterator<MachineBasicBlock> first,
+ ilist_iterator<MachineBasicBlock> last) {
+ if (Parent != toList.Parent)
+ for (; first != last; ++first)
+ first->Parent = toList.Parent;
+}
+
+MachineFunction::MachineFunction(const Function *F,
+ const TargetMachine &TM)
+ : Annotation(MF_AID), Fn(F), Target(TM) {
+ SSARegMapping = new SSARegMap();
+ MFInfo = 0;
+ FrameInfo = new MachineFrameInfo();
+ ConstantPool = new MachineConstantPool(TM.getTargetData());
+ UsedPhysRegs.resize(TM.getRegisterInfo()->getNumRegs());
+
+ // Set up jump table.
+ const TargetData &TD = *TM.getTargetData();
+ bool IsPic = TM.getRelocationModel() == Reloc::PIC_;
+ unsigned EntrySize = IsPic ? 4 : TD.getPointerSize();
+ unsigned Alignment = IsPic ? TD.getABITypeAlignment(Type::Int32Ty)
+ : TD.getPointerABIAlignment();
+ JumpTableInfo = new MachineJumpTableInfo(EntrySize, Alignment);
+
+ BasicBlocks.Parent = this;
+}
+
+MachineFunction::~MachineFunction() {
+ BasicBlocks.clear();
+ delete SSARegMapping;
+ delete MFInfo;
+ delete FrameInfo;
+ delete ConstantPool;
+ delete JumpTableInfo;
+}
+
+
+/// RenumberBlocks - This discards all of the MachineBasicBlock numbers and
+/// recomputes them. This guarantees that the MBB numbers are sequential,
+/// dense, and match the ordering of the blocks within the function. If a
+/// specific MachineBasicBlock is specified, only that block and those after
+/// it are renumbered.
+void MachineFunction::RenumberBlocks(MachineBasicBlock *MBB) {
+ if (empty()) { MBBNumbering.clear(); return; }
+ MachineFunction::iterator MBBI, E = end();
+ if (MBB == 0)
+ MBBI = begin();
+ else
+ MBBI = MBB;
+
+ // Figure out the block number this should have.
+ unsigned BlockNo = 0;
+ if (MBBI != begin())
+ BlockNo = prior(MBBI)->getNumber()+1;
+
+ for (; MBBI != E; ++MBBI, ++BlockNo) {
+ if (MBBI->getNumber() != (int)BlockNo) {
+ // Remove use of the old number.
+ if (MBBI->getNumber() != -1) {
+ assert(MBBNumbering[MBBI->getNumber()] == &*MBBI &&
+ "MBB number mismatch!");
+ MBBNumbering[MBBI->getNumber()] = 0;
+ }
+
+ // If BlockNo is already taken, set that block's number to -1.
+ if (MBBNumbering[BlockNo])
+ MBBNumbering[BlockNo]->setNumber(-1);
+
+ MBBNumbering[BlockNo] = MBBI;
+ MBBI->setNumber(BlockNo);
+ }
+ }
+
+ // Okay, all the blocks are renumbered. If we have compactified the block
+ // numbering, shrink MBBNumbering now.
+ assert(BlockNo <= MBBNumbering.size() && "Mismatch!");
+ MBBNumbering.resize(BlockNo);
+}
+
+
+void MachineFunction::dump() const { print(*cerr.stream()); }
+
+void MachineFunction::print(std::ostream &OS) const {
+ OS << "# Machine code for " << Fn->getName () << "():\n";
+
+ // Print Frame Information
+ getFrameInfo()->print(*this, OS);
+
+ // Print JumpTable Information
+ getJumpTableInfo()->print(OS);
+
+ // Print Constant Pool
+ getConstantPool()->print(OS);
+
+ const MRegisterInfo *MRI = getTarget().getRegisterInfo();
+
+ if (livein_begin() != livein_end()) {
+ OS << "Live Ins:";
+ for (livein_iterator I = livein_begin(), E = livein_end(); I != E; ++I) {
+ if (MRI)
+ OS << " " << MRI->getName(I->first);
+ else
+ OS << " Reg #" << I->first;
+
+ if (I->second)
+ OS << " in VR#" << I->second << " ";
+ }
+ OS << "\n";
+ }
+ if (liveout_begin() != liveout_end()) {
+ OS << "Live Outs:";
+ for (liveout_iterator I = liveout_begin(), E = liveout_end(); I != E; ++I)
+ if (MRI)
+ OS << " " << MRI->getName(*I);
+ else
+ OS << " Reg #" << *I;
+ OS << "\n";
+ }
+
+ for (const_iterator BB = begin(); BB != end(); ++BB)
+ BB->print(OS);
+
+ OS << "\n# End machine code for " << Fn->getName () << "().\n\n";
+}
+
+/// CFGOnly flag - This is used to control whether or not the CFG graph printer
+/// prints out the contents of basic blocks or not. This is acceptable because
+/// this code is only really used for debugging purposes.
+///
+static bool CFGOnly = false;
+
+namespace llvm {
+ template<>
+ struct DOTGraphTraits<const MachineFunction*> : public DefaultDOTGraphTraits {
+ static std::string getGraphName(const MachineFunction *F) {
+ return "CFG for '" + F->getFunction()->getName() + "' function";
+ }
+
+ static std::string getNodeLabel(const MachineBasicBlock *Node,
+ const MachineFunction *Graph) {
+ if (CFGOnly && Node->getBasicBlock() &&
+ !Node->getBasicBlock()->getName().empty())
+ return Node->getBasicBlock()->getName() + ":";
+
+ std::ostringstream Out;
+ if (CFGOnly) {
+ Out << Node->getNumber() << ':';
+ return Out.str();
+ }
+
+ Node->print(Out);
+
+ std::string OutStr = Out.str();
+ if (OutStr[0] == '\n') OutStr.erase(OutStr.begin());
+
+ // Process string output to make it nicer...
+ for (unsigned i = 0; i != OutStr.length(); ++i)
+ if (OutStr[i] == '\n') { // Left justify
+ OutStr[i] = '\\';
+ OutStr.insert(OutStr.begin()+i+1, 'l');
+ }
+ return OutStr;
+ }
+ };
+}
+
+void MachineFunction::viewCFG() const
+{
+#ifndef NDEBUG
+ ViewGraph(this, "mf" + getFunction()->getName());
+#else
+ cerr << "SelectionDAG::viewGraph is only available in debug builds on "
+ << "systems with Graphviz or gv!\n";
+#endif // NDEBUG
+}
+
+void MachineFunction::viewCFGOnly() const
+{
+ CFGOnly = true;
+ viewCFG();
+ CFGOnly = false;
+}
+
+// The next two methods are used to construct and to retrieve
+// the MachineCodeForFunction object for the given function.
+// construct() -- Allocates and initializes for a given function and target
+// get() -- Returns a handle to the object.
+// This should not be called before "construct()"
+// for a given Function.
+//
+MachineFunction&
+MachineFunction::construct(const Function *Fn, const TargetMachine &Tar)
+{
+ assert(Fn->getAnnotation(MF_AID) == 0 &&
+ "Object already exists for this function!");
+ MachineFunction* mcInfo = new MachineFunction(Fn, Tar);
+ Fn->addAnnotation(mcInfo);
+ return *mcInfo;
+}
+
+void MachineFunction::destruct(const Function *Fn) {
+ bool Deleted = Fn->deleteAnnotation(MF_AID);
+ assert(Deleted && "Machine code did not exist for function!");
+}
+
+MachineFunction& MachineFunction::get(const Function *F)
+{
+ MachineFunction *mc = (MachineFunction*)F->getAnnotation(MF_AID);
+ assert(mc && "Call construct() method first to allocate the object");
+ return *mc;
+}
+
+void MachineFunction::clearSSARegMap() {
+ delete SSARegMapping;
+ SSARegMapping = 0;
+}
+
+//===----------------------------------------------------------------------===//
+// MachineFrameInfo implementation
+//===----------------------------------------------------------------------===//
+
+void MachineFrameInfo::print(const MachineFunction &MF, std::ostream &OS) const{
+ int ValOffset = MF.getTarget().getFrameInfo()->getOffsetOfLocalArea();
+
+ for (unsigned i = 0, e = Objects.size(); i != e; ++i) {
+ const StackObject &SO = Objects[i];
+ OS << " <fi #" << (int)(i-NumFixedObjects) << ">: ";
+ if (SO.Size == 0)
+ OS << "variable sized";
+ else
+ OS << "size is " << SO.Size << " byte" << (SO.Size != 1 ? "s," : ",");
+ OS << " alignment is " << SO.Alignment << " byte"
+ << (SO.Alignment != 1 ? "s," : ",");
+
+ if (i < NumFixedObjects)
+ OS << " fixed";
+ if (i < NumFixedObjects || SO.SPOffset != -1) {
+ int64_t Off = SO.SPOffset - ValOffset;
+ OS << " at location [SP";
+ if (Off > 0)
+ OS << "+" << Off;
+ else if (Off < 0)
+ OS << Off;
+ OS << "]";
+ }
+ OS << "\n";
+ }
+
+ if (HasVarSizedObjects)
+ OS << " Stack frame contains variable sized objects\n";
+}
+
+void MachineFrameInfo::dump(const MachineFunction &MF) const {
+ print(MF, *cerr.stream());
+}
+
+
+//===----------------------------------------------------------------------===//
+// MachineJumpTableInfo implementation
+//===----------------------------------------------------------------------===//
+
+/// getJumpTableIndex - Create a new jump table entry in the jump table info
+/// or return an existing one.
+///
+unsigned MachineJumpTableInfo::getJumpTableIndex(
+ const std::vector<MachineBasicBlock*> &DestBBs) {
+ assert(!DestBBs.empty() && "Cannot create an empty jump table!");
+ for (unsigned i = 0, e = JumpTables.size(); i != e; ++i)
+ if (JumpTables[i].MBBs == DestBBs)
+ return i;
+
+ JumpTables.push_back(MachineJumpTableEntry(DestBBs));
+ return JumpTables.size()-1;
+}
+
+
+void MachineJumpTableInfo::print(std::ostream &OS) const {
+ // FIXME: this is lame, maybe we could print out the MBB numbers or something
+ // like {1, 2, 4, 5, 3, 0}
+ for (unsigned i = 0, e = JumpTables.size(); i != e; ++i) {
+ OS << " <jt #" << i << "> has " << JumpTables[i].MBBs.size()
+ << " entries\n";
+ }
+}
+
+void MachineJumpTableInfo::dump() const { print(*cerr.stream()); }
+
+
+//===----------------------------------------------------------------------===//
+// MachineConstantPool implementation
+//===----------------------------------------------------------------------===//
+
+const Type *MachineConstantPoolEntry::getType() const {
+ if (isMachineConstantPoolEntry())
+ return Val.MachineCPVal->getType();
+ return Val.ConstVal->getType();
+}
+
+MachineConstantPool::~MachineConstantPool() {
+ for (unsigned i = 0, e = Constants.size(); i != e; ++i)
+ if (Constants[i].isMachineConstantPoolEntry())
+ delete Constants[i].Val.MachineCPVal;
+}
+
+/// getConstantPoolIndex - Create a new entry in the constant pool or return
+/// an existing one. User must specify an alignment in bytes for the object.
+///
+unsigned MachineConstantPool::getConstantPoolIndex(Constant *C,
+ unsigned Alignment) {
+ assert(Alignment && "Alignment must be specified!");
+ if (Alignment > PoolAlignment) PoolAlignment = Alignment;
+
+ // Check to see if we already have this constant.
+ //
+ // FIXME, this could be made much more efficient for large constant pools.
+ unsigned AlignMask = (1 << Alignment)-1;
+ for (unsigned i = 0, e = Constants.size(); i != e; ++i)
+ if (Constants[i].Val.ConstVal == C && (Constants[i].Offset & AlignMask)== 0)
+ return i;
+
+ unsigned Offset = 0;
+ if (!Constants.empty()) {
+ Offset = Constants.back().getOffset();
+ Offset += TD->getTypeSize(Constants.back().getType());
+ Offset = (Offset+AlignMask)&~AlignMask;
+ }
+
+ Constants.push_back(MachineConstantPoolEntry(C, Offset));
+ return Constants.size()-1;
+}
+
+unsigned MachineConstantPool::getConstantPoolIndex(MachineConstantPoolValue *V,
+ unsigned Alignment) {
+ assert(Alignment && "Alignment must be specified!");
+ if (Alignment > PoolAlignment) PoolAlignment = Alignment;
+
+ // Check to see if we already have this constant.
+ //
+ // FIXME, this could be made much more efficient for large constant pools.
+ unsigned AlignMask = (1 << Alignment)-1;
+ int Idx = V->getExistingMachineCPValue(this, Alignment);
+ if (Idx != -1)
+ return (unsigned)Idx;
+
+ unsigned Offset = 0;
+ if (!Constants.empty()) {
+ Offset = Constants.back().getOffset();
+ Offset += TD->getTypeSize(Constants.back().getType());
+ Offset = (Offset+AlignMask)&~AlignMask;
+ }
+
+ Constants.push_back(MachineConstantPoolEntry(V, Offset));
+ return Constants.size()-1;
+}
+
+
+void MachineConstantPool::print(std::ostream &OS) const {
+ for (unsigned i = 0, e = Constants.size(); i != e; ++i) {
+ OS << " <cp #" << i << "> is";
+ if (Constants[i].isMachineConstantPoolEntry())
+ Constants[i].Val.MachineCPVal->print(OS);
+ else
+ OS << *(Value*)Constants[i].Val.ConstVal;
+ OS << " , offset=" << Constants[i].getOffset();
+ OS << "\n";
+ }
+}
+
+void MachineConstantPool::dump() const { print(*cerr.stream()); }
diff --git a/lib/CodeGen/MachineInstr.cpp b/lib/CodeGen/MachineInstr.cpp
new file mode 100644
index 0000000..c5e2ba8
--- /dev/null
+++ b/lib/CodeGen/MachineInstr.cpp
@@ -0,0 +1,426 @@
+//===-- MachineInstr.cpp --------------------------------------------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file was developed by the LLVM research group and is distributed under
+// the University of Illinois Open Source License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// Methods common to all machine instructions.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/CodeGen/MachineInstr.h"
+#include "llvm/CodeGen/MachineFunction.h"
+#include "llvm/Target/TargetMachine.h"
+#include "llvm/Target/TargetInstrInfo.h"
+#include "llvm/Target/MRegisterInfo.h"
+#include "llvm/Support/LeakDetector.h"
+#include "llvm/Support/Streams.h"
+#include <ostream>
+using namespace llvm;
+
+/// MachineInstr ctor - This constructor creates a dummy MachineInstr with
+/// TID NULL and no operands.
+MachineInstr::MachineInstr()
+ : TID(0), NumImplicitOps(0), parent(0) {
+ // Make sure that we get added to a machine basicblock
+ LeakDetector::addGarbageObject(this);
+}
+
+void MachineInstr::addImplicitDefUseOperands() {
+ if (TID->ImplicitDefs)
+ for (const unsigned *ImpDefs = TID->ImplicitDefs; *ImpDefs; ++ImpDefs) {
+ MachineOperand Op;
+ Op.opType = MachineOperand::MO_Register;
+ Op.IsDef = true;
+ Op.IsImp = true;
+ Op.IsKill = false;
+ Op.IsDead = false;
+ Op.contents.RegNo = *ImpDefs;
+ Op.auxInfo.subReg = 0;
+ Operands.push_back(Op);
+ }
+ if (TID->ImplicitUses)
+ for (const unsigned *ImpUses = TID->ImplicitUses; *ImpUses; ++ImpUses) {
+ MachineOperand Op;
+ Op.opType = MachineOperand::MO_Register;
+ Op.IsDef = false;
+ Op.IsImp = true;
+ Op.IsKill = false;
+ Op.IsDead = false;
+ Op.contents.RegNo = *ImpUses;
+ Op.auxInfo.subReg = 0;
+ Operands.push_back(Op);
+ }
+}
+
+/// MachineInstr ctor - This constructor create a MachineInstr and add the
+/// implicit operands. It reserves space for number of operands specified by
+/// TargetInstrDescriptor or the numOperands if it is not zero. (for
+/// instructions with variable number of operands).
+MachineInstr::MachineInstr(const TargetInstrDescriptor &tid)
+ : TID(&tid), NumImplicitOps(0), parent(0) {
+ if (TID->ImplicitDefs)
+ for (const unsigned *ImpDefs = TID->ImplicitDefs; *ImpDefs; ++ImpDefs)
+ NumImplicitOps++;
+ if (TID->ImplicitUses)
+ for (const unsigned *ImpUses = TID->ImplicitUses; *ImpUses; ++ImpUses)
+ NumImplicitOps++;
+ Operands.reserve(NumImplicitOps + TID->numOperands);
+ addImplicitDefUseOperands();
+ // Make sure that we get added to a machine basicblock
+ LeakDetector::addGarbageObject(this);
+}
+
+/// MachineInstr ctor - Work exactly the same as the ctor above, except that the
+/// MachineInstr is created and added to the end of the specified basic block.
+///
+MachineInstr::MachineInstr(MachineBasicBlock *MBB,
+ const TargetInstrDescriptor &tid)
+ : TID(&tid), NumImplicitOps(0), parent(0) {
+ assert(MBB && "Cannot use inserting ctor with null basic block!");
+ if (TID->ImplicitDefs)
+ for (const unsigned *ImpDefs = TID->ImplicitDefs; *ImpDefs; ++ImpDefs)
+ NumImplicitOps++;
+ if (TID->ImplicitUses)
+ for (const unsigned *ImpUses = TID->ImplicitUses; *ImpUses; ++ImpUses)
+ NumImplicitOps++;
+ Operands.reserve(NumImplicitOps + TID->numOperands);
+ addImplicitDefUseOperands();
+ // Make sure that we get added to a machine basicblock
+ LeakDetector::addGarbageObject(this);
+ MBB->push_back(this); // Add instruction to end of basic block!
+}
+
+/// MachineInstr ctor - Copies MachineInstr arg exactly
+///
+MachineInstr::MachineInstr(const MachineInstr &MI) {
+ TID = MI.getInstrDescriptor();
+ NumImplicitOps = MI.NumImplicitOps;
+ Operands.reserve(MI.getNumOperands());
+
+ // Add operands
+ for (unsigned i = 0; i != MI.getNumOperands(); ++i)
+ Operands.push_back(MI.getOperand(i));
+
+ // Set parent, next, and prev to null
+ parent = 0;
+ prev = 0;
+ next = 0;
+}
+
+
+MachineInstr::~MachineInstr() {
+ LeakDetector::removeGarbageObject(this);
+}
+
+/// getOpcode - Returns the opcode of this MachineInstr.
+///
+const int MachineInstr::getOpcode() const {
+ return TID->Opcode;
+}
+
+/// removeFromParent - This method unlinks 'this' from the containing basic
+/// block, and returns it, but does not delete it.
+MachineInstr *MachineInstr::removeFromParent() {
+ assert(getParent() && "Not embedded in a basic block!");
+ getParent()->remove(this);
+ return this;
+}
+
+
+/// OperandComplete - Return true if it's illegal to add a new operand
+///
+bool MachineInstr::OperandsComplete() const {
+ unsigned short NumOperands = TID->numOperands;
+ if ((TID->Flags & M_VARIABLE_OPS) == 0 &&
+ getNumOperands()-NumImplicitOps >= NumOperands)
+ return true; // Broken: we have all the operands of this instruction!
+ return false;
+}
+
+/// getNumExplicitOperands - Returns the number of non-implicit operands.
+///
+unsigned MachineInstr::getNumExplicitOperands() const {
+ unsigned NumOperands = TID->numOperands;
+ if ((TID->Flags & M_VARIABLE_OPS) == 0)
+ return NumOperands;
+
+ for (unsigned e = getNumOperands(); NumOperands != e; ++NumOperands) {
+ const MachineOperand &MO = getOperand(NumOperands);
+ if (!MO.isRegister() || !MO.isImplicit())
+ NumOperands++;
+ }
+ return NumOperands;
+}
+
+/// isIdenticalTo - Return true if this operand is identical to the specified
+/// operand.
+bool MachineOperand::isIdenticalTo(const MachineOperand &Other) const {
+ if (getType() != Other.getType()) return false;
+
+ switch (getType()) {
+ default: assert(0 && "Unrecognized operand type");
+ case MachineOperand::MO_Register:
+ return getReg() == Other.getReg() && isDef() == Other.isDef();
+ case MachineOperand::MO_Immediate:
+ return getImm() == Other.getImm();
+ case MachineOperand::MO_MachineBasicBlock:
+ return getMBB() == Other.getMBB();
+ case MachineOperand::MO_FrameIndex:
+ return getFrameIndex() == Other.getFrameIndex();
+ case MachineOperand::MO_ConstantPoolIndex:
+ return getConstantPoolIndex() == Other.getConstantPoolIndex() &&
+ getOffset() == Other.getOffset();
+ case MachineOperand::MO_JumpTableIndex:
+ return getJumpTableIndex() == Other.getJumpTableIndex();
+ case MachineOperand::MO_GlobalAddress:
+ return getGlobal() == Other.getGlobal() && getOffset() == Other.getOffset();
+ case MachineOperand::MO_ExternalSymbol:
+ return !strcmp(getSymbolName(), Other.getSymbolName()) &&
+ getOffset() == Other.getOffset();
+ }
+}
+
+/// findRegisterUseOperandIdx() - Returns the MachineOperand that is a use of
+/// the specific register or -1 if it is not found. It further tightening
+/// the search criteria to a use that kills the register if isKill is true.
+int MachineInstr::findRegisterUseOperandIdx(unsigned Reg, bool isKill) const {
+ for (unsigned i = 0, e = getNumOperands(); i != e; ++i) {
+ const MachineOperand &MO = getOperand(i);
+ if (MO.isReg() && MO.isUse() && MO.getReg() == Reg)
+ if (!isKill || MO.isKill())
+ return i;
+ }
+ return -1;
+}
+
+/// findRegisterDefOperand() - Returns the MachineOperand that is a def of
+/// the specific register or NULL if it is not found.
+MachineOperand *MachineInstr::findRegisterDefOperand(unsigned Reg) {
+ for (unsigned i = 0, e = getNumOperands(); i != e; ++i) {
+ MachineOperand &MO = getOperand(i);
+ if (MO.isReg() && MO.isDef() && MO.getReg() == Reg)
+ return &MO;
+ }
+ return NULL;
+}
+
+/// findFirstPredOperandIdx() - Find the index of the first operand in the
+/// operand list that is used to represent the predicate. It returns -1 if
+/// none is found.
+int MachineInstr::findFirstPredOperandIdx() const {
+ const TargetInstrDescriptor *TID = getInstrDescriptor();
+ if (TID->Flags & M_PREDICABLE) {
+ for (unsigned i = 0, e = getNumOperands(); i != e; ++i)
+ if ((TID->OpInfo[i].Flags & M_PREDICATE_OPERAND))
+ return i;
+ }
+
+ return -1;
+}
+
+/// copyKillDeadInfo - Copies kill / dead operand properties from MI.
+///
+void MachineInstr::copyKillDeadInfo(const MachineInstr *MI) {
+ for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
+ const MachineOperand &MO = MI->getOperand(i);
+ if (!MO.isReg() || (!MO.isKill() && !MO.isDead()))
+ continue;
+ for (unsigned j = 0, ee = getNumOperands(); j != ee; ++j) {
+ MachineOperand &MOp = getOperand(j);
+ if (!MOp.isIdenticalTo(MO))
+ continue;
+ if (MO.isKill())
+ MOp.setIsKill();
+ else
+ MOp.setIsDead();
+ break;
+ }
+ }
+}
+
+/// copyPredicates - Copies predicate operand(s) from MI.
+void MachineInstr::copyPredicates(const MachineInstr *MI) {
+ const TargetInstrDescriptor *TID = MI->getInstrDescriptor();
+ if (TID->Flags & M_PREDICABLE) {
+ for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
+ if ((TID->OpInfo[i].Flags & M_PREDICATE_OPERAND)) {
+ const MachineOperand &MO = MI->getOperand(i);
+ // Predicated operands must be last operands.
+ if (MO.isReg())
+ addRegOperand(MO.getReg(), false);
+ else {
+ addImmOperand(MO.getImm());
+ }
+ }
+ }
+ }
+}
+
+void MachineInstr::dump() const {
+ cerr << " " << *this;
+}
+
+static inline void OutputReg(std::ostream &os, unsigned RegNo,
+ const MRegisterInfo *MRI = 0) {
+ if (!RegNo || MRegisterInfo::isPhysicalRegister(RegNo)) {
+ if (MRI)
+ os << "%" << MRI->get(RegNo).Name;
+ else
+ os << "%mreg(" << RegNo << ")";
+ } else
+ os << "%reg" << RegNo;
+}
+
+static void print(const MachineOperand &MO, std::ostream &OS,
+ const TargetMachine *TM) {
+ const MRegisterInfo *MRI = 0;
+
+ if (TM) MRI = TM->getRegisterInfo();
+
+ switch (MO.getType()) {
+ case MachineOperand::MO_Register:
+ OutputReg(OS, MO.getReg(), MRI);
+ break;
+ case MachineOperand::MO_Immediate:
+ OS << MO.getImmedValue();
+ break;
+ case MachineOperand::MO_MachineBasicBlock:
+ OS << "mbb<"
+ << ((Value*)MO.getMachineBasicBlock()->getBasicBlock())->getName()
+ << "," << (void*)MO.getMachineBasicBlock() << ">";
+ break;
+ case MachineOperand::MO_FrameIndex:
+ OS << "<fi#" << MO.getFrameIndex() << ">";
+ break;
+ case MachineOperand::MO_ConstantPoolIndex:
+ OS << "<cp#" << MO.getConstantPoolIndex() << ">";
+ break;
+ case MachineOperand::MO_JumpTableIndex:
+ OS << "<jt#" << MO.getJumpTableIndex() << ">";
+ break;
+ case MachineOperand::MO_GlobalAddress:
+ OS << "<ga:" << ((Value*)MO.getGlobal())->getName();
+ if (MO.getOffset()) OS << "+" << MO.getOffset();
+ OS << ">";
+ break;
+ case MachineOperand::MO_ExternalSymbol:
+ OS << "<es:" << MO.getSymbolName();
+ if (MO.getOffset()) OS << "+" << MO.getOffset();
+ OS << ">";
+ break;
+ default:
+ assert(0 && "Unrecognized operand type");
+ }
+}
+
+void MachineInstr::print(std::ostream &OS, const TargetMachine *TM) const {
+ unsigned StartOp = 0;
+
+ // Specialize printing if op#0 is definition
+ if (getNumOperands() && getOperand(0).isReg() && getOperand(0).isDef()) {
+ ::print(getOperand(0), OS, TM);
+ if (getOperand(0).isDead())
+ OS << "<dead>";
+ OS << " = ";
+ ++StartOp; // Don't print this operand again!
+ }
+
+ if (TID)
+ OS << TID->Name;
+
+ for (unsigned i = StartOp, e = getNumOperands(); i != e; ++i) {
+ const MachineOperand& mop = getOperand(i);
+ if (i != StartOp)
+ OS << ",";
+ OS << " ";
+ ::print(mop, OS, TM);
+
+ if (mop.isReg()) {
+ if (mop.isDef() || mop.isKill() || mop.isDead() || mop.isImplicit()) {
+ OS << "<";
+ bool NeedComma = false;
+ if (mop.isImplicit()) {
+ OS << (mop.isDef() ? "imp-def" : "imp-use");
+ NeedComma = true;
+ } else if (mop.isDef()) {
+ OS << "def";
+ NeedComma = true;
+ }
+ if (mop.isKill() || mop.isDead()) {
+ if (NeedComma)
+ OS << ",";
+ if (mop.isKill())
+ OS << "kill";
+ if (mop.isDead())
+ OS << "dead";
+ }
+ OS << ">";
+ }
+ }
+ }
+
+ OS << "\n";
+}
+
+void MachineInstr::print(std::ostream &os) const {
+ // If the instruction is embedded into a basic block, we can find the target
+ // info for the instruction.
+ if (const MachineBasicBlock *MBB = getParent()) {
+ const MachineFunction *MF = MBB->getParent();
+ if (MF)
+ print(os, &MF->getTarget());
+ else
+ print(os, 0);
+ }
+
+ // Otherwise, print it out in the "raw" format without symbolic register names
+ // and such.
+ os << getInstrDescriptor()->Name;
+
+ for (unsigned i = 0, N = getNumOperands(); i < N; i++) {
+ os << "\t" << getOperand(i);
+ if (getOperand(i).isReg() && getOperand(i).isDef())
+ os << "<d>";
+ }
+
+ os << "\n";
+}
+
+void MachineOperand::print(std::ostream &OS) const {
+ switch (getType()) {
+ case MO_Register:
+ OutputReg(OS, getReg());
+ break;
+ case MO_Immediate:
+ OS << (long)getImmedValue();
+ break;
+ case MO_MachineBasicBlock:
+ OS << "<mbb:"
+ << ((Value*)getMachineBasicBlock()->getBasicBlock())->getName()
+ << "@" << (void*)getMachineBasicBlock() << ">";
+ break;
+ case MO_FrameIndex:
+ OS << "<fi#" << getFrameIndex() << ">";
+ break;
+ case MO_ConstantPoolIndex:
+ OS << "<cp#" << getConstantPoolIndex() << ">";
+ break;
+ case MO_JumpTableIndex:
+ OS << "<jt#" << getJumpTableIndex() << ">";
+ break;
+ case MO_GlobalAddress:
+ OS << "<ga:" << ((Value*)getGlobal())->getName() << ">";
+ break;
+ case MO_ExternalSymbol:
+ OS << "<es:" << getSymbolName() << ">";
+ break;
+ default:
+ assert(0 && "Unrecognized operand type");
+ break;
+ }
+}
+
diff --git a/lib/CodeGen/MachineModuleInfo.cpp b/lib/CodeGen/MachineModuleInfo.cpp
new file mode 100644
index 0000000..77fb643
--- /dev/null
+++ b/lib/CodeGen/MachineModuleInfo.cpp
@@ -0,0 +1,1905 @@
+//===-- llvm/CodeGen/MachineModuleInfo.cpp ----------------------*- C++ -*-===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file was developed by James M. Laskey and is distributed under
+// the University of Illinois Open Source License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/CodeGen/MachineModuleInfo.h"
+
+#include "llvm/Constants.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/MachineFunction.h"
+#include "llvm/CodeGen/MachineLocation.h"
+#include "llvm/Target/TargetInstrInfo.h"
+#include "llvm/Target/TargetMachine.h"
+#include "llvm/Target/TargetOptions.h"
+#include "llvm/DerivedTypes.h"
+#include "llvm/GlobalVariable.h"
+#include "llvm/Intrinsics.h"
+#include "llvm/Instructions.h"
+#include "llvm/Module.h"
+#include "llvm/Support/Dwarf.h"
+#include "llvm/Support/Streams.h"
+using namespace llvm;
+using namespace llvm::dwarf;
+
+// Handle the Pass registration stuff necessary to use TargetData's.
+namespace {
+ RegisterPass<MachineModuleInfo> X("machinemoduleinfo", "Module Information");
+}
+char MachineModuleInfo::ID = 0;
+
+//===----------------------------------------------------------------------===//
+
+/// getGlobalVariablesUsing - Return all of the GlobalVariables which have the
+/// specified value in their initializer somewhere.
+static void
+getGlobalVariablesUsing(Value *V, std::vector<GlobalVariable*> &Result) {
+ // Scan though value users.
+ for (Value::use_iterator I = V->use_begin(), E = V->use_end(); I != E; ++I) {
+ if (GlobalVariable *GV = dyn_cast<GlobalVariable>(*I)) {
+ // If the user is a GlobalVariable then add to result.
+ Result.push_back(GV);
+ } else if (Constant *C = dyn_cast<Constant>(*I)) {
+ // If the user is a constant variable then scan its users
+ getGlobalVariablesUsing(C, Result);
+ }
+ }
+}
+
+/// getGlobalVariablesUsing - Return all of the GlobalVariables that use the
+/// named GlobalVariable.
+static std::vector<GlobalVariable*>
+getGlobalVariablesUsing(Module &M, const std::string &RootName) {
+ std::vector<GlobalVariable*> Result; // GlobalVariables matching criteria.
+
+ std::vector<const Type*> FieldTypes;
+ FieldTypes.push_back(Type::Int32Ty);
+ FieldTypes.push_back(Type::Int32Ty);
+
+ // Get the GlobalVariable root.
+ GlobalVariable *UseRoot = M.getGlobalVariable(RootName,
+ StructType::get(FieldTypes));
+
+ // If present and linkonce then scan for users.
+ if (UseRoot && UseRoot->hasLinkOnceLinkage()) {
+ getGlobalVariablesUsing(UseRoot, Result);
+ }
+
+ return Result;
+}
+
+/// isStringValue - Return true if the given value can be coerced to a string.
+///
+static bool isStringValue(Value *V) {
+ if (GlobalVariable *GV = dyn_cast<GlobalVariable>(V)) {
+ if (GV->hasInitializer() && isa<ConstantArray>(GV->getInitializer())) {
+ ConstantArray *Init = cast<ConstantArray>(GV->getInitializer());
+ return Init->isString();
+ }
+ } else if (Constant *C = dyn_cast<Constant>(V)) {
+ if (GlobalValue *GV = dyn_cast<GlobalValue>(C))
+ return isStringValue(GV);
+ else if (ConstantExpr *CE = dyn_cast<ConstantExpr>(C)) {
+ if (CE->getOpcode() == Instruction::GetElementPtr) {
+ if (CE->getNumOperands() == 3 &&
+ cast<Constant>(CE->getOperand(1))->isNullValue() &&
+ isa<ConstantInt>(CE->getOperand(2))) {
+ return isStringValue(CE->getOperand(0));
+ }
+ }
+ }
+ }
+ return false;
+}
+
+/// getGlobalVariable - Return either a direct or cast Global value.
+///
+static GlobalVariable *getGlobalVariable(Value *V) {
+ if (GlobalVariable *GV = dyn_cast<GlobalVariable>(V)) {
+ return GV;
+ } else if (ConstantExpr *CE = dyn_cast<ConstantExpr>(V)) {
+ if (CE->getOpcode() == Instruction::BitCast) {
+ return dyn_cast<GlobalVariable>(CE->getOperand(0));
+ }
+ }
+ return NULL;
+}
+
+/// isGlobalVariable - Return true if the given value can be coerced to a
+/// GlobalVariable.
+static bool isGlobalVariable(Value *V) {
+ if (isa<GlobalVariable>(V) || isa<ConstantPointerNull>(V)) {
+ return true;
+ } else if (ConstantExpr *CE = dyn_cast<ConstantExpr>(V)) {
+ if (CE->getOpcode() == Instruction::BitCast) {
+ return isa<GlobalVariable>(CE->getOperand(0));
+ }
+ }
+ return false;
+}
+
+/// getUIntOperand - Return ith operand if it is an unsigned integer.
+///
+static ConstantInt *getUIntOperand(GlobalVariable *GV, unsigned i) {
+ // Make sure the GlobalVariable has an initializer.
+ if (!GV->hasInitializer()) return NULL;
+
+ // Get the initializer constant.
+ ConstantStruct *CI = dyn_cast<ConstantStruct>(GV->getInitializer());
+ if (!CI) return NULL;
+
+ // Check if there is at least i + 1 operands.
+ unsigned N = CI->getNumOperands();
+ if (i >= N) return NULL;
+
+ // Check constant.
+ return dyn_cast<ConstantInt>(CI->getOperand(i));
+}
+
+//===----------------------------------------------------------------------===//
+
+/// ApplyToFields - Target the visitor to each field of the debug information
+/// descriptor.
+void DIVisitor::ApplyToFields(DebugInfoDesc *DD) {
+ DD->ApplyToFields(this);
+}
+
+//===----------------------------------------------------------------------===//
+/// DICountVisitor - This DIVisitor counts all the fields in the supplied debug
+/// the supplied DebugInfoDesc.
+class DICountVisitor : public DIVisitor {
+private:
+ unsigned Count; // Running count of fields.
+
+public:
+ DICountVisitor() : DIVisitor(), Count(0) {}
+
+ // Accessors.
+ unsigned getCount() const { return Count; }
+
+ /// Apply - Count each of the fields.
+ ///
+ virtual void Apply(int &Field) { ++Count; }
+ virtual void Apply(unsigned &Field) { ++Count; }
+ virtual void Apply(int64_t &Field) { ++Count; }
+ virtual void Apply(uint64_t &Field) { ++Count; }
+ virtual void Apply(bool &Field) { ++Count; }
+ virtual void Apply(std::string &Field) { ++Count; }
+ virtual void Apply(DebugInfoDesc *&Field) { ++Count; }
+ virtual void Apply(GlobalVariable *&Field) { ++Count; }
+ virtual void Apply(std::vector<DebugInfoDesc *> &Field) {
+ ++Count;
+ }
+};
+
+//===----------------------------------------------------------------------===//
+/// DIDeserializeVisitor - This DIVisitor deserializes all the fields in the
+/// supplied DebugInfoDesc.
+class DIDeserializeVisitor : public DIVisitor {
+private:
+ DIDeserializer &DR; // Active deserializer.
+ unsigned I; // Current operand index.
+ ConstantStruct *CI; // GlobalVariable constant initializer.
+
+public:
+ DIDeserializeVisitor(DIDeserializer &D, GlobalVariable *GV)
+ : DIVisitor()
+ , DR(D)
+ , I(0)
+ , CI(cast<ConstantStruct>(GV->getInitializer()))
+ {}
+
+ /// Apply - Set the value of each of the fields.
+ ///
+ virtual void Apply(int &Field) {
+ Constant *C = CI->getOperand(I++);
+ Field = cast<ConstantInt>(C)->getSExtValue();
+ }
+ virtual void Apply(unsigned &Field) {
+ Constant *C = CI->getOperand(I++);
+ Field = cast<ConstantInt>(C)->getZExtValue();
+ }
+ virtual void Apply(int64_t &Field) {
+ Constant *C = CI->getOperand(I++);
+ Field = cast<ConstantInt>(C)->getSExtValue();
+ }
+ virtual void Apply(uint64_t &Field) {
+ Constant *C = CI->getOperand(I++);
+ Field = cast<ConstantInt>(C)->getZExtValue();
+ }
+ virtual void Apply(bool &Field) {
+ Constant *C = CI->getOperand(I++);
+ Field = cast<ConstantInt>(C)->getZExtValue();
+ }
+ virtual void Apply(std::string &Field) {
+ Constant *C = CI->getOperand(I++);
+ Field = C->getStringValue();
+ }
+ virtual void Apply(DebugInfoDesc *&Field) {
+ Constant *C = CI->getOperand(I++);
+ Field = DR.Deserialize(C);
+ }
+ virtual void Apply(GlobalVariable *&Field) {
+ Constant *C = CI->getOperand(I++);
+ Field = getGlobalVariable(C);
+ }
+ virtual void Apply(std::vector<DebugInfoDesc *> &Field) {
+ Field.resize(0);
+ Constant *C = CI->getOperand(I++);
+ GlobalVariable *GV = getGlobalVariable(C);
+ if (GV->hasInitializer()) {
+ if (ConstantArray *CA = dyn_cast<ConstantArray>(GV->getInitializer())) {
+ for (unsigned i = 0, N = CA->getNumOperands(); i < N; ++i) {
+ GlobalVariable *GVE = getGlobalVariable(CA->getOperand(i));
+ DebugInfoDesc *DE = DR.Deserialize(GVE);
+ Field.push_back(DE);
+ }
+ } else if (GV->getInitializer()->isNullValue()) {
+ if (const ArrayType *T =
+ dyn_cast<ArrayType>(GV->getType()->getElementType())) {
+ Field.resize(T->getNumElements());
+ }
+ }
+ }
+ }
+};
+
+//===----------------------------------------------------------------------===//
+/// DISerializeVisitor - This DIVisitor serializes all the fields in
+/// the supplied DebugInfoDesc.
+class DISerializeVisitor : public DIVisitor {
+private:
+ DISerializer &SR; // Active serializer.
+ std::vector<Constant*> &Elements; // Element accumulator.
+
+public:
+ DISerializeVisitor(DISerializer &S, std::vector<Constant*> &E)
+ : DIVisitor()
+ , SR(S)
+ , Elements(E)
+ {}
+
+ /// Apply - Set the value of each of the fields.
+ ///
+ virtual void Apply(int &Field) {
+ Elements.push_back(ConstantInt::get(Type::Int32Ty, int32_t(Field)));
+ }
+ virtual void Apply(unsigned &Field) {
+ Elements.push_back(ConstantInt::get(Type::Int32Ty, uint32_t(Field)));
+ }
+ virtual void Apply(int64_t &Field) {
+ Elements.push_back(ConstantInt::get(Type::Int64Ty, int64_t(Field)));
+ }
+ virtual void Apply(uint64_t &Field) {
+ Elements.push_back(ConstantInt::get(Type::Int64Ty, uint64_t(Field)));
+ }
+ virtual void Apply(bool &Field) {
+ Elements.push_back(ConstantInt::get(Type::Int1Ty, Field));
+ }
+ virtual void Apply(std::string &Field) {
+ Elements.push_back(SR.getString(Field));
+ }
+ virtual void Apply(DebugInfoDesc *&Field) {
+ GlobalVariable *GV = NULL;
+
+ // If non-NULL then convert to global.
+ if (Field) GV = SR.Serialize(Field);
+
+ // FIXME - At some point should use specific type.
+ const PointerType *EmptyTy = SR.getEmptyStructPtrType();
+
+ if (GV) {
+ // Set to pointer to global.
+ Elements.push_back(ConstantExpr::getBitCast(GV, EmptyTy));
+ } else {
+ // Use NULL.
+ Elements.push_back(ConstantPointerNull::get(EmptyTy));
+ }
+ }
+ virtual void Apply(GlobalVariable *&Field) {
+ const PointerType *EmptyTy = SR.getEmptyStructPtrType();
+ if (Field) {
+ Elements.push_back(ConstantExpr::getBitCast(Field, EmptyTy));
+ } else {
+ Elements.push_back(ConstantPointerNull::get(EmptyTy));
+ }
+ }
+ virtual void Apply(std::vector<DebugInfoDesc *> &Field) {
+ const PointerType *EmptyTy = SR.getEmptyStructPtrType();
+ unsigned N = Field.size();
+ ArrayType *AT = ArrayType::get(EmptyTy, N);
+ std::vector<Constant *> ArrayElements;
+
+ for (unsigned i = 0, N = Field.size(); i < N; ++i) {
+ if (DebugInfoDesc *Element = Field[i]) {
+ GlobalVariable *GVE = SR.Serialize(Element);
+ Constant *CE = ConstantExpr::getBitCast(GVE, EmptyTy);
+ ArrayElements.push_back(cast<Constant>(CE));
+ } else {
+ ArrayElements.push_back(ConstantPointerNull::get(EmptyTy));
+ }
+ }
+
+ Constant *CA = ConstantArray::get(AT, ArrayElements);
+ GlobalVariable *CAGV = new GlobalVariable(AT, true,
+ GlobalValue::InternalLinkage,
+ CA, "llvm.dbg.array",
+ SR.getModule());
+ CAGV->setSection("llvm.metadata");
+ Constant *CAE = ConstantExpr::getBitCast(CAGV, EmptyTy);
+ Elements.push_back(CAE);
+ }
+};
+
+//===----------------------------------------------------------------------===//
+/// DIGetTypesVisitor - This DIVisitor gathers all the field types in
+/// the supplied DebugInfoDesc.
+class DIGetTypesVisitor : public DIVisitor {
+private:
+ DISerializer &SR; // Active serializer.
+ std::vector<const Type*> &Fields; // Type accumulator.
+
+public:
+ DIGetTypesVisitor(DISerializer &S, std::vector<const Type*> &F)
+ : DIVisitor()
+ , SR(S)
+ , Fields(F)
+ {}
+
+ /// Apply - Set the value of each of the fields.
+ ///
+ virtual void Apply(int &Field) {
+ Fields.push_back(Type::Int32Ty);
+ }
+ virtual void Apply(unsigned &Field) {
+ Fields.push_back(Type::Int32Ty);
+ }
+ virtual void Apply(int64_t &Field) {
+ Fields.push_back(Type::Int64Ty);
+ }
+ virtual void Apply(uint64_t &Field) {
+ Fields.push_back(Type::Int64Ty);
+ }
+ virtual void Apply(bool &Field) {
+ Fields.push_back(Type::Int1Ty);
+ }
+ virtual void Apply(std::string &Field) {
+ Fields.push_back(SR.getStrPtrType());
+ }
+ virtual void Apply(DebugInfoDesc *&Field) {
+ // FIXME - At some point should use specific type.
+ const PointerType *EmptyTy = SR.getEmptyStructPtrType();
+ Fields.push_back(EmptyTy);
+ }
+ virtual void Apply(GlobalVariable *&Field) {
+ const PointerType *EmptyTy = SR.getEmptyStructPtrType();
+ Fields.push_back(EmptyTy);
+ }
+ virtual void Apply(std::vector<DebugInfoDesc *> &Field) {
+ const PointerType *EmptyTy = SR.getEmptyStructPtrType();
+ Fields.push_back(EmptyTy);
+ }
+};
+
+//===----------------------------------------------------------------------===//
+/// DIVerifyVisitor - This DIVisitor verifies all the field types against
+/// a constant initializer.
+class DIVerifyVisitor : public DIVisitor {
+private:
+ DIVerifier &VR; // Active verifier.
+ bool IsValid; // Validity status.
+ unsigned I; // Current operand index.
+ ConstantStruct *CI; // GlobalVariable constant initializer.
+
+public:
+ DIVerifyVisitor(DIVerifier &V, GlobalVariable *GV)
+ : DIVisitor()
+ , VR(V)
+ , IsValid(true)
+ , I(0)
+ , CI(cast<ConstantStruct>(GV->getInitializer()))
+ {
+ }
+
+ // Accessors.
+ bool isValid() const { return IsValid; }
+
+ /// Apply - Set the value of each of the fields.
+ ///
+ virtual void Apply(int &Field) {
+ Constant *C = CI->getOperand(I++);
+ IsValid = IsValid && isa<ConstantInt>(C);
+ }
+ virtual void Apply(unsigned &Field) {
+ Constant *C = CI->getOperand(I++);
+ IsValid = IsValid && isa<ConstantInt>(C);
+ }
+ virtual void Apply(int64_t &Field) {
+ Constant *C = CI->getOperand(I++);
+ IsValid = IsValid && isa<ConstantInt>(C);
+ }
+ virtual void Apply(uint64_t &Field) {
+ Constant *C = CI->getOperand(I++);
+ IsValid = IsValid && isa<ConstantInt>(C);
+ }
+ virtual void Apply(bool &Field) {
+ Constant *C = CI->getOperand(I++);
+ IsValid = IsValid && isa<ConstantInt>(C) && C->getType() == Type::Int1Ty;
+ }
+ virtual void Apply(std::string &Field) {
+ Constant *C = CI->getOperand(I++);
+ IsValid = IsValid &&
+ (!C || isStringValue(C) || C->isNullValue());
+ }
+ virtual void Apply(DebugInfoDesc *&Field) {
+ // FIXME - Prepare the correct descriptor.
+ Constant *C = CI->getOperand(I++);
+ IsValid = IsValid && isGlobalVariable(C);
+ }
+ virtual void Apply(GlobalVariable *&Field) {
+ Constant *C = CI->getOperand(I++);
+ IsValid = IsValid && isGlobalVariable(C);
+ }
+ virtual void Apply(std::vector<DebugInfoDesc *> &Field) {
+ Constant *C = CI->getOperand(I++);
+ IsValid = IsValid && isGlobalVariable(C);
+ if (!IsValid) return;
+
+ GlobalVariable *GV = getGlobalVariable(C);
+ IsValid = IsValid && GV && GV->hasInitializer();
+ if (!IsValid) return;
+
+ ConstantArray *CA = dyn_cast<ConstantArray>(GV->getInitializer());
+ IsValid = IsValid && CA;
+ if (!IsValid) return;
+
+ for (unsigned i = 0, N = CA->getNumOperands(); IsValid && i < N; ++i) {
+ IsValid = IsValid && isGlobalVariable(CA->getOperand(i));
+ if (!IsValid) return;
+
+ GlobalVariable *GVE = getGlobalVariable(CA->getOperand(i));
+ VR.Verify(GVE);
+ }
+ }
+};
+
+
+//===----------------------------------------------------------------------===//
+
+/// TagFromGlobal - Returns the tag number from a debug info descriptor
+/// GlobalVariable. Return DIIValid if operand is not an unsigned int.
+unsigned DebugInfoDesc::TagFromGlobal(GlobalVariable *GV) {
+ ConstantInt *C = getUIntOperand(GV, 0);
+ return C ? ((unsigned)C->getZExtValue() & ~LLVMDebugVersionMask) :
+ (unsigned)DW_TAG_invalid;
+}
+
+/// VersionFromGlobal - Returns the version number from a debug info
+/// descriptor GlobalVariable. Return DIIValid if operand is not an unsigned
+/// int.
+unsigned DebugInfoDesc::VersionFromGlobal(GlobalVariable *GV) {
+ ConstantInt *C = getUIntOperand(GV, 0);
+ return C ? ((unsigned)C->getZExtValue() & LLVMDebugVersionMask) :
+ (unsigned)DW_TAG_invalid;
+}
+
+/// DescFactory - Create an instance of debug info descriptor based on Tag.
+/// Return NULL if not a recognized Tag.
+DebugInfoDesc *DebugInfoDesc::DescFactory(unsigned Tag) {
+ switch (Tag) {
+ case DW_TAG_anchor: return new AnchorDesc();
+ case DW_TAG_compile_unit: return new CompileUnitDesc();
+ case DW_TAG_variable: return new GlobalVariableDesc();
+ case DW_TAG_subprogram: return new SubprogramDesc();
+ case DW_TAG_lexical_block: return new BlockDesc();
+ case DW_TAG_base_type: return new BasicTypeDesc();
+ case DW_TAG_typedef:
+ case DW_TAG_pointer_type:
+ case DW_TAG_reference_type:
+ case DW_TAG_const_type:
+ case DW_TAG_volatile_type:
+ case DW_TAG_restrict_type:
+ case DW_TAG_member:
+ case DW_TAG_inheritance: return new DerivedTypeDesc(Tag);
+ case DW_TAG_array_type:
+ case DW_TAG_structure_type:
+ case DW_TAG_union_type:
+ case DW_TAG_enumeration_type:
+ case DW_TAG_vector_type:
+ case DW_TAG_subroutine_type: return new CompositeTypeDesc(Tag);
+ case DW_TAG_subrange_type: return new SubrangeDesc();
+ case DW_TAG_enumerator: return new EnumeratorDesc();
+ case DW_TAG_return_variable:
+ case DW_TAG_arg_variable:
+ case DW_TAG_auto_variable: return new VariableDesc(Tag);
+ default: break;
+ }
+ return NULL;
+}
+
+/// getLinkage - get linkage appropriate for this type of descriptor.
+///
+GlobalValue::LinkageTypes DebugInfoDesc::getLinkage() const {
+ return GlobalValue::InternalLinkage;
+}
+
+/// ApplyToFields - Target the vistor to the fields of the descriptor.
+///
+void DebugInfoDesc::ApplyToFields(DIVisitor *Visitor) {
+ Visitor->Apply(Tag);
+}
+
+//===----------------------------------------------------------------------===//
+
+AnchorDesc::AnchorDesc()
+: DebugInfoDesc(DW_TAG_anchor)
+, AnchorTag(0)
+{}
+AnchorDesc::AnchorDesc(AnchoredDesc *D)
+: DebugInfoDesc(DW_TAG_anchor)
+, AnchorTag(D->getTag())
+{}
+
+// Implement isa/cast/dyncast.
+bool AnchorDesc::classof(const DebugInfoDesc *D) {
+ return D->getTag() == DW_TAG_anchor;
+}
+
+/// getLinkage - get linkage appropriate for this type of descriptor.
+///
+GlobalValue::LinkageTypes AnchorDesc::getLinkage() const {
+ return GlobalValue::LinkOnceLinkage;
+}
+
+/// ApplyToFields - Target the visitor to the fields of the TransUnitDesc.
+///
+void AnchorDesc::ApplyToFields(DIVisitor *Visitor) {
+ DebugInfoDesc::ApplyToFields(Visitor);
+
+ Visitor->Apply(AnchorTag);
+}
+
+/// getDescString - Return a string used to compose global names and labels. A
+/// A global variable name needs to be defined for each debug descriptor that is
+/// anchored. NOTE: that each global variable named here also needs to be added
+/// to the list of names left external in the internalizer.
+/// ExternalNames.insert("llvm.dbg.compile_units");
+/// ExternalNames.insert("llvm.dbg.global_variables");
+/// ExternalNames.insert("llvm.dbg.subprograms");
+const char *AnchorDesc::getDescString() const {
+ switch (AnchorTag) {
+ case DW_TAG_compile_unit: return CompileUnitDesc::AnchorString;
+ case DW_TAG_variable: return GlobalVariableDesc::AnchorString;
+ case DW_TAG_subprogram: return SubprogramDesc::AnchorString;
+ default: break;
+ }
+
+ assert(0 && "Tag does not have a case for anchor string");
+ return "";
+}
+
+/// getTypeString - Return a string used to label this descriptors type.
+///
+const char *AnchorDesc::getTypeString() const {
+ return "llvm.dbg.anchor.type";
+}
+
+#ifndef NDEBUG
+void AnchorDesc::dump() {
+ cerr << getDescString() << " "
+ << "Version(" << getVersion() << "), "
+ << "Tag(" << getTag() << "), "
+ << "AnchorTag(" << AnchorTag << ")\n";
+}
+#endif
+
+//===----------------------------------------------------------------------===//
+
+AnchoredDesc::AnchoredDesc(unsigned T)
+: DebugInfoDesc(T)
+, Anchor(NULL)
+{}
+
+/// ApplyToFields - Target the visitor to the fields of the AnchoredDesc.
+///
+void AnchoredDesc::ApplyToFields(DIVisitor *Visitor) {
+ DebugInfoDesc::ApplyToFields(Visitor);
+
+ Visitor->Apply(Anchor);
+}
+
+//===----------------------------------------------------------------------===//
+
+CompileUnitDesc::CompileUnitDesc()
+: AnchoredDesc(DW_TAG_compile_unit)
+, Language(0)
+, FileName("")
+, Directory("")
+, Producer("")
+{}
+
+// Implement isa/cast/dyncast.
+bool CompileUnitDesc::classof(const DebugInfoDesc *D) {
+ return D->getTag() == DW_TAG_compile_unit;
+}
+
+/// ApplyToFields - Target the visitor to the fields of the CompileUnitDesc.
+///
+void CompileUnitDesc::ApplyToFields(DIVisitor *Visitor) {
+ AnchoredDesc::ApplyToFields(Visitor);
+
+ // Handle cases out of sync with compiler.
+ if (getVersion() == 0) {
+ unsigned DebugVersion;
+ Visitor->Apply(DebugVersion);
+ }
+
+ Visitor->Apply(Language);
+ Visitor->Apply(FileName);
+ Visitor->Apply(Directory);
+ Visitor->Apply(Producer);
+}
+
+/// getDescString - Return a string used to compose global names and labels.
+///
+const char *CompileUnitDesc::getDescString() const {
+ return "llvm.dbg.compile_unit";
+}
+
+/// getTypeString - Return a string used to label this descriptors type.
+///
+const char *CompileUnitDesc::getTypeString() const {
+ return "llvm.dbg.compile_unit.type";
+}
+
+/// getAnchorString - Return a string used to label this descriptor's anchor.
+///
+const char *CompileUnitDesc::AnchorString = "llvm.dbg.compile_units";
+const char *CompileUnitDesc::getAnchorString() const {
+ return AnchorString;
+}
+
+#ifndef NDEBUG
+void CompileUnitDesc::dump() {
+ cerr << getDescString() << " "
+ << "Version(" << getVersion() << "), "
+ << "Tag(" << getTag() << "), "
+ << "Anchor(" << getAnchor() << "), "
+ << "Language(" << Language << "), "
+ << "FileName(\"" << FileName << "\"), "
+ << "Directory(\"" << Directory << "\"), "
+ << "Producer(\"" << Producer << "\")\n";
+}
+#endif
+
+//===----------------------------------------------------------------------===//
+
+TypeDesc::TypeDesc(unsigned T)
+: DebugInfoDesc(T)
+, Context(NULL)
+, Name("")
+, File(NULL)
+, Line(0)
+, Size(0)
+, Align(0)
+, Offset(0)
+, Flags(0)
+{}
+
+/// ApplyToFields - Target the visitor to the fields of the TypeDesc.
+///
+void TypeDesc::ApplyToFields(DIVisitor *Visitor) {
+ DebugInfoDesc::ApplyToFields(Visitor);
+
+ Visitor->Apply(Context);
+ Visitor->Apply(Name);
+ Visitor->Apply(File);
+ Visitor->Apply(Line);
+ Visitor->Apply(Size);
+ Visitor->Apply(Align);
+ Visitor->Apply(Offset);
+ if (getVersion() > LLVMDebugVersion4) Visitor->Apply(Flags);
+}
+
+/// getDescString - Return a string used to compose global names and labels.
+///
+const char *TypeDesc::getDescString() const {
+ return "llvm.dbg.type";
+}
+
+/// getTypeString - Return a string used to label this descriptor's type.
+///
+const char *TypeDesc::getTypeString() const {
+ return "llvm.dbg.type.type";
+}
+
+#ifndef NDEBUG
+void TypeDesc::dump() {
+ cerr << getDescString() << " "
+ << "Version(" << getVersion() << "), "
+ << "Tag(" << getTag() << "), "
+ << "Context(" << Context << "), "
+ << "Name(\"" << Name << "\"), "
+ << "File(" << File << "), "
+ << "Line(" << Line << "), "
+ << "Size(" << Size << "), "
+ << "Align(" << Align << "), "
+ << "Offset(" << Offset << "), "
+ << "Flags(" << Flags << ")\n";
+}
+#endif
+
+//===----------------------------------------------------------------------===//
+
+BasicTypeDesc::BasicTypeDesc()
+: TypeDesc(DW_TAG_base_type)
+, Encoding(0)
+{}
+
+// Implement isa/cast/dyncast.
+bool BasicTypeDesc::classof(const DebugInfoDesc *D) {
+ return D->getTag() == DW_TAG_base_type;
+}
+
+/// ApplyToFields - Target the visitor to the fields of the BasicTypeDesc.
+///
+void BasicTypeDesc::ApplyToFields(DIVisitor *Visitor) {
+ TypeDesc::ApplyToFields(Visitor);
+
+ Visitor->Apply(Encoding);
+}
+
+/// getDescString - Return a string used to compose global names and labels.
+///
+const char *BasicTypeDesc::getDescString() const {
+ return "llvm.dbg.basictype";
+}
+
+/// getTypeString - Return a string used to label this descriptor's type.
+///
+const char *BasicTypeDesc::getTypeString() const {
+ return "llvm.dbg.basictype.type";
+}
+
+#ifndef NDEBUG
+void BasicTypeDesc::dump() {
+ cerr << getDescString() << " "
+ << "Version(" << getVersion() << "), "
+ << "Tag(" << getTag() << "), "
+ << "Context(" << getContext() << "), "
+ << "Name(\"" << getName() << "\"), "
+ << "Size(" << getSize() << "), "
+ << "Encoding(" << Encoding << ")\n";
+}
+#endif
+
+//===----------------------------------------------------------------------===//
+
+DerivedTypeDesc::DerivedTypeDesc(unsigned T)
+: TypeDesc(T)
+, FromType(NULL)
+{}
+
+// Implement isa/cast/dyncast.
+bool DerivedTypeDesc::classof(const DebugInfoDesc *D) {
+ unsigned T = D->getTag();
+ switch (T) {
+ case DW_TAG_typedef:
+ case DW_TAG_pointer_type:
+ case DW_TAG_reference_type:
+ case DW_TAG_const_type:
+ case DW_TAG_volatile_type:
+ case DW_TAG_restrict_type:
+ case DW_TAG_member:
+ case DW_TAG_inheritance:
+ return true;
+ default: break;
+ }
+ return false;
+}
+
+/// ApplyToFields - Target the visitor to the fields of the DerivedTypeDesc.
+///
+void DerivedTypeDesc::ApplyToFields(DIVisitor *Visitor) {
+ TypeDesc::ApplyToFields(Visitor);
+
+ Visitor->Apply(FromType);
+}
+
+/// getDescString - Return a string used to compose global names and labels.
+///
+const char *DerivedTypeDesc::getDescString() const {
+ return "llvm.dbg.derivedtype";
+}
+
+/// getTypeString - Return a string used to label this descriptor's type.
+///
+const char *DerivedTypeDesc::getTypeString() const {
+ return "llvm.dbg.derivedtype.type";
+}
+
+#ifndef NDEBUG
+void DerivedTypeDesc::dump() {
+ cerr << getDescString() << " "
+ << "Version(" << getVersion() << "), "
+ << "Tag(" << getTag() << "), "
+ << "Context(" << getContext() << "), "
+ << "Name(\"" << getName() << "\"), "
+ << "Size(" << getSize() << "), "
+ << "File(" << getFile() << "), "
+ << "Line(" << getLine() << "), "
+ << "FromType(" << FromType << ")\n";
+}
+#endif
+
+//===----------------------------------------------------------------------===//
+
+CompositeTypeDesc::CompositeTypeDesc(unsigned T)
+: DerivedTypeDesc(T)
+, Elements()
+{}
+
+// Implement isa/cast/dyncast.
+bool CompositeTypeDesc::classof(const DebugInfoDesc *D) {
+ unsigned T = D->getTag();
+ switch (T) {
+ case DW_TAG_array_type:
+ case DW_TAG_structure_type:
+ case DW_TAG_union_type:
+ case DW_TAG_enumeration_type:
+ case DW_TAG_vector_type:
+ case DW_TAG_subroutine_type:
+ return true;
+ default: break;
+ }
+ return false;
+}
+
+/// ApplyToFields - Target the visitor to the fields of the CompositeTypeDesc.
+///
+void CompositeTypeDesc::ApplyToFields(DIVisitor *Visitor) {
+ DerivedTypeDesc::ApplyToFields(Visitor);
+
+ Visitor->Apply(Elements);
+}
+
+/// getDescString - Return a string used to compose global names and labels.
+///
+const char *CompositeTypeDesc::getDescString() const {
+ return "llvm.dbg.compositetype";
+}
+
+/// getTypeString - Return a string used to label this descriptor's type.
+///
+const char *CompositeTypeDesc::getTypeString() const {
+ return "llvm.dbg.compositetype.type";
+}
+
+#ifndef NDEBUG
+void CompositeTypeDesc::dump() {
+ cerr << getDescString() << " "
+ << "Version(" << getVersion() << "), "
+ << "Tag(" << getTag() << "), "
+ << "Context(" << getContext() << "), "
+ << "Name(\"" << getName() << "\"), "
+ << "Size(" << getSize() << "), "
+ << "File(" << getFile() << "), "
+ << "Line(" << getLine() << "), "
+ << "FromType(" << getFromType() << "), "
+ << "Elements.size(" << Elements.size() << ")\n";
+}
+#endif
+
+//===----------------------------------------------------------------------===//
+
+SubrangeDesc::SubrangeDesc()
+: DebugInfoDesc(DW_TAG_subrange_type)
+, Lo(0)
+, Hi(0)
+{}
+
+// Implement isa/cast/dyncast.
+bool SubrangeDesc::classof(const DebugInfoDesc *D) {
+ return D->getTag() == DW_TAG_subrange_type;
+}
+
+/// ApplyToFields - Target the visitor to the fields of the SubrangeDesc.
+///
+void SubrangeDesc::ApplyToFields(DIVisitor *Visitor) {
+ DebugInfoDesc::ApplyToFields(Visitor);
+
+ Visitor->Apply(Lo);
+ Visitor->Apply(Hi);
+}
+
+/// getDescString - Return a string used to compose global names and labels.
+///
+const char *SubrangeDesc::getDescString() const {
+ return "llvm.dbg.subrange";
+}
+
+/// getTypeString - Return a string used to label this descriptor's type.
+///
+const char *SubrangeDesc::getTypeString() const {
+ return "llvm.dbg.subrange.type";
+}
+
+#ifndef NDEBUG
+void SubrangeDesc::dump() {
+ cerr << getDescString() << " "
+ << "Version(" << getVersion() << "), "
+ << "Tag(" << getTag() << "), "
+ << "Lo(" << Lo << "), "
+ << "Hi(" << Hi << ")\n";
+}
+#endif
+
+//===----------------------------------------------------------------------===//
+
+EnumeratorDesc::EnumeratorDesc()
+: DebugInfoDesc(DW_TAG_enumerator)
+, Name("")
+, Value(0)
+{}
+
+// Implement isa/cast/dyncast.
+bool EnumeratorDesc::classof(const DebugInfoDesc *D) {
+ return D->getTag() == DW_TAG_enumerator;
+}
+
+/// ApplyToFields - Target the visitor to the fields of the EnumeratorDesc.
+///
+void EnumeratorDesc::ApplyToFields(DIVisitor *Visitor) {
+ DebugInfoDesc::ApplyToFields(Visitor);
+
+ Visitor->Apply(Name);
+ Visitor->Apply(Value);
+}
+
+/// getDescString - Return a string used to compose global names and labels.
+///
+const char *EnumeratorDesc::getDescString() const {
+ return "llvm.dbg.enumerator";
+}
+
+/// getTypeString - Return a string used to label this descriptor's type.
+///
+const char *EnumeratorDesc::getTypeString() const {
+ return "llvm.dbg.enumerator.type";
+}
+
+#ifndef NDEBUG
+void EnumeratorDesc::dump() {
+ cerr << getDescString() << " "
+ << "Version(" << getVersion() << "), "
+ << "Tag(" << getTag() << "), "
+ << "Name(" << Name << "), "
+ << "Value(" << Value << ")\n";
+}
+#endif
+
+//===----------------------------------------------------------------------===//
+
+VariableDesc::VariableDesc(unsigned T)
+: DebugInfoDesc(T)
+, Context(NULL)
+, Name("")
+, File(NULL)
+, Line(0)
+, TyDesc(0)
+{}
+
+// Implement isa/cast/dyncast.
+bool VariableDesc::classof(const DebugInfoDesc *D) {
+ unsigned T = D->getTag();
+ switch (T) {
+ case DW_TAG_auto_variable:
+ case DW_TAG_arg_variable:
+ case DW_TAG_return_variable:
+ return true;
+ default: break;
+ }
+ return false;
+}
+
+/// ApplyToFields - Target the visitor to the fields of the VariableDesc.
+///
+void VariableDesc::ApplyToFields(DIVisitor *Visitor) {
+ DebugInfoDesc::ApplyToFields(Visitor);
+
+ Visitor->Apply(Context);
+ Visitor->Apply(Name);
+ Visitor->Apply(File);
+ Visitor->Apply(Line);
+ Visitor->Apply(TyDesc);
+}
+
+/// getDescString - Return a string used to compose global names and labels.
+///
+const char *VariableDesc::getDescString() const {
+ return "llvm.dbg.variable";
+}
+
+/// getTypeString - Return a string used to label this descriptor's type.
+///
+const char *VariableDesc::getTypeString() const {
+ return "llvm.dbg.variable.type";
+}
+
+#ifndef NDEBUG
+void VariableDesc::dump() {
+ cerr << getDescString() << " "
+ << "Version(" << getVersion() << "), "
+ << "Tag(" << getTag() << "), "
+ << "Context(" << Context << "), "
+ << "Name(\"" << Name << "\"), "
+ << "File(" << File << "), "
+ << "Line(" << Line << "), "
+ << "TyDesc(" << TyDesc << ")\n";
+}
+#endif
+
+//===----------------------------------------------------------------------===//
+
+GlobalDesc::GlobalDesc(unsigned T)
+: AnchoredDesc(T)
+, Context(0)
+, Name("")
+, FullName("")
+, LinkageName("")
+, File(NULL)
+, Line(0)
+, TyDesc(NULL)
+, IsStatic(false)
+, IsDefinition(false)
+{}
+
+/// ApplyToFields - Target the visitor to the fields of the global.
+///
+void GlobalDesc::ApplyToFields(DIVisitor *Visitor) {
+ AnchoredDesc::ApplyToFields(Visitor);
+
+ Visitor->Apply(Context);
+ Visitor->Apply(Name);
+ Visitor->Apply(FullName);
+ Visitor->Apply(LinkageName);
+ Visitor->Apply(File);
+ Visitor->Apply(Line);
+ Visitor->Apply(TyDesc);
+ Visitor->Apply(IsStatic);
+ Visitor->Apply(IsDefinition);
+}
+
+//===----------------------------------------------------------------------===//
+
+GlobalVariableDesc::GlobalVariableDesc()
+: GlobalDesc(DW_TAG_variable)
+, Global(NULL)
+{}
+
+// Implement isa/cast/dyncast.
+bool GlobalVariableDesc::classof(const DebugInfoDesc *D) {
+ return D->getTag() == DW_TAG_variable;
+}
+
+/// ApplyToFields - Target the visitor to the fields of the GlobalVariableDesc.
+///
+void GlobalVariableDesc::ApplyToFields(DIVisitor *Visitor) {
+ GlobalDesc::ApplyToFields(Visitor);
+
+ Visitor->Apply(Global);
+}
+
+/// getDescString - Return a string used to compose global names and labels.
+///
+const char *GlobalVariableDesc::getDescString() const {
+ return "llvm.dbg.global_variable";
+}
+
+/// getTypeString - Return a string used to label this descriptors type.
+///
+const char *GlobalVariableDesc::getTypeString() const {
+ return "llvm.dbg.global_variable.type";
+}
+
+/// getAnchorString - Return a string used to label this descriptor's anchor.
+///
+const char *GlobalVariableDesc::AnchorString = "llvm.dbg.global_variables";
+const char *GlobalVariableDesc::getAnchorString() const {
+ return AnchorString;
+}
+
+#ifndef NDEBUG
+void GlobalVariableDesc::dump() {
+ cerr << getDescString() << " "
+ << "Version(" << getVersion() << "), "
+ << "Tag(" << getTag() << "), "
+ << "Anchor(" << getAnchor() << "), "
+ << "Name(\"" << getName() << "\"), "
+ << "FullName(\"" << getFullName() << "\"), "
+ << "LinkageName(\"" << getLinkageName() << "\"), "
+ << "File(" << getFile() << "),"
+ << "Line(" << getLine() << "),"
+ << "Type(" << getType() << "), "
+ << "IsStatic(" << (isStatic() ? "true" : "false") << "), "
+ << "IsDefinition(" << (isDefinition() ? "true" : "false") << "), "
+ << "Global(" << Global << ")\n";
+}
+#endif
+
+//===----------------------------------------------------------------------===//
+
+SubprogramDesc::SubprogramDesc()
+: GlobalDesc(DW_TAG_subprogram)
+{}
+
+// Implement isa/cast/dyncast.
+bool SubprogramDesc::classof(const DebugInfoDesc *D) {
+ return D->getTag() == DW_TAG_subprogram;
+}
+
+/// ApplyToFields - Target the visitor to the fields of the
+/// SubprogramDesc.
+void SubprogramDesc::ApplyToFields(DIVisitor *Visitor) {
+ GlobalDesc::ApplyToFields(Visitor);
+}
+
+/// getDescString - Return a string used to compose global names and labels.
+///
+const char *SubprogramDesc::getDescString() const {
+ return "llvm.dbg.subprogram";
+}
+
+/// getTypeString - Return a string used to label this descriptors type.
+///
+const char *SubprogramDesc::getTypeString() const {
+ return "llvm.dbg.subprogram.type";
+}
+
+/// getAnchorString - Return a string used to label this descriptor's anchor.
+///
+const char *SubprogramDesc::AnchorString = "llvm.dbg.subprograms";
+const char *SubprogramDesc::getAnchorString() const {
+ return AnchorString;
+}
+
+#ifndef NDEBUG
+void SubprogramDesc::dump() {
+ cerr << getDescString() << " "
+ << "Version(" << getVersion() << "), "
+ << "Tag(" << getTag() << "), "
+ << "Anchor(" << getAnchor() << "), "
+ << "Name(\"" << getName() << "\"), "
+ << "FullName(\"" << getFullName() << "\"), "
+ << "LinkageName(\"" << getLinkageName() << "\"), "
+ << "File(" << getFile() << "),"
+ << "Line(" << getLine() << "),"
+ << "Type(" << getType() << "), "
+ << "IsStatic(" << (isStatic() ? "true" : "false") << "), "
+ << "IsDefinition(" << (isDefinition() ? "true" : "false") << ")\n";
+}
+#endif
+
+//===----------------------------------------------------------------------===//
+
+BlockDesc::BlockDesc()
+: DebugInfoDesc(DW_TAG_lexical_block)
+, Context(NULL)
+{}
+
+// Implement isa/cast/dyncast.
+bool BlockDesc::classof(const DebugInfoDesc *D) {
+ return D->getTag() == DW_TAG_lexical_block;
+}
+
+/// ApplyToFields - Target the visitor to the fields of the BlockDesc.
+///
+void BlockDesc::ApplyToFields(DIVisitor *Visitor) {
+ DebugInfoDesc::ApplyToFields(Visitor);
+
+ Visitor->Apply(Context);
+}
+
+/// getDescString - Return a string used to compose global names and labels.
+///
+const char *BlockDesc::getDescString() const {
+ return "llvm.dbg.block";
+}
+
+/// getTypeString - Return a string used to label this descriptors type.
+///
+const char *BlockDesc::getTypeString() const {
+ return "llvm.dbg.block.type";
+}
+
+#ifndef NDEBUG
+void BlockDesc::dump() {
+ cerr << getDescString() << " "
+ << "Version(" << getVersion() << "), "
+ << "Tag(" << getTag() << "),"
+ << "Context(" << Context << ")\n";
+}
+#endif
+
+//===----------------------------------------------------------------------===//
+
+DebugInfoDesc *DIDeserializer::Deserialize(Value *V) {
+ return Deserialize(getGlobalVariable(V));
+}
+DebugInfoDesc *DIDeserializer::Deserialize(GlobalVariable *GV) {
+ // Handle NULL.
+ if (!GV) return NULL;
+
+ // Check to see if it has been already deserialized.
+ DebugInfoDesc *&Slot = GlobalDescs[GV];
+ if (Slot) return Slot;
+
+ // Get the Tag from the global.
+ unsigned Tag = DebugInfoDesc::TagFromGlobal(GV);
+
+ // Create an empty instance of the correct sort.
+ Slot = DebugInfoDesc::DescFactory(Tag);
+
+ // If not a user defined descriptor.
+ if (Slot) {
+ // Deserialize the fields.
+ DIDeserializeVisitor DRAM(*this, GV);
+ DRAM.ApplyToFields(Slot);
+ }
+
+ return Slot;
+}
+
+//===----------------------------------------------------------------------===//
+
+/// getStrPtrType - Return a "sbyte *" type.
+///
+const PointerType *DISerializer::getStrPtrType() {
+ // If not already defined.
+ if (!StrPtrTy) {
+ // Construct the pointer to signed bytes.
+ StrPtrTy = PointerType::get(Type::Int8Ty);
+ }
+
+ return StrPtrTy;
+}
+
+/// getEmptyStructPtrType - Return a "{ }*" type.
+///
+const PointerType *DISerializer::getEmptyStructPtrType() {
+ // If not already defined.
+ if (!EmptyStructPtrTy) {
+ // Construct the empty structure type.
+ const StructType *EmptyStructTy =
+ StructType::get(std::vector<const Type*>());
+ // Construct the pointer to empty structure type.
+ EmptyStructPtrTy = PointerType::get(EmptyStructTy);
+ }
+
+ return EmptyStructPtrTy;
+}
+
+/// getTagType - Return the type describing the specified descriptor (via tag.)
+///
+const StructType *DISerializer::getTagType(DebugInfoDesc *DD) {
+ // Attempt to get the previously defined type.
+ StructType *&Ty = TagTypes[DD->getTag()];
+
+ // If not already defined.
+ if (!Ty) {
+ // Set up fields vector.
+ std::vector<const Type*> Fields;
+ // Get types of fields.
+ DIGetTypesVisitor GTAM(*this, Fields);
+ GTAM.ApplyToFields(DD);
+
+ // Construct structured type.
+ Ty = StructType::get(Fields);
+
+ // Register type name with module.
+ M->addTypeName(DD->getTypeString(), Ty);
+ }
+
+ return Ty;
+}
+
+/// getString - Construct the string as constant string global.
+///
+Constant *DISerializer::getString(const std::string &String) {
+ // Check string cache for previous edition.
+ Constant *&Slot = StringCache[String];
+ // Return Constant if previously defined.
+ if (Slot) return Slot;
+ // If empty string then use a sbyte* null instead.
+ if (String.empty()) {
+ Slot = ConstantPointerNull::get(getStrPtrType());
+ } else {
+ // Construct string as an llvm constant.
+ Constant *ConstStr = ConstantArray::get(String);
+ // Otherwise create and return a new string global.
+ GlobalVariable *StrGV = new GlobalVariable(ConstStr->getType(), true,
+ GlobalVariable::InternalLinkage,
+ ConstStr, ".str", M);
+ StrGV->setSection("llvm.metadata");
+ // Convert to generic string pointer.
+ Slot = ConstantExpr::getBitCast(StrGV, getStrPtrType());
+ }
+ return Slot;
+
+}
+
+/// Serialize - Recursively cast the specified descriptor into a GlobalVariable
+/// so that it can be serialized to a .bc or .ll file.
+GlobalVariable *DISerializer::Serialize(DebugInfoDesc *DD) {
+ // Check if the DebugInfoDesc is already in the map.
+ GlobalVariable *&Slot = DescGlobals[DD];
+
+ // See if DebugInfoDesc exists, if so return prior GlobalVariable.
+ if (Slot) return Slot;
+
+ // Get the type associated with the Tag.
+ const StructType *Ty = getTagType(DD);
+
+ // Create the GlobalVariable early to prevent infinite recursion.
+ GlobalVariable *GV = new GlobalVariable(Ty, true, DD->getLinkage(),
+ NULL, DD->getDescString(), M);
+ GV->setSection("llvm.metadata");
+
+ // Insert new GlobalVariable in DescGlobals map.
+ Slot = GV;
+
+ // Set up elements vector
+ std::vector<Constant*> Elements;
+ // Add fields.
+ DISerializeVisitor SRAM(*this, Elements);
+ SRAM.ApplyToFields(DD);
+
+ // Set the globals initializer.
+ GV->setInitializer(ConstantStruct::get(Ty, Elements));
+
+ return GV;
+}
+
+//===----------------------------------------------------------------------===//
+
+/// Verify - Return true if the GlobalVariable appears to be a valid
+/// serialization of a DebugInfoDesc.
+bool DIVerifier::Verify(Value *V) {
+ return !V || Verify(getGlobalVariable(V));
+}
+bool DIVerifier::Verify(GlobalVariable *GV) {
+ // NULLs are valid.
+ if (!GV) return true;
+
+ // Check prior validity.
+ unsigned &ValiditySlot = Validity[GV];
+
+ // If visited before then use old state.
+ if (ValiditySlot) return ValiditySlot == Valid;
+
+ // Assume validity for the time being (recursion.)
+ ValiditySlot = Valid;
+
+ // Make sure the global is internal or link once (anchor.)
+ if (GV->getLinkage() != GlobalValue::InternalLinkage &&
+ GV->getLinkage() != GlobalValue::LinkOnceLinkage) {
+ ValiditySlot = Invalid;
+ return false;
+ }
+
+ // Get the Tag.
+ unsigned Tag = DebugInfoDesc::TagFromGlobal(GV);
+
+ // Check for user defined descriptors.
+ if (Tag == DW_TAG_invalid) {
+ ValiditySlot = Valid;
+ return true;
+ }
+
+ // Get the Version.
+ unsigned Version = DebugInfoDesc::VersionFromGlobal(GV);
+
+ // Check for version mismatch.
+ if (Version != LLVMDebugVersion) {
+ ValiditySlot = Invalid;
+ return false;
+ }
+
+ // Construct an empty DebugInfoDesc.
+ DebugInfoDesc *DD = DebugInfoDesc::DescFactory(Tag);
+
+ // Allow for user defined descriptors.
+ if (!DD) return true;
+
+ // Get the initializer constant.
+ ConstantStruct *CI = cast<ConstantStruct>(GV->getInitializer());
+
+ // Get the operand count.
+ unsigned N = CI->getNumOperands();
+
+ // Get the field count.
+ unsigned &CountSlot = Counts[Tag];
+ if (!CountSlot) {
+ // Check the operand count to the field count
+ DICountVisitor CTAM;
+ CTAM.ApplyToFields(DD);
+ CountSlot = CTAM.getCount();
+ }
+
+ // Field count must be at most equal operand count.
+ if (CountSlot > N) {
+ delete DD;
+ ValiditySlot = Invalid;
+ return false;
+ }
+
+ // Check each field for valid type.
+ DIVerifyVisitor VRAM(*this, GV);
+ VRAM.ApplyToFields(DD);
+
+ // Release empty DebugInfoDesc.
+ delete DD;
+
+ // If fields are not valid.
+ if (!VRAM.isValid()) {
+ ValiditySlot = Invalid;
+ return false;
+ }
+
+ return true;
+}
+
+//===----------------------------------------------------------------------===//
+
+DebugScope::~DebugScope() {
+ for (unsigned i = 0, N = Scopes.size(); i < N; ++i) delete Scopes[i];
+ for (unsigned j = 0, M = Variables.size(); j < M; ++j) delete Variables[j];
+}
+
+//===----------------------------------------------------------------------===//
+
+MachineModuleInfo::MachineModuleInfo()
+: ImmutablePass((intptr_t)&ID)
+, DR()
+, VR()
+, CompileUnits()
+, Directories()
+, SourceFiles()
+, Lines()
+, LabelIDList()
+, ScopeMap()
+, RootScope(NULL)
+, FrameMoves()
+, LandingPads()
+, Personalities()
+, CallsEHReturn(0)
+, CallsUnwindInit(0)
+{
+ // Always emit "no personality" info
+ Personalities.push_back(NULL);
+}
+MachineModuleInfo::~MachineModuleInfo() {
+
+}
+
+/// doInitialization - Initialize the state for a new module.
+///
+bool MachineModuleInfo::doInitialization() {
+ return false;
+}
+
+/// doFinalization - Tear down the state after completion of a module.
+///
+bool MachineModuleInfo::doFinalization() {
+ return false;
+}
+
+/// BeginFunction - Begin gathering function meta information.
+///
+void MachineModuleInfo::BeginFunction(MachineFunction *MF) {
+ // Coming soon.
+}
+
+/// EndFunction - Discard function meta information.
+///
+void MachineModuleInfo::EndFunction() {
+ // Clean up scope information.
+ if (RootScope) {
+ delete RootScope;
+ ScopeMap.clear();
+ RootScope = NULL;
+ }
+
+ // Clean up line info.
+ Lines.clear();
+
+ // Clean up frame info.
+ FrameMoves.clear();
+
+ // Clean up exception info.
+ LandingPads.clear();
+ TypeInfos.clear();
+ FilterIds.clear();
+ FilterEnds.clear();
+ CallsEHReturn = 0;
+ CallsUnwindInit = 0;
+}
+
+/// getDescFor - Convert a Value to a debug information descriptor.
+///
+// FIXME - use new Value type when available.
+DebugInfoDesc *MachineModuleInfo::getDescFor(Value *V) {
+ return DR.Deserialize(V);
+}
+
+/// Verify - Verify that a Value is debug information descriptor.
+///
+bool MachineModuleInfo::Verify(Value *V) {
+ return VR.Verify(V);
+}
+
+/// AnalyzeModule - Scan the module for global debug information.
+///
+void MachineModuleInfo::AnalyzeModule(Module &M) {
+ SetupCompileUnits(M);
+}
+
+/// needsFrameInfo - Returns true if we need to gather callee-saved register
+/// move info for the frame.
+bool MachineModuleInfo::needsFrameInfo() const {
+ return hasDebugInfo() || ExceptionHandling;
+}
+
+/// SetupCompileUnits - Set up the unique vector of compile units.
+///
+void MachineModuleInfo::SetupCompileUnits(Module &M) {
+ std::vector<CompileUnitDesc *>CU = getAnchoredDescriptors<CompileUnitDesc>(M);
+
+ for (unsigned i = 0, N = CU.size(); i < N; i++) {
+ CompileUnits.insert(CU[i]);
+ }
+}
+
+/// getCompileUnits - Return a vector of debug compile units.
+///
+const UniqueVector<CompileUnitDesc *> MachineModuleInfo::getCompileUnits()const{
+ return CompileUnits;
+}
+
+/// getGlobalVariablesUsing - Return all of the GlobalVariables that use the
+/// named GlobalVariable.
+std::vector<GlobalVariable*>
+MachineModuleInfo::getGlobalVariablesUsing(Module &M,
+ const std::string &RootName) {
+ return ::getGlobalVariablesUsing(M, RootName);
+}
+
+/// RecordLabel - Records location information and associates it with a
+/// debug label. Returns a unique label ID used to generate a label and
+/// provide correspondence to the source line list.
+unsigned MachineModuleInfo::RecordLabel(unsigned Line, unsigned Column,
+ unsigned Source) {
+ unsigned ID = NextLabelID();
+ Lines.push_back(SourceLineInfo(Line, Column, Source, ID));
+ return ID;
+}
+
+/// RecordSource - Register a source file with debug info. Returns an source
+/// ID.
+unsigned MachineModuleInfo::RecordSource(const std::string &Directory,
+ const std::string &Source) {
+ unsigned DirectoryID = Directories.insert(Directory);
+ return SourceFiles.insert(SourceFileInfo(DirectoryID, Source));
+}
+unsigned MachineModuleInfo::RecordSource(const CompileUnitDesc *CompileUnit) {
+ return RecordSource(CompileUnit->getDirectory(),
+ CompileUnit->getFileName());
+}
+
+/// RecordRegionStart - Indicate the start of a region.
+///
+unsigned MachineModuleInfo::RecordRegionStart(Value *V) {
+ // FIXME - need to be able to handle split scopes because of bb cloning.
+ DebugInfoDesc *ScopeDesc = DR.Deserialize(V);
+ DebugScope *Scope = getOrCreateScope(ScopeDesc);
+ unsigned ID = NextLabelID();
+ if (!Scope->getStartLabelID()) Scope->setStartLabelID(ID);
+ return ID;
+}
+
+/// RecordRegionEnd - Indicate the end of a region.
+///
+unsigned MachineModuleInfo::RecordRegionEnd(Value *V) {
+ // FIXME - need to be able to handle split scopes because of bb cloning.
+ DebugInfoDesc *ScopeDesc = DR.Deserialize(V);
+ DebugScope *Scope = getOrCreateScope(ScopeDesc);
+ unsigned ID = NextLabelID();
+ Scope->setEndLabelID(ID);
+ return ID;
+}
+
+/// RecordVariable - Indicate the declaration of a local variable.
+///
+void MachineModuleInfo::RecordVariable(Value *V, unsigned FrameIndex) {
+ VariableDesc *VD = cast<VariableDesc>(DR.Deserialize(V));
+ DebugScope *Scope = getOrCreateScope(VD->getContext());
+ DebugVariable *DV = new DebugVariable(VD, FrameIndex);
+ Scope->AddVariable(DV);
+}
+
+/// getOrCreateScope - Returns the scope associated with the given descriptor.
+///
+DebugScope *MachineModuleInfo::getOrCreateScope(DebugInfoDesc *ScopeDesc) {
+ DebugScope *&Slot = ScopeMap[ScopeDesc];
+ if (!Slot) {
+ // FIXME - breaks down when the context is an inlined function.
+ DebugInfoDesc *ParentDesc = NULL;
+ if (BlockDesc *Block = dyn_cast<BlockDesc>(ScopeDesc)) {
+ ParentDesc = Block->getContext();
+ }
+ DebugScope *Parent = ParentDesc ? getOrCreateScope(ParentDesc) : NULL;
+ Slot = new DebugScope(Parent, ScopeDesc);
+ if (Parent) {
+ Parent->AddScope(Slot);
+ } else if (RootScope) {
+ // FIXME - Add inlined function scopes to the root so we can delete
+ // them later. Long term, handle inlined functions properly.
+ RootScope->AddScope(Slot);
+ } else {
+ // First function is top level function.
+ RootScope = Slot;
+ }
+ }
+ return Slot;
+}
+
+//===-EH-------------------------------------------------------------------===//
+
+/// getOrCreateLandingPadInfo - Find or create an LandingPadInfo for the
+/// specified MachineBasicBlock.
+LandingPadInfo &MachineModuleInfo::getOrCreateLandingPadInfo
+ (MachineBasicBlock *LandingPad) {
+ unsigned N = LandingPads.size();
+ for (unsigned i = 0; i < N; ++i) {
+ LandingPadInfo &LP = LandingPads[i];
+ if (LP.LandingPadBlock == LandingPad)
+ return LP;
+ }
+
+ LandingPads.push_back(LandingPadInfo(LandingPad));
+ return LandingPads[N];
+}
+
+/// addInvoke - Provide the begin and end labels of an invoke style call and
+/// associate it with a try landing pad block.
+void MachineModuleInfo::addInvoke(MachineBasicBlock *LandingPad,
+ unsigned BeginLabel, unsigned EndLabel) {
+ LandingPadInfo &LP = getOrCreateLandingPadInfo(LandingPad);
+ LP.BeginLabels.push_back(BeginLabel);
+ LP.EndLabels.push_back(EndLabel);
+}
+
+/// addLandingPad - Provide the label of a try LandingPad block.
+///
+unsigned MachineModuleInfo::addLandingPad(MachineBasicBlock *LandingPad) {
+ unsigned LandingPadLabel = NextLabelID();
+ LandingPadInfo &LP = getOrCreateLandingPadInfo(LandingPad);
+ LP.LandingPadLabel = LandingPadLabel;
+ return LandingPadLabel;
+}
+
+/// addPersonality - Provide the personality function for the exception
+/// information.
+void MachineModuleInfo::addPersonality(MachineBasicBlock *LandingPad,
+ Function *Personality) {
+ LandingPadInfo &LP = getOrCreateLandingPadInfo(LandingPad);
+ LP.Personality = Personality;
+
+ for (unsigned i = 0; i < Personalities.size(); ++i)
+ if (Personalities[i] == Personality)
+ return;
+
+ Personalities.push_back(Personality);
+}
+
+/// addCatchTypeInfo - Provide the catch typeinfo for a landing pad.
+///
+void MachineModuleInfo::addCatchTypeInfo(MachineBasicBlock *LandingPad,
+ std::vector<GlobalVariable *> &TyInfo) {
+ LandingPadInfo &LP = getOrCreateLandingPadInfo(LandingPad);
+ for (unsigned N = TyInfo.size(); N; --N)
+ LP.TypeIds.push_back(getTypeIDFor(TyInfo[N - 1]));
+}
+
+/// addFilterTypeInfo - Provide the filter typeinfo for a landing pad.
+///
+void MachineModuleInfo::addFilterTypeInfo(MachineBasicBlock *LandingPad,
+ std::vector<GlobalVariable *> &TyInfo) {
+ LandingPadInfo &LP = getOrCreateLandingPadInfo(LandingPad);
+ std::vector<unsigned> IdsInFilter (TyInfo.size());
+ for (unsigned I = 0, E = TyInfo.size(); I != E; ++I)
+ IdsInFilter[I] = getTypeIDFor(TyInfo[I]);
+ LP.TypeIds.push_back(getFilterIDFor(IdsInFilter));
+}
+
+/// TidyLandingPads - Remap landing pad labels and remove any deleted landing
+/// pads.
+void MachineModuleInfo::TidyLandingPads() {
+ for (unsigned i = 0; i != LandingPads.size(); ) {
+ LandingPadInfo &LandingPad = LandingPads[i];
+ LandingPad.LandingPadLabel = MappedLabel(LandingPad.LandingPadLabel);
+
+ if (!LandingPad.LandingPadBlock)
+ // Must not have cleanups if no landing pad.
+ LandingPad.TypeIds.clear();
+
+ // Special case: we *should* emit LPs with null LP MBB. This indicates
+ // "rethrow" case.
+ if (!LandingPad.LandingPadLabel && LandingPad.LandingPadBlock) {
+ LandingPads.erase(LandingPads.begin() + i);
+ continue;
+ }
+
+ for (unsigned j=0; j != LandingPads[i].BeginLabels.size(); ) {
+ unsigned BeginLabel = MappedLabel(LandingPad.BeginLabels[j]);
+ unsigned EndLabel = MappedLabel(LandingPad.EndLabels[j]);
+
+
+ if (!BeginLabel || !EndLabel) {
+ LandingPad.BeginLabels.erase(LandingPad.BeginLabels.begin() + j);
+ LandingPad.EndLabels.erase(LandingPad.EndLabels.begin() + j);
+ continue;
+ }
+
+ LandingPad.BeginLabels[j] = BeginLabel;
+ LandingPad.EndLabels[j] = EndLabel;
+ ++j;
+ }
+
+ ++i;
+ }
+}
+
+/// getTypeIDFor - Return the type id for the specified typeinfo. This is
+/// function wide.
+unsigned MachineModuleInfo::getTypeIDFor(GlobalVariable *TI) {
+ for (unsigned i = 0, N = TypeInfos.size(); i != N; ++i)
+ if (TypeInfos[i] == TI) return i + 1;
+
+ TypeInfos.push_back(TI);
+ return TypeInfos.size();
+}
+
+/// getFilterIDFor - Return the filter id for the specified typeinfos. This is
+/// function wide.
+int MachineModuleInfo::getFilterIDFor(std::vector<unsigned> &TyIds) {
+ // If the new filter coincides with the tail of an existing filter, then
+ // re-use the existing filter. Folding filters more than this requires
+ // re-ordering filters and/or their elements - probably not worth it.
+ for (std::vector<unsigned>::iterator I = FilterEnds.begin(),
+ E = FilterEnds.end(); I != E; ++I) {
+ unsigned i = *I, j = TyIds.size();
+
+ while (i && j)
+ if (FilterIds[--i] != TyIds[--j])
+ goto try_next;
+
+ if (!j)
+ // The new filter coincides with range [i, end) of the existing filter.
+ return -(1 + i);
+
+try_next:;
+ }
+
+ // Add the new filter.
+ int FilterID = -(1 + FilterIds.size());
+ FilterIds.reserve(FilterIds.size() + TyIds.size() + 1);
+ for (unsigned I = 0, N = TyIds.size(); I != N; ++I)
+ FilterIds.push_back(TyIds[I]);
+ FilterEnds.push_back(FilterIds.size());
+ FilterIds.push_back(0); // terminator
+ return FilterID;
+}
+
+/// getPersonality - Return the personality function for the current function.
+Function *MachineModuleInfo::getPersonality() const {
+ // FIXME: Until PR1414 will be fixed, we're using 1 personality function per
+ // function
+ return !LandingPads.empty() ? LandingPads[0].Personality : NULL;
+}
+
+/// getPersonalityIndex - Return unique index for current personality
+/// function. NULL personality function should always get zero index.
+unsigned MachineModuleInfo::getPersonalityIndex() const {
+ const Function* Personality = NULL;
+
+ // Scan landing pads. If there is at least one non-NULL personality - use it.
+ for (unsigned i = 0; i != LandingPads.size(); ++i)
+ if (LandingPads[i].Personality) {
+ Personality = LandingPads[i].Personality;
+ break;
+ }
+
+ for (unsigned i = 0; i < Personalities.size(); ++i) {
+ if (Personalities[i] == Personality)
+ return i;
+ }
+
+ // This should never happen
+ assert(0 && "Personality function should be set!");
+ return 0;
+}
+
+//===----------------------------------------------------------------------===//
+/// DebugLabelFolding pass - This pass prunes out redundant labels. This allows
+/// a info consumer to determine if the range of two labels is empty, by seeing
+/// if the labels map to the same reduced label.
+
+namespace llvm {
+
+struct DebugLabelFolder : public MachineFunctionPass {
+ static char ID;
+ DebugLabelFolder() : MachineFunctionPass((intptr_t)&ID) {}
+
+ virtual bool runOnMachineFunction(MachineFunction &MF);
+ virtual const char *getPassName() const { return "Label Folder"; }
+};
+
+char DebugLabelFolder::ID = 0;
+
+bool DebugLabelFolder::runOnMachineFunction(MachineFunction &MF) {
+ // Get machine module info.
+ MachineModuleInfo *MMI = getAnalysisToUpdate<MachineModuleInfo>();
+ if (!MMI) return false;
+ // Get target instruction info.
+ const TargetInstrInfo *TII = MF.getTarget().getInstrInfo();
+ if (!TII) return false;
+
+ // Track if change is made.
+ bool MadeChange = false;
+ // No prior label to begin.
+ unsigned PriorLabel = 0;
+
+ // Iterate through basic blocks.
+ for (MachineFunction::iterator BB = MF.begin(), E = MF.end();
+ BB != E; ++BB) {
+ // Iterate through instructions.
+ for (MachineBasicBlock::iterator I = BB->begin(), E = BB->end(); I != E; ) {
+ // Is it a label.
+ if ((unsigned)I->getOpcode() == TargetInstrInfo::LABEL) {
+ // The label ID # is always operand #0, an immediate.
+ unsigned NextLabel = I->getOperand(0).getImm();
+
+ // If there was an immediate prior label.
+ if (PriorLabel) {
+ // Remap the current label to prior label.
+ MMI->RemapLabel(NextLabel, PriorLabel);
+ // Delete the current label.
+ I = BB->erase(I);
+ // Indicate a change has been made.
+ MadeChange = true;
+ continue;
+ } else {
+ // Start a new round.
+ PriorLabel = NextLabel;
+ }
+ } else {
+ // No consecutive labels.
+ PriorLabel = 0;
+ }
+
+ ++I;
+ }
+ }
+
+ return MadeChange;
+}
+
+FunctionPass *createDebugLabelFoldingPass() { return new DebugLabelFolder(); }
+
+}
+
diff --git a/lib/CodeGen/MachinePassRegistry.cpp b/lib/CodeGen/MachinePassRegistry.cpp
new file mode 100644
index 0000000..a7ba5bb
--- /dev/null
+++ b/lib/CodeGen/MachinePassRegistry.cpp
@@ -0,0 +1,41 @@
+//===-- CodeGen/MachineInstr.cpp ------------------------------------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file was developed by James M. Laskey and is distributed under
+// the University of Illinois Open Source License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains the machine function pass registry for register allocators
+// and instruction schedulers.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/CodeGen/MachinePassRegistry.h"
+
+using namespace llvm;
+
+
+/// Add - Adds a function pass to the registration list.
+///
+void MachinePassRegistry::Add(MachinePassRegistryNode *Node) {
+ Node->setNext(List);
+ List = Node;
+ if (Listener) Listener->NotifyAdd(Node->getName(),
+ Node->getCtor(),
+ Node->getDescription());
+}
+
+
+/// Remove - Removes a function pass from the registration list.
+///
+void MachinePassRegistry::Remove(MachinePassRegistryNode *Node) {
+ for (MachinePassRegistryNode **I = &List; *I; I = (*I)->getNextAddress()) {
+ if (*I == Node) {
+ if (Listener) Listener->NotifyRemove(Node->getName());
+ *I = (*I)->getNext();
+ break;
+ }
+ }
+}
diff --git a/lib/CodeGen/Makefile b/lib/CodeGen/Makefile
new file mode 100644
index 0000000..7e5d8ee
--- /dev/null
+++ b/lib/CodeGen/Makefile
@@ -0,0 +1,22 @@
+##===- lib/CodeGen/Makefile --------------------------------*- Makefile -*-===##
+#
+# The LLVM Compiler Infrastructure
+#
+# This file was developed by the LLVM research group and is distributed under
+# the University of Illinois Open Source License. See LICENSE.TXT for details.
+#
+##===----------------------------------------------------------------------===##
+
+LEVEL = ../..
+LIBRARYNAME = LLVMCodeGen
+PARALLEL_DIRS = SelectionDAG
+BUILD_ARCHIVE = 1
+
+include $(LEVEL)/Makefile.common
+
+# Xcode prior to 2.4 generates an error in -pedantic mode with use of HUGE_VAL
+# in this directory. Disable -pedantic for this broken compiler.
+ifneq ($(HUGE_VAL_SANITY),yes)
+CompileCommonOpts := $(filter-out -pedantic, $(CompileCommonOpts))
+endif
+
diff --git a/lib/CodeGen/PHIElimination.cpp b/lib/CodeGen/PHIElimination.cpp
new file mode 100644
index 0000000..fec9e2e
--- /dev/null
+++ b/lib/CodeGen/PHIElimination.cpp
@@ -0,0 +1,342 @@
+//===-- PhiElimination.cpp - Eliminate PHI nodes by inserting copies ------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file was developed by the LLVM research group and is distributed under
+// the University of Illinois Open Source License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This pass eliminates machine instruction PHI nodes by inserting copy
+// instructions. This destroys SSA information, but is the desired input for
+// some register allocators.
+//
+//===----------------------------------------------------------------------===//
+
+#define DEBUG_TYPE "phielim"
+#include "llvm/CodeGen/LiveVariables.h"
+#include "llvm/CodeGen/Passes.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/MachineInstr.h"
+#include "llvm/CodeGen/SSARegMap.h"
+#include "llvm/Target/TargetInstrInfo.h"
+#include "llvm/Target/TargetMachine.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/Statistic.h"
+#include "llvm/Support/Compiler.h"
+#include <set>
+#include <algorithm>
+using namespace llvm;
+
+STATISTIC(NumAtomic, "Number of atomic phis lowered");
+//STATISTIC(NumSimple, "Number of simple phis lowered");
+
+namespace {
+ struct VISIBILITY_HIDDEN PNE : public MachineFunctionPass {
+ static char ID; // Pass identification, replacement for typeid
+ PNE() : MachineFunctionPass((intptr_t)&ID) {}
+
+ bool runOnMachineFunction(MachineFunction &Fn) {
+ analyzePHINodes(Fn);
+
+ bool Changed = false;
+
+ // Eliminate PHI instructions by inserting copies into predecessor blocks.
+ for (MachineFunction::iterator I = Fn.begin(), E = Fn.end(); I != E; ++I)
+ Changed |= EliminatePHINodes(Fn, *I);
+
+ VRegPHIUseCount.clear();
+ return Changed;
+ }
+
+ virtual void getAnalysisUsage(AnalysisUsage &AU) const {
+ AU.addPreserved<LiveVariables>();
+ MachineFunctionPass::getAnalysisUsage(AU);
+ }
+
+ private:
+ /// EliminatePHINodes - Eliminate phi nodes by inserting copy instructions
+ /// in predecessor basic blocks.
+ ///
+ bool EliminatePHINodes(MachineFunction &MF, MachineBasicBlock &MBB);
+ void LowerAtomicPHINode(MachineBasicBlock &MBB,
+ MachineBasicBlock::iterator AfterPHIsIt);
+
+ /// analyzePHINodes - Gather information about the PHI nodes in
+ /// here. In particular, we want to map the number of uses of a virtual
+ /// register which is used in a PHI node. We map that to the BB the
+ /// vreg is coming from. This is used later to determine when the vreg
+ /// is killed in the BB.
+ ///
+ void analyzePHINodes(const MachineFunction& Fn);
+
+ typedef std::pair<const MachineBasicBlock*, unsigned> BBVRegPair;
+ typedef std::map<BBVRegPair, unsigned> VRegPHIUse;
+
+ VRegPHIUse VRegPHIUseCount;
+ };
+
+ char PNE::ID = 0;
+ RegisterPass<PNE> X("phi-node-elimination",
+ "Eliminate PHI nodes for register allocation");
+}
+
+const PassInfo *llvm::PHIEliminationID = X.getPassInfo();
+
+/// EliminatePHINodes - Eliminate phi nodes by inserting copy instructions in
+/// predecessor basic blocks.
+///
+bool PNE::EliminatePHINodes(MachineFunction &MF, MachineBasicBlock &MBB) {
+ if (MBB.empty() || MBB.front().getOpcode() != TargetInstrInfo::PHI)
+ return false; // Quick exit for basic blocks without PHIs.
+
+ // Get an iterator to the first instruction after the last PHI node (this may
+ // also be the end of the basic block).
+ MachineBasicBlock::iterator AfterPHIsIt = MBB.begin();
+ while (AfterPHIsIt != MBB.end() &&
+ AfterPHIsIt->getOpcode() == TargetInstrInfo::PHI)
+ ++AfterPHIsIt; // Skip over all of the PHI nodes...
+
+ while (MBB.front().getOpcode() == TargetInstrInfo::PHI)
+ LowerAtomicPHINode(MBB, AfterPHIsIt);
+
+ return true;
+}
+
+/// InstructionUsesRegister - Return true if the specified machine instr has a
+/// use of the specified register.
+static bool InstructionUsesRegister(MachineInstr *MI, unsigned SrcReg) {
+ for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i)
+ if (MI->getOperand(i).isRegister() &&
+ MI->getOperand(i).getReg() == SrcReg &&
+ MI->getOperand(i).isUse())
+ return true;
+ return false;
+}
+
+/// LowerAtomicPHINode - Lower the PHI node at the top of the specified block,
+/// under the assuption that it needs to be lowered in a way that supports
+/// atomic execution of PHIs. This lowering method is always correct all of the
+/// time.
+void PNE::LowerAtomicPHINode(MachineBasicBlock &MBB,
+ MachineBasicBlock::iterator AfterPHIsIt) {
+ // Unlink the PHI node from the basic block, but don't delete the PHI yet.
+ MachineInstr *MPhi = MBB.remove(MBB.begin());
+
+ unsigned DestReg = MPhi->getOperand(0).getReg();
+
+ // Create a new register for the incoming PHI arguments.
+ MachineFunction &MF = *MBB.getParent();
+ const TargetRegisterClass *RC = MF.getSSARegMap()->getRegClass(DestReg);
+ unsigned IncomingReg = MF.getSSARegMap()->createVirtualRegister(RC);
+
+ // Insert a register to register copy in the top of the current block (but
+ // after any remaining phi nodes) which copies the new incoming register
+ // into the phi node destination.
+ //
+ const MRegisterInfo *RegInfo = MF.getTarget().getRegisterInfo();
+ RegInfo->copyRegToReg(MBB, AfterPHIsIt, DestReg, IncomingReg, RC);
+
+ // Update live variable information if there is any...
+ LiveVariables *LV = getAnalysisToUpdate<LiveVariables>();
+ if (LV) {
+ MachineInstr *PHICopy = prior(AfterPHIsIt);
+
+ // Increment use count of the newly created virtual register.
+ LV->getVarInfo(IncomingReg).NumUses++;
+
+ // Add information to LiveVariables to know that the incoming value is
+ // killed. Note that because the value is defined in several places (once
+ // each for each incoming block), the "def" block and instruction fields
+ // for the VarInfo is not filled in.
+ //
+ LV->addVirtualRegisterKilled(IncomingReg, PHICopy);
+
+ // Since we are going to be deleting the PHI node, if it is the last use
+ // of any registers, or if the value itself is dead, we need to move this
+ // information over to the new copy we just inserted.
+ //
+ LV->removeVirtualRegistersKilled(MPhi);
+
+ // If the result is dead, update LV.
+ if (LV->RegisterDefIsDead(MPhi, DestReg)) {
+ LV->addVirtualRegisterDead(DestReg, PHICopy);
+ LV->removeVirtualRegistersDead(MPhi);
+ }
+
+ // Realize that the destination register is defined by the PHI copy now, not
+ // the PHI itself.
+ LV->getVarInfo(DestReg).DefInst = PHICopy;
+ }
+
+ // Adjust the VRegPHIUseCount map to account for the removal of this PHI
+ // node.
+ for (unsigned i = 1; i != MPhi->getNumOperands(); i += 2)
+ --VRegPHIUseCount[BBVRegPair(
+ MPhi->getOperand(i + 1).getMachineBasicBlock(),
+ MPhi->getOperand(i).getReg())];
+
+ // Now loop over all of the incoming arguments, changing them to copy into
+ // the IncomingReg register in the corresponding predecessor basic block.
+ //
+ std::set<MachineBasicBlock*> MBBsInsertedInto;
+ for (int i = MPhi->getNumOperands() - 1; i >= 2; i-=2) {
+ unsigned SrcReg = MPhi->getOperand(i-1).getReg();
+ assert(MRegisterInfo::isVirtualRegister(SrcReg) &&
+ "Machine PHI Operands must all be virtual registers!");
+
+ // Get the MachineBasicBlock equivalent of the BasicBlock that is the
+ // source path the PHI.
+ MachineBasicBlock &opBlock = *MPhi->getOperand(i).getMachineBasicBlock();
+
+ // Check to make sure we haven't already emitted the copy for this block.
+ // This can happen because PHI nodes may have multiple entries for the
+ // same basic block.
+ if (!MBBsInsertedInto.insert(&opBlock).second)
+ continue; // If the copy has already been emitted, we're done.
+
+ // Get an iterator pointing to the first terminator in the block (or end()).
+ // This is the point where we can insert a copy if we'd like to.
+ MachineBasicBlock::iterator I = opBlock.getFirstTerminator();
+
+ // Insert the copy.
+ RegInfo->copyRegToReg(opBlock, I, IncomingReg, SrcReg, RC);
+
+ // Now update live variable information if we have it. Otherwise we're done
+ if (!LV) continue;
+
+ // We want to be able to insert a kill of the register if this PHI
+ // (aka, the copy we just inserted) is the last use of the source
+ // value. Live variable analysis conservatively handles this by
+ // saying that the value is live until the end of the block the PHI
+ // entry lives in. If the value really is dead at the PHI copy, there
+ // will be no successor blocks which have the value live-in.
+ //
+ // Check to see if the copy is the last use, and if so, update the
+ // live variables information so that it knows the copy source
+ // instruction kills the incoming value.
+ //
+ LiveVariables::VarInfo &InRegVI = LV->getVarInfo(SrcReg);
+
+ // Loop over all of the successors of the basic block, checking to see
+ // if the value is either live in the block, or if it is killed in the
+ // block. Also check to see if this register is in use by another PHI
+ // node which has not yet been eliminated. If so, it will be killed
+ // at an appropriate point later.
+ //
+
+ // Is it used by any PHI instructions in this block?
+ bool ValueIsLive = VRegPHIUseCount[BBVRegPair(&opBlock, SrcReg)] != 0;
+
+ std::vector<MachineBasicBlock*> OpSuccBlocks;
+
+ // Otherwise, scan successors, including the BB the PHI node lives in.
+ for (MachineBasicBlock::succ_iterator SI = opBlock.succ_begin(),
+ E = opBlock.succ_end(); SI != E && !ValueIsLive; ++SI) {
+ MachineBasicBlock *SuccMBB = *SI;
+
+ // Is it alive in this successor?
+ unsigned SuccIdx = SuccMBB->getNumber();
+ if (SuccIdx < InRegVI.AliveBlocks.size() &&
+ InRegVI.AliveBlocks[SuccIdx]) {
+ ValueIsLive = true;
+ break;
+ }
+
+ OpSuccBlocks.push_back(SuccMBB);
+ }
+
+ // Check to see if this value is live because there is a use in a successor
+ // that kills it.
+ if (!ValueIsLive) {
+ switch (OpSuccBlocks.size()) {
+ case 1: {
+ MachineBasicBlock *MBB = OpSuccBlocks[0];
+ for (unsigned i = 0, e = InRegVI.Kills.size(); i != e; ++i)
+ if (InRegVI.Kills[i]->getParent() == MBB) {
+ ValueIsLive = true;
+ break;
+ }
+ break;
+ }
+ case 2: {
+ MachineBasicBlock *MBB1 = OpSuccBlocks[0], *MBB2 = OpSuccBlocks[1];
+ for (unsigned i = 0, e = InRegVI.Kills.size(); i != e; ++i)
+ if (InRegVI.Kills[i]->getParent() == MBB1 ||
+ InRegVI.Kills[i]->getParent() == MBB2) {
+ ValueIsLive = true;
+ break;
+ }
+ break;
+ }
+ default:
+ std::sort(OpSuccBlocks.begin(), OpSuccBlocks.end());
+ for (unsigned i = 0, e = InRegVI.Kills.size(); i != e; ++i)
+ if (std::binary_search(OpSuccBlocks.begin(), OpSuccBlocks.end(),
+ InRegVI.Kills[i]->getParent())) {
+ ValueIsLive = true;
+ break;
+ }
+ }
+ }
+
+ // Okay, if we now know that the value is not live out of the block,
+ // we can add a kill marker in this block saying that it kills the incoming
+ // value!
+ if (!ValueIsLive) {
+ // In our final twist, we have to decide which instruction kills the
+ // register. In most cases this is the copy, however, the first
+ // terminator instruction at the end of the block may also use the value.
+ // In this case, we should mark *it* as being the killing block, not the
+ // copy.
+ bool FirstTerminatorUsesValue = false;
+ if (I != opBlock.end()) {
+ FirstTerminatorUsesValue = InstructionUsesRegister(I, SrcReg);
+
+ // Check that no other terminators use values.
+#ifndef NDEBUG
+ for (MachineBasicBlock::iterator TI = next(I); TI != opBlock.end();
+ ++TI) {
+ assert(!InstructionUsesRegister(TI, SrcReg) &&
+ "Terminator instructions cannot use virtual registers unless"
+ "they are the first terminator in a block!");
+ }
+#endif
+ }
+
+ MachineBasicBlock::iterator KillInst;
+ if (!FirstTerminatorUsesValue)
+ KillInst = prior(I);
+ else
+ KillInst = I;
+
+ // Finally, mark it killed.
+ LV->addVirtualRegisterKilled(SrcReg, KillInst);
+
+ // This vreg no longer lives all of the way through opBlock.
+ unsigned opBlockNum = opBlock.getNumber();
+ if (opBlockNum < InRegVI.AliveBlocks.size())
+ InRegVI.AliveBlocks[opBlockNum] = false;
+ }
+ }
+
+ // Really delete the PHI instruction now!
+ delete MPhi;
+ ++NumAtomic;
+}
+
+/// analyzePHINodes - Gather information about the PHI nodes in here. In
+/// particular, we want to map the number of uses of a virtual register which is
+/// used in a PHI node. We map that to the BB the vreg is coming from. This is
+/// used later to determine when the vreg is killed in the BB.
+///
+void PNE::analyzePHINodes(const MachineFunction& Fn) {
+ for (MachineFunction::const_iterator I = Fn.begin(), E = Fn.end();
+ I != E; ++I)
+ for (MachineBasicBlock::const_iterator BBI = I->begin(), BBE = I->end();
+ BBI != BBE && BBI->getOpcode() == TargetInstrInfo::PHI; ++BBI)
+ for (unsigned i = 1, e = BBI->getNumOperands(); i != e; i += 2)
+ ++VRegPHIUseCount[BBVRegPair(
+ BBI->getOperand(i + 1).getMachineBasicBlock(),
+ BBI->getOperand(i).getReg())];
+}
diff --git a/lib/CodeGen/Passes.cpp b/lib/CodeGen/Passes.cpp
new file mode 100644
index 0000000..87510e4
--- /dev/null
+++ b/lib/CodeGen/Passes.cpp
@@ -0,0 +1,56 @@
+//===-- Passes.cpp - Target independent code generation passes ------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file was developed by the LLVM research group and is distributed under
+// the University of Illinois Open Source License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines interfaces to access the target independent code
+// generation passes provided by the LLVM backend.
+//
+//===---------------------------------------------------------------------===//
+
+#include "llvm/CodeGen/RegAllocRegistry.h"
+#include "llvm/CodeGen/Passes.h"
+
+using namespace llvm;
+
+//===---------------------------------------------------------------------===//
+///
+/// RegisterRegAlloc class - Track the registration of register allocators.
+///
+//===---------------------------------------------------------------------===//
+MachinePassRegistry RegisterRegAlloc::Registry;
+
+
+//===---------------------------------------------------------------------===//
+///
+/// RegAlloc command line options.
+///
+//===---------------------------------------------------------------------===//
+namespace {
+ cl::opt<RegisterRegAlloc::FunctionPassCtor, false,
+ RegisterPassParser<RegisterRegAlloc> >
+ RegAlloc("regalloc",
+ cl::init(&createLinearScanRegisterAllocator),
+ cl::desc("Register allocator to use: (default = linearscan)"));
+}
+
+
+//===---------------------------------------------------------------------===//
+///
+/// createRegisterAllocator - choose the appropriate register allocator.
+///
+//===---------------------------------------------------------------------===//
+FunctionPass *llvm::createRegisterAllocator() {
+ RegisterRegAlloc::FunctionPassCtor Ctor = RegisterRegAlloc::getDefault();
+
+ if (!Ctor) {
+ Ctor = RegAlloc;
+ RegisterRegAlloc::setDefault(RegAlloc);
+ }
+
+ return Ctor();
+}
diff --git a/lib/CodeGen/PhysRegTracker.h b/lib/CodeGen/PhysRegTracker.h
new file mode 100644
index 0000000..f5a2402
--- /dev/null
+++ b/lib/CodeGen/PhysRegTracker.h
@@ -0,0 +1,73 @@
+//===-- llvm/CodeGen/PhysRegTracker.h - Physical Register Tracker -*- C++ -*-=//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file was developed by the LLVM research group and is distributed under
+// the University of Illinois Open Source License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements a physical register tracker. The tracker
+// tracks physical register usage through addRegUse and
+// delRegUse. isRegAvail checks if a physical register is available or
+// not taking into consideration register aliases.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CODEGEN_PHYSREGTRACKER_H
+#define LLVM_CODEGEN_PHYSREGTRACKER_H
+
+#include "llvm/Target/MRegisterInfo.h"
+
+namespace llvm {
+
+ class PhysRegTracker {
+ const MRegisterInfo* mri_;
+ std::vector<unsigned> regUse_;
+
+ public:
+ PhysRegTracker(const MRegisterInfo& mri)
+ : mri_(&mri),
+ regUse_(mri_->getNumRegs(), 0) {
+ }
+
+ PhysRegTracker(const PhysRegTracker& rhs)
+ : mri_(rhs.mri_),
+ regUse_(rhs.regUse_) {
+ }
+
+ const PhysRegTracker& operator=(const PhysRegTracker& rhs) {
+ mri_ = rhs.mri_;
+ regUse_ = rhs.regUse_;
+ return *this;
+ }
+
+ void addRegUse(unsigned physReg) {
+ assert(MRegisterInfo::isPhysicalRegister(physReg) &&
+ "should be physical register!");
+ ++regUse_[physReg];
+ for (const unsigned* as = mri_->getAliasSet(physReg); *as; ++as)
+ ++regUse_[*as];
+ }
+
+ void delRegUse(unsigned physReg) {
+ assert(MRegisterInfo::isPhysicalRegister(physReg) &&
+ "should be physical register!");
+ assert(regUse_[physReg] != 0);
+ --regUse_[physReg];
+ for (const unsigned* as = mri_->getAliasSet(physReg); *as; ++as) {
+ assert(regUse_[*as] != 0);
+ --regUse_[*as];
+ }
+ }
+
+ bool isRegAvail(unsigned physReg) const {
+ assert(MRegisterInfo::isPhysicalRegister(physReg) &&
+ "should be physical register!");
+ return regUse_[physReg] == 0;
+ }
+ };
+
+} // End llvm namespace
+
+#endif
diff --git a/lib/CodeGen/PostRASchedulerList.cpp b/lib/CodeGen/PostRASchedulerList.cpp
new file mode 100644
index 0000000..06f47c7
--- /dev/null
+++ b/lib/CodeGen/PostRASchedulerList.cpp
@@ -0,0 +1,81 @@
+//===----- SchedulePostRAList.cpp - list scheduler ------------------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file was developed by Dale Johannesen and is distributed under the
+// University of Illinois Open Source License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This implements a top-down list scheduler, using standard algorithms.
+// The basic approach uses a priority queue of available nodes to schedule.
+// One at a time, nodes are taken from the priority queue (thus in priority
+// order), checked for legality to schedule, and emitted if legal.
+//
+// Nodes may not be legal to schedule either due to structural hazards (e.g.
+// pipeline or resource constraints) or because an input to the instruction has
+// not completed execution.
+//
+//===----------------------------------------------------------------------===//
+
+#define DEBUG_TYPE "post-RA-sched"
+#include "llvm/CodeGen/Passes.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/Support/Debug.h"
+//#include "llvm/ADT/Statistic.h"
+//#include <climits>
+//#include <queue>
+#include "llvm/Support/CommandLine.h"
+using namespace llvm;
+
+namespace {
+ bool NoPostRAScheduling;
+
+ // When this works it will be on by default.
+ cl::opt<bool, true>
+ DisablePostRAScheduler("disable-post-RA-scheduler",
+ cl::desc("Disable scheduling after register allocation"),
+ cl::location(NoPostRAScheduling),
+ cl::init(true));
+
+ class VISIBILITY_HIDDEN SchedulePostRATDList : public MachineFunctionPass {
+ public:
+ static char ID;
+ SchedulePostRATDList() : MachineFunctionPass((intptr_t)&ID) {}
+ private:
+ MachineFunction *MF;
+ const TargetMachine *TM;
+ public:
+ const char *getPassName() const {
+ return "Post RA top-down list latency scheduler (STUB)";
+ }
+
+ bool runOnMachineFunction(MachineFunction &Fn);
+ };
+ char SchedulePostRATDList::ID = 0;
+}
+
+bool SchedulePostRATDList::runOnMachineFunction(MachineFunction &Fn) {
+ if (NoPostRAScheduling)
+ return true;
+
+ DOUT << "SchedulePostRATDList\n";
+ MF = &Fn;
+ TM = &MF->getTarget();
+
+ // Loop over all of the basic blocks
+ for (MachineFunction::iterator MBB = Fn.begin(), MBBe = Fn.end();
+ MBB != MBBe; ++MBB)
+ ;
+
+ return true;
+}
+
+
+//===----------------------------------------------------------------------===//
+// Public Constructor Functions
+//===----------------------------------------------------------------------===//
+
+FunctionPass *llvm::createPostRAScheduler() {
+ return new SchedulePostRATDList();
+}
diff --git a/lib/CodeGen/PrologEpilogInserter.cpp b/lib/CodeGen/PrologEpilogInserter.cpp
new file mode 100644
index 0000000..882c6a7
--- /dev/null
+++ b/lib/CodeGen/PrologEpilogInserter.cpp
@@ -0,0 +1,549 @@
+//===-- PrologEpilogInserter.cpp - Insert Prolog/Epilog code in function --===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file was developed by the LLVM research group and is distributed under
+// the University of Illinois Open Source License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This pass is responsible for finalizing the functions frame layout, saving
+// callee saved registers, and for emitting prolog & epilog code for the
+// function.
+//
+// This pass must be run after register allocation. After this pass is
+// executed, it is illegal to construct MO_FrameIndex operands.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/CodeGen/Passes.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/MachineInstr.h"
+#include "llvm/CodeGen/MachineFrameInfo.h"
+#include "llvm/CodeGen/RegisterScavenging.h"
+#include "llvm/Target/TargetMachine.h"
+#include "llvm/Target/MRegisterInfo.h"
+#include "llvm/Target/TargetFrameInfo.h"
+#include "llvm/Target/TargetInstrInfo.h"
+#include "llvm/Support/Compiler.h"
+#include "llvm/ADT/STLExtras.h"
+#include <climits>
+using namespace llvm;
+
+namespace {
+ struct VISIBILITY_HIDDEN PEI : public MachineFunctionPass {
+ static char ID;
+ PEI() : MachineFunctionPass((intptr_t)&ID) {}
+
+ const char *getPassName() const {
+ return "Prolog/Epilog Insertion & Frame Finalization";
+ }
+
+ /// runOnMachineFunction - Insert prolog/epilog code and replace abstract
+ /// frame indexes with appropriate references.
+ ///
+ bool runOnMachineFunction(MachineFunction &Fn) {
+ const MRegisterInfo *MRI = Fn.getTarget().getRegisterInfo();
+ RS = MRI->requiresRegisterScavenging(Fn) ? new RegScavenger() : NULL;
+
+ // Get MachineModuleInfo so that we can track the construction of the
+ // frame.
+ if (MachineModuleInfo *MMI = getAnalysisToUpdate<MachineModuleInfo>()) {
+ Fn.getFrameInfo()->setMachineModuleInfo(MMI);
+ }
+
+ // Allow the target machine to make some adjustments to the function
+ // e.g. UsedPhysRegs before calculateCalleeSavedRegisters.
+ MRI->processFunctionBeforeCalleeSavedScan(Fn, RS);
+
+ // Scan the function for modified callee saved registers and insert spill
+ // code for any callee saved registers that are modified. Also calculate
+ // the MaxCallFrameSize and HasCalls variables for the function's frame
+ // information and eliminates call frame pseudo instructions.
+ calculateCalleeSavedRegisters(Fn);
+
+ // Add the code to save and restore the callee saved registers
+ saveCalleeSavedRegisters(Fn);
+
+ // Allow the target machine to make final modifications to the function
+ // before the frame layout is finalized.
+ Fn.getTarget().getRegisterInfo()->processFunctionBeforeFrameFinalized(Fn);
+
+ // Calculate actual frame offsets for all of the abstract stack objects...
+ calculateFrameObjectOffsets(Fn);
+
+ // Add prolog and epilog code to the function. This function is required
+ // to align the stack frame as necessary for any stack variables or
+ // called functions. Because of this, calculateCalleeSavedRegisters
+ // must be called before this function in order to set the HasCalls
+ // and MaxCallFrameSize variables.
+ insertPrologEpilogCode(Fn);
+
+ // Replace all MO_FrameIndex operands with physical register references
+ // and actual offsets.
+ //
+ replaceFrameIndices(Fn);
+
+ delete RS;
+ return true;
+ }
+
+ private:
+ RegScavenger *RS;
+
+ // MinCSFrameIndex, MaxCSFrameIndex - Keeps the range of callee saved
+ // stack frame indexes.
+ unsigned MinCSFrameIndex, MaxCSFrameIndex;
+
+ void calculateCalleeSavedRegisters(MachineFunction &Fn);
+ void saveCalleeSavedRegisters(MachineFunction &Fn);
+ void calculateFrameObjectOffsets(MachineFunction &Fn);
+ void replaceFrameIndices(MachineFunction &Fn);
+ void insertPrologEpilogCode(MachineFunction &Fn);
+ };
+ char PEI::ID = 0;
+}
+
+
+/// createPrologEpilogCodeInserter - This function returns a pass that inserts
+/// prolog and epilog code, and eliminates abstract frame references.
+///
+FunctionPass *llvm::createPrologEpilogCodeInserter() { return new PEI(); }
+
+
+/// calculateCalleeSavedRegisters - Scan the function for modified callee saved
+/// registers. Also calculate the MaxCallFrameSize and HasCalls variables for
+/// the function's frame information and eliminates call frame pseudo
+/// instructions.
+///
+void PEI::calculateCalleeSavedRegisters(MachineFunction &Fn) {
+ const MRegisterInfo *RegInfo = Fn.getTarget().getRegisterInfo();
+ const TargetFrameInfo *TFI = Fn.getTarget().getFrameInfo();
+
+ // Get the callee saved register list...
+ const unsigned *CSRegs = RegInfo->getCalleeSavedRegs(&Fn);
+
+ // Get the function call frame set-up and tear-down instruction opcode
+ int FrameSetupOpcode = RegInfo->getCallFrameSetupOpcode();
+ int FrameDestroyOpcode = RegInfo->getCallFrameDestroyOpcode();
+
+ // These are used to keep track the callee-save area. Initialize them.
+ MinCSFrameIndex = INT_MAX;
+ MaxCSFrameIndex = 0;
+
+ // Early exit for targets which have no callee saved registers and no call
+ // frame setup/destroy pseudo instructions.
+ if ((CSRegs == 0 || CSRegs[0] == 0) &&
+ FrameSetupOpcode == -1 && FrameDestroyOpcode == -1)
+ return;
+
+ unsigned MaxCallFrameSize = 0;
+ bool HasCalls = false;
+
+ std::vector<MachineBasicBlock::iterator> FrameSDOps;
+ for (MachineFunction::iterator BB = Fn.begin(), E = Fn.end(); BB != E; ++BB)
+ for (MachineBasicBlock::iterator I = BB->begin(); I != BB->end(); ++I)
+ if (I->getOpcode() == FrameSetupOpcode ||
+ I->getOpcode() == FrameDestroyOpcode) {
+ assert(I->getNumOperands() >= 1 && "Call Frame Setup/Destroy Pseudo"
+ " instructions should have a single immediate argument!");
+ unsigned Size = I->getOperand(0).getImmedValue();
+ if (Size > MaxCallFrameSize) MaxCallFrameSize = Size;
+ HasCalls = true;
+ FrameSDOps.push_back(I);
+ }
+
+ MachineFrameInfo *FFI = Fn.getFrameInfo();
+ FFI->setHasCalls(HasCalls);
+ FFI->setMaxCallFrameSize(MaxCallFrameSize);
+
+ for (unsigned i = 0, e = FrameSDOps.size(); i != e; ++i) {
+ MachineBasicBlock::iterator I = FrameSDOps[i];
+ // If call frames are not being included as part of the stack frame,
+ // and there is no dynamic allocation (therefore referencing frame slots
+ // off sp), leave the pseudo ops alone. We'll eliminate them later.
+ if (RegInfo->hasReservedCallFrame(Fn) || RegInfo->hasFP(Fn))
+ RegInfo->eliminateCallFramePseudoInstr(Fn, *I->getParent(), I);
+ }
+
+ // Now figure out which *callee saved* registers are modified by the current
+ // function, thus needing to be saved and restored in the prolog/epilog.
+ //
+ const TargetRegisterClass* const *CSRegClasses =
+ RegInfo->getCalleeSavedRegClasses(&Fn);
+ std::vector<CalleeSavedInfo> CSI;
+ for (unsigned i = 0; CSRegs[i]; ++i) {
+ unsigned Reg = CSRegs[i];
+ if (Fn.isPhysRegUsed(Reg)) {
+ // If the reg is modified, save it!
+ CSI.push_back(CalleeSavedInfo(Reg, CSRegClasses[i]));
+ } else {
+ for (const unsigned *AliasSet = RegInfo->getAliasSet(Reg);
+ *AliasSet; ++AliasSet) { // Check alias registers too.
+ if (Fn.isPhysRegUsed(*AliasSet)) {
+ CSI.push_back(CalleeSavedInfo(Reg, CSRegClasses[i]));
+ break;
+ }
+ }
+ }
+ }
+
+ if (CSI.empty())
+ return; // Early exit if no callee saved registers are modified!
+
+ unsigned NumFixedSpillSlots;
+ const std::pair<unsigned,int> *FixedSpillSlots =
+ TFI->getCalleeSavedSpillSlots(NumFixedSpillSlots);
+
+ // Now that we know which registers need to be saved and restored, allocate
+ // stack slots for them.
+ for (unsigned i = 0, e = CSI.size(); i != e; ++i) {
+ unsigned Reg = CSI[i].getReg();
+ const TargetRegisterClass *RC = CSI[i].getRegClass();
+
+ // Check to see if this physreg must be spilled to a particular stack slot
+ // on this target.
+ const std::pair<unsigned,int> *FixedSlot = FixedSpillSlots;
+ while (FixedSlot != FixedSpillSlots+NumFixedSpillSlots &&
+ FixedSlot->first != Reg)
+ ++FixedSlot;
+
+ int FrameIdx;
+ if (FixedSlot == FixedSpillSlots+NumFixedSpillSlots) {
+ // Nope, just spill it anywhere convenient.
+ unsigned Align = RC->getAlignment();
+ unsigned StackAlign = TFI->getStackAlignment();
+ // We may not be able to sastify the desired alignment specification of
+ // the TargetRegisterClass if the stack alignment is smaller. Use the min.
+ Align = std::min(Align, StackAlign);
+ FrameIdx = FFI->CreateStackObject(RC->getSize(), Align);
+ if ((unsigned)FrameIdx < MinCSFrameIndex) MinCSFrameIndex = FrameIdx;
+ if ((unsigned)FrameIdx > MaxCSFrameIndex) MaxCSFrameIndex = FrameIdx;
+ } else {
+ // Spill it to the stack where we must.
+ FrameIdx = FFI->CreateFixedObject(RC->getSize(), FixedSlot->second);
+ }
+ CSI[i].setFrameIdx(FrameIdx);
+ }
+
+ FFI->setCalleeSavedInfo(CSI);
+}
+
+/// saveCalleeSavedRegisters - Insert spill code for any callee saved registers
+/// that are modified in the function.
+///
+void PEI::saveCalleeSavedRegisters(MachineFunction &Fn) {
+ // Get callee saved register information.
+ MachineFrameInfo *FFI = Fn.getFrameInfo();
+ const std::vector<CalleeSavedInfo> &CSI = FFI->getCalleeSavedInfo();
+
+ // Early exit if no callee saved registers are modified!
+ if (CSI.empty())
+ return;
+
+ const MRegisterInfo *RegInfo = Fn.getTarget().getRegisterInfo();
+
+ // Now that we have a stack slot for each register to be saved, insert spill
+ // code into the entry block.
+ MachineBasicBlock *MBB = Fn.begin();
+ MachineBasicBlock::iterator I = MBB->begin();
+ if (!RegInfo->spillCalleeSavedRegisters(*MBB, I, CSI)) {
+ for (unsigned i = 0, e = CSI.size(); i != e; ++i) {
+ // Add the callee-saved register as live-in. It's killed at the spill.
+ MBB->addLiveIn(CSI[i].getReg());
+
+ // Insert the spill to the stack frame.
+ RegInfo->storeRegToStackSlot(*MBB, I, CSI[i].getReg(),
+ CSI[i].getFrameIdx(), CSI[i].getRegClass());
+ }
+ }
+
+ // Add code to restore the callee-save registers in each exiting block.
+ const TargetInstrInfo &TII = *Fn.getTarget().getInstrInfo();
+ for (MachineFunction::iterator FI = Fn.begin(), E = Fn.end(); FI != E; ++FI)
+ // If last instruction is a return instruction, add an epilogue.
+ if (!FI->empty() && TII.isReturn(FI->back().getOpcode())) {
+ MBB = FI;
+ I = MBB->end(); --I;
+
+ // Skip over all terminator instructions, which are part of the return
+ // sequence.
+ MachineBasicBlock::iterator I2 = I;
+ while (I2 != MBB->begin() && TII.isTerminatorInstr((--I2)->getOpcode()))
+ I = I2;
+
+ bool AtStart = I == MBB->begin();
+ MachineBasicBlock::iterator BeforeI = I;
+ if (!AtStart)
+ --BeforeI;
+
+ // Restore all registers immediately before the return and any terminators
+ // that preceed it.
+ if (!RegInfo->restoreCalleeSavedRegisters(*MBB, I, CSI)) {
+ for (unsigned i = 0, e = CSI.size(); i != e; ++i) {
+ RegInfo->loadRegFromStackSlot(*MBB, I, CSI[i].getReg(),
+ CSI[i].getFrameIdx(),
+ CSI[i].getRegClass());
+ assert(I != MBB->begin() &&
+ "loadRegFromStackSlot didn't insert any code!");
+ // Insert in reverse order. loadRegFromStackSlot can insert multiple
+ // instructions.
+ if (AtStart)
+ I = MBB->begin();
+ else {
+ I = BeforeI;
+ ++I;
+ }
+ }
+ }
+ }
+}
+
+
+/// calculateFrameObjectOffsets - Calculate actual frame offsets for all of the
+/// abstract stack objects.
+///
+void PEI::calculateFrameObjectOffsets(MachineFunction &Fn) {
+ const TargetFrameInfo &TFI = *Fn.getTarget().getFrameInfo();
+
+ bool StackGrowsDown =
+ TFI.getStackGrowthDirection() == TargetFrameInfo::StackGrowsDown;
+
+ // Loop over all of the stack objects, assigning sequential addresses...
+ MachineFrameInfo *FFI = Fn.getFrameInfo();
+
+ unsigned MaxAlign = 0;
+
+ // Start at the beginning of the local area.
+ // The Offset is the distance from the stack top in the direction
+ // of stack growth -- so it's always positive.
+ int64_t Offset = TFI.getOffsetOfLocalArea();
+ if (StackGrowsDown)
+ Offset = -Offset;
+ assert(Offset >= 0
+ && "Local area offset should be in direction of stack growth");
+
+ // If there are fixed sized objects that are preallocated in the local area,
+ // non-fixed objects can't be allocated right at the start of local area.
+ // We currently don't support filling in holes in between fixed sized objects,
+ // so we adjust 'Offset' to point to the end of last fixed sized
+ // preallocated object.
+ for (int i = FFI->getObjectIndexBegin(); i != 0; ++i) {
+ int64_t FixedOff;
+ if (StackGrowsDown) {
+ // The maximum distance from the stack pointer is at lower address of
+ // the object -- which is given by offset. For down growing stack
+ // the offset is negative, so we negate the offset to get the distance.
+ FixedOff = -FFI->getObjectOffset(i);
+ } else {
+ // The maximum distance from the start pointer is at the upper
+ // address of the object.
+ FixedOff = FFI->getObjectOffset(i) + FFI->getObjectSize(i);
+ }
+ if (FixedOff > Offset) Offset = FixedOff;
+ }
+
+ // First assign frame offsets to stack objects that are used to spill
+ // callee saved registers.
+ if (StackGrowsDown) {
+ for (unsigned i = MinCSFrameIndex; i <= MaxCSFrameIndex; ++i) {
+ // If stack grows down, we need to add size of find the lowest
+ // address of the object.
+ Offset += FFI->getObjectSize(i);
+
+ unsigned Align = FFI->getObjectAlignment(i);
+ // If the alignment of this object is greater than that of the stack, then
+ // increase the stack alignment to match.
+ MaxAlign = std::max(MaxAlign, Align);
+ // Adjust to alignment boundary
+ Offset = (Offset+Align-1)/Align*Align;
+
+ FFI->setObjectOffset(i, -Offset); // Set the computed offset
+ }
+ } else {
+ for (unsigned i = MaxCSFrameIndex; i >= MinCSFrameIndex; --i) {
+ unsigned Align = FFI->getObjectAlignment(i);
+ // If the alignment of this object is greater than that of the stack, then
+ // increase the stack alignment to match.
+ MaxAlign = std::max(MaxAlign, Align);
+ // Adjust to alignment boundary
+ Offset = (Offset+Align-1)/Align*Align;
+
+ FFI->setObjectOffset(i, Offset);
+ Offset += FFI->getObjectSize(i);
+ }
+ }
+
+ // Make sure the special register scavenging spill slot is closest to the
+ // frame pointer if a frame pointer is required.
+ const MRegisterInfo *RegInfo = Fn.getTarget().getRegisterInfo();
+ if (RS && RegInfo->hasFP(Fn)) {
+ int SFI = RS->getScavengingFrameIndex();
+ if (SFI >= 0) {
+ // If stack grows down, we need to add size of the lowest
+ // address of the object.
+ if (StackGrowsDown)
+ Offset += FFI->getObjectSize(SFI);
+
+ unsigned Align = FFI->getObjectAlignment(SFI);
+ // Adjust to alignment boundary
+ Offset = (Offset+Align-1)/Align*Align;
+
+ if (StackGrowsDown) {
+ FFI->setObjectOffset(SFI, -Offset); // Set the computed offset
+ } else {
+ FFI->setObjectOffset(SFI, Offset);
+ Offset += FFI->getObjectSize(SFI);
+ }
+ }
+ }
+
+ // Then assign frame offsets to stack objects that are not used to spill
+ // callee saved registers.
+ for (unsigned i = 0, e = FFI->getObjectIndexEnd(); i != e; ++i) {
+ if (i >= MinCSFrameIndex && i <= MaxCSFrameIndex)
+ continue;
+ if (RS && (int)i == RS->getScavengingFrameIndex())
+ continue;
+
+ // If stack grows down, we need to add size of find the lowest
+ // address of the object.
+ if (StackGrowsDown)
+ Offset += FFI->getObjectSize(i);
+
+ unsigned Align = FFI->getObjectAlignment(i);
+ // If the alignment of this object is greater than that of the stack, then
+ // increase the stack alignment to match.
+ MaxAlign = std::max(MaxAlign, Align);
+ // Adjust to alignment boundary
+ Offset = (Offset+Align-1)/Align*Align;
+
+ if (StackGrowsDown) {
+ FFI->setObjectOffset(i, -Offset); // Set the computed offset
+ } else {
+ FFI->setObjectOffset(i, Offset);
+ Offset += FFI->getObjectSize(i);
+ }
+ }
+
+ // Make sure the special register scavenging spill slot is closest to the
+ // stack pointer.
+ if (RS && !RegInfo->hasFP(Fn)) {
+ int SFI = RS->getScavengingFrameIndex();
+ if (SFI >= 0) {
+ // If stack grows down, we need to add size of find the lowest
+ // address of the object.
+ if (StackGrowsDown)
+ Offset += FFI->getObjectSize(SFI);
+
+ unsigned Align = FFI->getObjectAlignment(SFI);
+ // Adjust to alignment boundary
+ Offset = (Offset+Align-1)/Align*Align;
+
+ if (StackGrowsDown) {
+ FFI->setObjectOffset(SFI, -Offset); // Set the computed offset
+ } else {
+ FFI->setObjectOffset(SFI, Offset);
+ Offset += FFI->getObjectSize(SFI);
+ }
+ }
+ }
+
+ // Round up the size to a multiple of the alignment, but only if there are
+ // calls or alloca's in the function. This ensures that any calls to
+ // subroutines have their stack frames suitable aligned.
+ if (!RegInfo->targetHandlesStackFrameRounding() &&
+ (FFI->hasCalls() || FFI->hasVarSizedObjects())) {
+ // If we have reserved argument space for call sites in the function
+ // immediately on entry to the current function, count it as part of the
+ // overall stack size.
+ if (RegInfo->hasReservedCallFrame(Fn))
+ Offset += FFI->getMaxCallFrameSize();
+
+ unsigned AlignMask = TFI.getStackAlignment() - 1;
+ Offset = (Offset + AlignMask) & ~uint64_t(AlignMask);
+ }
+
+ // Update frame info to pretend that this is part of the stack...
+ FFI->setStackSize(Offset+TFI.getOffsetOfLocalArea());
+
+ // Remember the required stack alignment in case targets need it to perform
+ // dynamic stack alignment.
+ assert(FFI->getMaxAlignment() == MaxAlign &&
+ "Stack alignment calculation broken!");
+}
+
+
+/// insertPrologEpilogCode - Scan the function for modified callee saved
+/// registers, insert spill code for these callee saved registers, then add
+/// prolog and epilog code to the function.
+///
+void PEI::insertPrologEpilogCode(MachineFunction &Fn) {
+ // Add prologue to the function...
+ Fn.getTarget().getRegisterInfo()->emitPrologue(Fn);
+
+ // Add epilogue to restore the callee-save registers in each exiting block
+ const TargetInstrInfo &TII = *Fn.getTarget().getInstrInfo();
+ for (MachineFunction::iterator I = Fn.begin(), E = Fn.end(); I != E; ++I) {
+ // If last instruction is a return instruction, add an epilogue
+ if (!I->empty() && TII.isReturn(I->back().getOpcode()))
+ Fn.getTarget().getRegisterInfo()->emitEpilogue(Fn, *I);
+ }
+}
+
+
+/// replaceFrameIndices - Replace all MO_FrameIndex operands with physical
+/// register references and actual offsets.
+///
+void PEI::replaceFrameIndices(MachineFunction &Fn) {
+ if (!Fn.getFrameInfo()->hasStackObjects()) return; // Nothing to do?
+
+ const TargetMachine &TM = Fn.getTarget();
+ assert(TM.getRegisterInfo() && "TM::getRegisterInfo() must be implemented!");
+ const MRegisterInfo &MRI = *TM.getRegisterInfo();
+ const TargetFrameInfo *TFI = TM.getFrameInfo();
+ bool StackGrowsDown =
+ TFI->getStackGrowthDirection() == TargetFrameInfo::StackGrowsDown;
+ int FrameSetupOpcode = MRI.getCallFrameSetupOpcode();
+ int FrameDestroyOpcode = MRI.getCallFrameDestroyOpcode();
+
+ for (MachineFunction::iterator BB = Fn.begin(), E = Fn.end(); BB != E; ++BB) {
+ int SPAdj = 0; // SP offset due to call frame setup / destroy.
+ if (RS) RS->enterBasicBlock(BB);
+ for (MachineBasicBlock::iterator I = BB->begin(); I != BB->end(); ) {
+ MachineInstr *MI = I;
+
+ // Remember how much SP has been adjustment to create the call frame.
+ if (I->getOpcode() == FrameSetupOpcode ||
+ I->getOpcode() == FrameDestroyOpcode) {
+ int Size = I->getOperand(0).getImmedValue();
+ if ((!StackGrowsDown && I->getOpcode() == FrameSetupOpcode) ||
+ (StackGrowsDown && I->getOpcode() == FrameDestroyOpcode))
+ Size = -Size;
+ SPAdj += Size;
+ MachineBasicBlock::iterator PrevI = prior(I);
+ MRI.eliminateCallFramePseudoInstr(Fn, *BB, I);
+ // Visit the instructions created by eliminateCallFramePseudoInstr().
+ I = next(PrevI);
+ MI = NULL;
+ } else {
+ I++;
+ for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i)
+ if (MI->getOperand(i).isFrameIndex()) {
+ // If this instruction has a FrameIndex operand, we need to use that
+ // target machine register info object to eliminate it.
+ MRI.eliminateFrameIndex(MI, SPAdj, RS);
+
+ // Revisit the instruction in full. Some instructions (e.g. inline
+ // asm instructions) can have multiple frame indices.
+ --I;
+ MI = 0;
+ break;
+ }
+ }
+ // Update register states.
+ if (RS && MI) RS->forward(MI);
+ }
+ assert(SPAdj == 0 && "Unbalanced call frame setup / destroy pairs?");
+ }
+}
diff --git a/lib/CodeGen/README.txt b/lib/CodeGen/README.txt
new file mode 100644
index 0000000..4fff15c
--- /dev/null
+++ b/lib/CodeGen/README.txt
@@ -0,0 +1,145 @@
+//===---------------------------------------------------------------------===//
+
+Common register allocation / spilling problem:
+
+ mul lr, r4, lr
+ str lr, [sp, #+52]
+ ldr lr, [r1, #+32]
+ sxth r3, r3
+ ldr r4, [sp, #+52]
+ mla r4, r3, lr, r4
+
+can be:
+
+ mul lr, r4, lr
+ mov r4, lr
+ str lr, [sp, #+52]
+ ldr lr, [r1, #+32]
+ sxth r3, r3
+ mla r4, r3, lr, r4
+
+and then "merge" mul and mov:
+
+ mul r4, r4, lr
+ str lr, [sp, #+52]
+ ldr lr, [r1, #+32]
+ sxth r3, r3
+ mla r4, r3, lr, r4
+
+It also increase the likelyhood the store may become dead.
+
+//===---------------------------------------------------------------------===//
+
+I think we should have a "hasSideEffects" flag (which is automatically set for
+stuff that "isLoad" "isCall" etc), and the remat pass should eventually be able
+to remat any instruction that has no side effects, if it can handle it and if
+profitable.
+
+For now, I'd suggest having the remat stuff work like this:
+
+1. I need to spill/reload this thing.
+2. Check to see if it has side effects.
+3. Check to see if it is simple enough: e.g. it only has one register
+destination and no register input.
+4. If so, clone the instruction, do the xform, etc.
+
+Advantages of this are:
+
+1. the .td file describes the behavior of the instructions, not the way the
+ algorithm should work.
+2. as remat gets smarter in the future, we shouldn't have to be changing the .td
+ files.
+3. it is easier to explain what the flag means in the .td file, because you
+ don't have to pull in the explanation of how the current remat algo works.
+
+Some potential added complexities:
+
+1. Some instructions have to be glued to it's predecessor or successor. All of
+ the PC relative instructions and condition code setting instruction. We could
+ mark them as hasSideEffects, but that's not quite right. PC relative loads
+ from constantpools can be remat'ed, for example. But it requires more than
+ just cloning the instruction. Some instructions can be remat'ed but it
+ expands to more than one instruction. But allocator will have to make a
+ decision.
+
+4. As stated in 3, not as simple as cloning in some cases. The target will have
+ to decide how to remat it. For example, an ARM 2-piece constant generation
+ instruction is remat'ed as a load from constantpool.
+
+//===---------------------------------------------------------------------===//
+
+bb27 ...
+ ...
+ %reg1037 = ADDri %reg1039, 1
+ %reg1038 = ADDrs %reg1032, %reg1039, %NOREG, 10
+ Successors according to CFG: 0x8b03bf0 (#5)
+
+bb76 (0x8b03bf0, LLVM BB @0x8b032d0, ID#5):
+ Predecessors according to CFG: 0x8b0c5f0 (#3) 0x8b0a7c0 (#4)
+ %reg1039 = PHI %reg1070, mbb<bb76.outer,0x8b0c5f0>, %reg1037, mbb<bb27,0x8b0a7c0>
+
+Note ADDri is not a two-address instruction. However, its result %reg1037 is an
+operand of the PHI node in bb76 and its operand %reg1039 is the result of the
+PHI node. We should treat it as a two-address code and make sure the ADDri is
+scheduled after any node that reads %reg1039.
+
+//===---------------------------------------------------------------------===//
+
+Use local info (i.e. register scavenger) to assign it a free register to allow
+reuse:
+ ldr r3, [sp, #+4]
+ add r3, r3, #3
+ ldr r2, [sp, #+8]
+ add r2, r2, #2
+ ldr r1, [sp, #+4] <==
+ add r1, r1, #1
+ ldr r0, [sp, #+4]
+ add r0, r0, #2
+
+//===---------------------------------------------------------------------===//
+
+LLVM aggressively lift CSE out of loop. Sometimes this can be negative side-
+effects:
+
+R1 = X + 4
+R2 = X + 7
+R3 = X + 15
+
+loop:
+load [i + R1]
+...
+load [i + R2]
+...
+load [i + R3]
+
+Suppose there is high register pressure, R1, R2, R3, can be spilled. We need
+to implement proper re-materialization to handle this:
+
+R1 = X + 4
+R2 = X + 7
+R3 = X + 15
+
+loop:
+R1 = X + 4 @ re-materialized
+load [i + R1]
+...
+R2 = X + 7 @ re-materialized
+load [i + R2]
+...
+R3 = X + 15 @ re-materialized
+load [i + R3]
+
+Furthermore, with re-association, we can enable sharing:
+
+R1 = X + 4
+R2 = X + 7
+R3 = X + 15
+
+loop:
+T = i + X
+load [T + 4]
+...
+load [T + 7]
+...
+load [T + 15]
+//===---------------------------------------------------------------------===//
diff --git a/lib/CodeGen/RegAllocBigBlock.cpp b/lib/CodeGen/RegAllocBigBlock.cpp
new file mode 100644
index 0000000..c7f23f5
--- /dev/null
+++ b/lib/CodeGen/RegAllocBigBlock.cpp
@@ -0,0 +1,893 @@
+//===- RegAllocBigBlock.cpp - A register allocator for large basic blocks -===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file was developed by Duraid Madina and is distributed under the
+// University of Illinois Open Source License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements the RABigBlock class
+//
+//===----------------------------------------------------------------------===//
+
+// This register allocator is derived from RegAllocLocal.cpp. Like it, this
+// allocator works on one basic block at a time, oblivious to others.
+// However, the algorithm used here is suited for long blocks of
+// instructions - registers are spilled by greedily choosing those holding
+// values that will not be needed for the longest amount of time. This works
+// particularly well for blocks with 10 or more times as many instructions
+// as machine registers, but can be used for general code.
+//
+//===----------------------------------------------------------------------===//
+//
+// TODO: - automagically invoke linearscan for (groups of) small BBs?
+// - break ties when picking regs? (probably not worth it in a
+// JIT context)
+//
+//===----------------------------------------------------------------------===//
+
+#define DEBUG_TYPE "regalloc"
+#include "llvm/BasicBlock.h"
+#include "llvm/CodeGen/Passes.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/MachineInstr.h"
+#include "llvm/CodeGen/SSARegMap.h"
+#include "llvm/CodeGen/MachineFrameInfo.h"
+#include "llvm/CodeGen/LiveVariables.h"
+#include "llvm/CodeGen/RegAllocRegistry.h"
+#include "llvm/Target/TargetInstrInfo.h"
+#include "llvm/Target/TargetMachine.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/Compiler.h"
+#include "llvm/ADT/IndexedMap.h"
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/SmallPtrSet.h"
+#include "llvm/ADT/Statistic.h"
+#include <algorithm>
+using namespace llvm;
+
+STATISTIC(NumStores, "Number of stores added");
+STATISTIC(NumLoads , "Number of loads added");
+STATISTIC(NumFolded, "Number of loads/stores folded into instructions");
+
+namespace {
+ static RegisterRegAlloc
+ bigBlockRegAlloc("bigblock", " Big-block register allocator",
+ createBigBlockRegisterAllocator);
+
+/// VRegKeyInfo - Defines magic values required to use VirtRegs as DenseMap
+/// keys.
+ struct VRegKeyInfo {
+ static inline unsigned getEmptyKey() { return -1U; }
+ static inline unsigned getTombstoneKey() { return -2U; }
+ static unsigned getHashValue(const unsigned &Key) { return Key; }
+ };
+
+
+/// This register allocator is derived from RegAllocLocal.cpp. Like it, this
+/// allocator works on one basic block at a time, oblivious to others.
+/// However, the algorithm used here is suited for long blocks of
+/// instructions - registers are spilled by greedily choosing those holding
+/// values that will not be needed for the longest amount of time. This works
+/// particularly well for blocks with 10 or more times as many instructions
+/// as machine registers, but can be used for general code.
+///
+/// TODO: - automagically invoke linearscan for (groups of) small BBs?
+/// - break ties when picking regs? (probably not worth it in a
+/// JIT context)
+///
+ class VISIBILITY_HIDDEN RABigBlock : public MachineFunctionPass {
+ public:
+ static char ID;
+ RABigBlock() : MachineFunctionPass((intptr_t)&ID) {}
+ private:
+ /// TM - For getting at TargetMachine info
+ ///
+ const TargetMachine *TM;
+
+ /// MF - Our generic MachineFunction pointer
+ ///
+ MachineFunction *MF;
+
+ /// RegInfo - For dealing with machine register info (aliases, folds
+ /// etc)
+ const MRegisterInfo *RegInfo;
+
+ /// LV - Our generic LiveVariables pointer
+ ///
+ LiveVariables *LV;
+
+ typedef SmallVector<unsigned, 2> VRegTimes;
+
+ /// VRegReadTable - maps VRegs in a BB to the set of times they are read
+ ///
+ DenseMap<unsigned, VRegTimes*, VRegKeyInfo> VRegReadTable;
+
+ /// VRegReadIdx - keeps track of the "current time" in terms of
+ /// positions in VRegReadTable
+ DenseMap<unsigned, unsigned , VRegKeyInfo> VRegReadIdx;
+
+ /// StackSlotForVirtReg - Maps virtual regs to the frame index where these
+ /// values are spilled.
+ IndexedMap<unsigned, VirtReg2IndexFunctor> StackSlotForVirtReg;
+
+ /// Virt2PhysRegMap - This map contains entries for each virtual register
+ /// that is currently available in a physical register.
+ IndexedMap<unsigned, VirtReg2IndexFunctor> Virt2PhysRegMap;
+
+ /// PhysRegsUsed - This array is effectively a map, containing entries for
+ /// each physical register that currently has a value (ie, it is in
+ /// Virt2PhysRegMap). The value mapped to is the virtual register
+ /// corresponding to the physical register (the inverse of the
+ /// Virt2PhysRegMap), or 0. The value is set to 0 if this register is pinned
+ /// because it is used by a future instruction, and to -2 if it is not
+ /// allocatable. If the entry for a physical register is -1, then the
+ /// physical register is "not in the map".
+ ///
+ std::vector<int> PhysRegsUsed;
+
+ /// VirtRegModified - This bitset contains information about which virtual
+ /// registers need to be spilled back to memory when their registers are
+ /// scavenged. If a virtual register has simply been rematerialized, there
+ /// is no reason to spill it to memory when we need the register back.
+ ///
+ std::vector<int> VirtRegModified;
+
+ /// MBBLastInsnTime - the number of the the last instruction in MBB
+ ///
+ int MBBLastInsnTime;
+
+ /// MBBCurTime - the number of the the instruction being currently processed
+ ///
+ int MBBCurTime;
+
+ unsigned &getVirt2PhysRegMapSlot(unsigned VirtReg) {
+ return Virt2PhysRegMap[VirtReg];
+ }
+
+ unsigned &getVirt2StackSlot(unsigned VirtReg) {
+ return StackSlotForVirtReg[VirtReg];
+ }
+
+ /// markVirtRegModified - Lets us flip bits in the VirtRegModified bitset
+ ///
+ void markVirtRegModified(unsigned Reg, bool Val = true) {
+ assert(MRegisterInfo::isVirtualRegister(Reg) && "Illegal VirtReg!");
+ Reg -= MRegisterInfo::FirstVirtualRegister;
+ if (VirtRegModified.size() <= Reg)
+ VirtRegModified.resize(Reg+1);
+ VirtRegModified[Reg] = Val;
+ }
+
+ /// isVirtRegModified - Lets us query the VirtRegModified bitset
+ ///
+ bool isVirtRegModified(unsigned Reg) const {
+ assert(MRegisterInfo::isVirtualRegister(Reg) && "Illegal VirtReg!");
+ assert(Reg - MRegisterInfo::FirstVirtualRegister < VirtRegModified.size()
+ && "Illegal virtual register!");
+ return VirtRegModified[Reg - MRegisterInfo::FirstVirtualRegister];
+ }
+
+ public:
+ /// getPassName - returns the BigBlock allocator's name
+ ///
+ virtual const char *getPassName() const {
+ return "BigBlock Register Allocator";
+ }
+
+ /// getAnalaysisUsage - declares the required analyses
+ ///
+ virtual void getAnalysisUsage(AnalysisUsage &AU) const {
+ AU.addRequired<LiveVariables>();
+ AU.addRequiredID(PHIEliminationID);
+ AU.addRequiredID(TwoAddressInstructionPassID);
+ MachineFunctionPass::getAnalysisUsage(AU);
+ }
+
+ private:
+ /// runOnMachineFunction - Register allocate the whole function
+ ///
+ bool runOnMachineFunction(MachineFunction &Fn);
+
+ /// AllocateBasicBlock - Register allocate the specified basic block.
+ ///
+ void AllocateBasicBlock(MachineBasicBlock &MBB);
+
+ /// FillVRegReadTable - Fill out the table of vreg read times given a BB
+ ///
+ void FillVRegReadTable(MachineBasicBlock &MBB);
+
+ /// areRegsEqual - This method returns true if the specified registers are
+ /// related to each other. To do this, it checks to see if they are equal
+ /// or if the first register is in the alias set of the second register.
+ ///
+ bool areRegsEqual(unsigned R1, unsigned R2) const {
+ if (R1 == R2) return true;
+ for (const unsigned *AliasSet = RegInfo->getAliasSet(R2);
+ *AliasSet; ++AliasSet) {
+ if (*AliasSet == R1) return true;
+ }
+ return false;
+ }
+
+ /// getStackSpaceFor - This returns the frame index of the specified virtual
+ /// register on the stack, allocating space if necessary.
+ int getStackSpaceFor(unsigned VirtReg, const TargetRegisterClass *RC);
+
+ /// removePhysReg - This method marks the specified physical register as no
+ /// longer being in use.
+ ///
+ void removePhysReg(unsigned PhysReg);
+
+ /// spillVirtReg - This method spills the value specified by PhysReg into
+ /// the virtual register slot specified by VirtReg. It then updates the RA
+ /// data structures to indicate the fact that PhysReg is now available.
+ ///
+ void spillVirtReg(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
+ unsigned VirtReg, unsigned PhysReg);
+
+ /// spillPhysReg - This method spills the specified physical register into
+ /// the virtual register slot associated with it. If OnlyVirtRegs is set to
+ /// true, then the request is ignored if the physical register does not
+ /// contain a virtual register.
+ ///
+ void spillPhysReg(MachineBasicBlock &MBB, MachineInstr *I,
+ unsigned PhysReg, bool OnlyVirtRegs = false);
+
+ /// assignVirtToPhysReg - This method updates local state so that we know
+ /// that PhysReg is the proper container for VirtReg now. The physical
+ /// register must not be used for anything else when this is called.
+ ///
+ void assignVirtToPhysReg(unsigned VirtReg, unsigned PhysReg);
+
+ /// isPhysRegAvailable - Return true if the specified physical register is
+ /// free and available for use. This also includes checking to see if
+ /// aliased registers are all free...
+ ///
+ bool isPhysRegAvailable(unsigned PhysReg) const;
+
+ /// getFreeReg - Look to see if there is a free register available in the
+ /// specified register class. If not, return 0.
+ ///
+ unsigned getFreeReg(const TargetRegisterClass *RC);
+
+ /// chooseReg - Pick a physical register to hold the specified
+ /// virtual register by choosing the one which will be read furthest
+ /// in the future.
+ ///
+ unsigned chooseReg(MachineBasicBlock &MBB, MachineInstr *MI,
+ unsigned VirtReg);
+
+ /// reloadVirtReg - This method transforms the specified specified virtual
+ /// register use to refer to a physical register. This method may do this
+ /// in one of several ways: if the register is available in a physical
+ /// register already, it uses that physical register. If the value is not
+ /// in a physical register, and if there are physical registers available,
+ /// it loads it into a register. If register pressure is high, and it is
+ /// possible, it tries to fold the load of the virtual register into the
+ /// instruction itself. It avoids doing this if register pressure is low to
+ /// improve the chance that subsequent instructions can use the reloaded
+ /// value. This method returns the modified instruction.
+ ///
+ MachineInstr *reloadVirtReg(MachineBasicBlock &MBB, MachineInstr *MI,
+ unsigned OpNum);
+
+ };
+ char RABigBlock::ID = 0;
+}
+
+/// getStackSpaceFor - This allocates space for the specified virtual register
+/// to be held on the stack.
+int RABigBlock::getStackSpaceFor(unsigned VirtReg, const TargetRegisterClass *RC) {
+ // Find the location Reg would belong...
+ int FrameIdx = getVirt2StackSlot(VirtReg);
+
+ if (FrameIdx)
+ return FrameIdx - 1; // Already has space allocated?
+
+ // Allocate a new stack object for this spill location...
+ FrameIdx = MF->getFrameInfo()->CreateStackObject(RC->getSize(),
+ RC->getAlignment());
+
+ // Assign the slot...
+ getVirt2StackSlot(VirtReg) = FrameIdx + 1;
+ return FrameIdx;
+}
+
+
+/// removePhysReg - This method marks the specified physical register as no
+/// longer being in use.
+///
+void RABigBlock::removePhysReg(unsigned PhysReg) {
+ PhysRegsUsed[PhysReg] = -1; // PhyReg no longer used
+}
+
+
+/// spillVirtReg - This method spills the value specified by PhysReg into the
+/// virtual register slot specified by VirtReg. It then updates the RA data
+/// structures to indicate the fact that PhysReg is now available.
+///
+void RABigBlock::spillVirtReg(MachineBasicBlock &MBB,
+ MachineBasicBlock::iterator I,
+ unsigned VirtReg, unsigned PhysReg) {
+ assert(VirtReg && "Spilling a physical register is illegal!"
+ " Must not have appropriate kill for the register or use exists beyond"
+ " the intended one.");
+ DOUT << " Spilling register " << RegInfo->getName(PhysReg)
+ << " containing %reg" << VirtReg;
+ if (!isVirtRegModified(VirtReg))
+ DOUT << " which has not been modified, so no store necessary!";
+
+ // Otherwise, there is a virtual register corresponding to this physical
+ // register. We only need to spill it into its stack slot if it has been
+ // modified.
+ if (isVirtRegModified(VirtReg)) {
+ const TargetRegisterClass *RC = MF->getSSARegMap()->getRegClass(VirtReg);
+ int FrameIndex = getStackSpaceFor(VirtReg, RC);
+ DOUT << " to stack slot #" << FrameIndex;
+ RegInfo->storeRegToStackSlot(MBB, I, PhysReg, FrameIndex, RC);
+ ++NumStores; // Update statistics
+ }
+
+ getVirt2PhysRegMapSlot(VirtReg) = 0; // VirtReg no longer available
+
+ DOUT << "\n";
+ removePhysReg(PhysReg);
+}
+
+
+/// spillPhysReg - This method spills the specified physical register into the
+/// virtual register slot associated with it. If OnlyVirtRegs is set to true,
+/// then the request is ignored if the physical register does not contain a
+/// virtual register.
+///
+void RABigBlock::spillPhysReg(MachineBasicBlock &MBB, MachineInstr *I,
+ unsigned PhysReg, bool OnlyVirtRegs) {
+ if (PhysRegsUsed[PhysReg] != -1) { // Only spill it if it's used!
+ assert(PhysRegsUsed[PhysReg] != -2 && "Non allocable reg used!");
+ if (PhysRegsUsed[PhysReg] || !OnlyVirtRegs)
+ spillVirtReg(MBB, I, PhysRegsUsed[PhysReg], PhysReg);
+ } else {
+ // If the selected register aliases any other registers, we must make
+ // sure that one of the aliases isn't alive.
+ for (const unsigned *AliasSet = RegInfo->getAliasSet(PhysReg);
+ *AliasSet; ++AliasSet)
+ if (PhysRegsUsed[*AliasSet] != -1 && // Spill aliased register.
+ PhysRegsUsed[*AliasSet] != -2) // If allocatable.
+ if (PhysRegsUsed[*AliasSet])
+ spillVirtReg(MBB, I, PhysRegsUsed[*AliasSet], *AliasSet);
+ }
+}
+
+
+/// assignVirtToPhysReg - This method updates local state so that we know
+/// that PhysReg is the proper container for VirtReg now. The physical
+/// register must not be used for anything else when this is called.
+///
+void RABigBlock::assignVirtToPhysReg(unsigned VirtReg, unsigned PhysReg) {
+ assert(PhysRegsUsed[PhysReg] == -1 && "Phys reg already assigned!");
+ // Update information to note the fact that this register was just used, and
+ // it holds VirtReg.
+ PhysRegsUsed[PhysReg] = VirtReg;
+ getVirt2PhysRegMapSlot(VirtReg) = PhysReg;
+}
+
+
+/// isPhysRegAvailable - Return true if the specified physical register is free
+/// and available for use. This also includes checking to see if aliased
+/// registers are all free...
+///
+bool RABigBlock::isPhysRegAvailable(unsigned PhysReg) const {
+ if (PhysRegsUsed[PhysReg] != -1) return false;
+
+ // If the selected register aliases any other allocated registers, it is
+ // not free!
+ for (const unsigned *AliasSet = RegInfo->getAliasSet(PhysReg);
+ *AliasSet; ++AliasSet)
+ if (PhysRegsUsed[*AliasSet] != -1) // Aliased register in use?
+ return false; // Can't use this reg then.
+ return true;
+}
+
+
+/// getFreeReg - Look to see if there is a free register available in the
+/// specified register class. If not, return 0.
+///
+unsigned RABigBlock::getFreeReg(const TargetRegisterClass *RC) {
+ // Get iterators defining the range of registers that are valid to allocate in
+ // this class, which also specifies the preferred allocation order.
+ TargetRegisterClass::iterator RI = RC->allocation_order_begin(*MF);
+ TargetRegisterClass::iterator RE = RC->allocation_order_end(*MF);
+
+ for (; RI != RE; ++RI)
+ if (isPhysRegAvailable(*RI)) { // Is reg unused?
+ assert(*RI != 0 && "Cannot use register!");
+ return *RI; // Found an unused register!
+ }
+ return 0;
+}
+
+
+/// chooseReg - Pick a physical register to hold the specified
+/// virtual register by choosing the one whose value will be read
+/// furthest in the future.
+///
+unsigned RABigBlock::chooseReg(MachineBasicBlock &MBB, MachineInstr *I,
+ unsigned VirtReg) {
+ const TargetRegisterClass *RC = MF->getSSARegMap()->getRegClass(VirtReg);
+ // First check to see if we have a free register of the requested type...
+ unsigned PhysReg = getFreeReg(RC);
+
+ // If we didn't find an unused register, find the one which will be
+ // read at the most distant point in time.
+ if (PhysReg == 0) {
+ unsigned delay=0, longest_delay=0;
+ VRegTimes* ReadTimes;
+
+ unsigned curTime = MBBCurTime;
+
+ // for all physical regs in the RC,
+ for(TargetRegisterClass::iterator pReg = RC->begin();
+ pReg != RC->end(); ++pReg) {
+ // how long until they're read?
+ if(PhysRegsUsed[*pReg]>0) { // ignore non-allocatable regs
+ ReadTimes = VRegReadTable[PhysRegsUsed[*pReg]];
+ if(ReadTimes && !ReadTimes->empty()) {
+ unsigned& pt = VRegReadIdx[PhysRegsUsed[*pReg]];
+ while(pt < ReadTimes->size() && (*ReadTimes)[pt] < curTime) {
+ ++pt;
+ }
+
+ if(pt < ReadTimes->size())
+ delay = (*ReadTimes)[pt] - curTime;
+ else
+ delay = MBBLastInsnTime + 1 - curTime;
+ } else {
+ // This register is only defined, but never
+ // read in this MBB. Therefore the next read
+ // happens after the end of this MBB
+ delay = MBBLastInsnTime + 1 - curTime;
+ }
+
+
+ if(delay > longest_delay) {
+ longest_delay = delay;
+ PhysReg = *pReg;
+ }
+ }
+ }
+
+ if(PhysReg == 0) { // ok, now we're desperate. We couldn't choose
+ // a register to spill by looking through the
+ // read timetable, so now we just spill the
+ // first allocatable register we find.
+
+ // for all physical regs in the RC,
+ for(TargetRegisterClass::iterator pReg = RC->begin();
+ pReg != RC->end(); ++pReg) {
+ // if we find a register we can spill
+ if(PhysRegsUsed[*pReg]>=-1)
+ PhysReg = *pReg; // choose it to be spilled
+ }
+ }
+
+ assert(PhysReg && "couldn't choose a register to spill :( ");
+ // TODO: assert that RC->contains(PhysReg) / handle aliased registers?
+
+ // since we needed to look in the table we need to spill this register.
+ spillPhysReg(MBB, I, PhysReg);
+ }
+
+ // assign the vreg to our chosen physical register
+ assignVirtToPhysReg(VirtReg, PhysReg);
+ return PhysReg; // and return it
+}
+
+
+/// reloadVirtReg - This method transforms an instruction with a virtual
+/// register use to one that references a physical register. It does this as
+/// follows:
+///
+/// 1) If the register is already in a physical register, it uses it.
+/// 2) Otherwise, if there is a free physical register, it uses that.
+/// 3) Otherwise, it calls chooseReg() to get the physical register
+/// holding the most distantly needed value, generating a spill in
+/// the process.
+///
+/// This method returns the modified instruction.
+MachineInstr *RABigBlock::reloadVirtReg(MachineBasicBlock &MBB, MachineInstr *MI,
+ unsigned OpNum) {
+ unsigned VirtReg = MI->getOperand(OpNum).getReg();
+
+ // If the virtual register is already available in a physical register,
+ // just update the instruction and return.
+ if (unsigned PR = getVirt2PhysRegMapSlot(VirtReg)) {
+ MI->getOperand(OpNum).setReg(PR);
+ return MI;
+ }
+
+ // Otherwise, if we have free physical registers available to hold the
+ // value, use them.
+ const TargetRegisterClass *RC = MF->getSSARegMap()->getRegClass(VirtReg);
+ unsigned PhysReg = getFreeReg(RC);
+ int FrameIndex = getStackSpaceFor(VirtReg, RC);
+
+ if (PhysReg) { // we have a free register, so use it.
+ assignVirtToPhysReg(VirtReg, PhysReg);
+ } else { // no free registers available.
+ // try to fold the spill into the instruction
+ if(MachineInstr* FMI = RegInfo->foldMemoryOperand(MI, OpNum, FrameIndex)) {
+ ++NumFolded;
+ // Since we changed the address of MI, make sure to update live variables
+ // to know that the new instruction has the properties of the old one.
+ LV->instructionChanged(MI, FMI);
+ return MBB.insert(MBB.erase(MI), FMI);
+ }
+
+ // determine which of the physical registers we'll kill off, since we
+ // couldn't fold.
+ PhysReg = chooseReg(MBB, MI, VirtReg);
+ }
+
+ // this virtual register is now unmodified (since we just reloaded it)
+ markVirtRegModified(VirtReg, false);
+
+ DOUT << " Reloading %reg" << VirtReg << " into "
+ << RegInfo->getName(PhysReg) << "\n";
+
+ // Add move instruction(s)
+ RegInfo->loadRegFromStackSlot(MBB, MI, PhysReg, FrameIndex, RC);
+ ++NumLoads; // Update statistics
+
+ MF->setPhysRegUsed(PhysReg);
+ MI->getOperand(OpNum).setReg(PhysReg); // Assign the input register
+ return MI;
+}
+
+/// Fill out the vreg read timetable. Since ReadTime increases
+/// monotonically, the individual readtime sets will be sorted
+/// in ascending order.
+void RABigBlock::FillVRegReadTable(MachineBasicBlock &MBB) {
+ // loop over each instruction
+ MachineBasicBlock::iterator MII;
+ unsigned ReadTime;
+
+ for(ReadTime=0, MII = MBB.begin(); MII != MBB.end(); ++ReadTime, ++MII) {
+ MachineInstr *MI = MII;
+
+ for (unsigned i = 0; i != MI->getNumOperands(); ++i) {
+ MachineOperand& MO = MI->getOperand(i);
+ // look for vreg reads..
+ if (MO.isRegister() && !MO.isDef() && MO.getReg() &&
+ MRegisterInfo::isVirtualRegister(MO.getReg())) {
+ // ..and add them to the read table.
+ VRegTimes* &Times = VRegReadTable[MO.getReg()];
+ if(!VRegReadTable[MO.getReg()]) {
+ Times = new VRegTimes;
+ VRegReadIdx[MO.getReg()] = 0;
+ }
+ Times->push_back(ReadTime);
+ }
+ }
+
+ }
+
+ MBBLastInsnTime = ReadTime;
+
+ for(DenseMap<unsigned, VRegTimes*, VRegKeyInfo>::iterator Reads = VRegReadTable.begin();
+ Reads != VRegReadTable.end(); ++Reads) {
+ if(Reads->second) {
+ DOUT << "Reads[" << Reads->first << "]=" << Reads->second->size() << "\n";
+ }
+ }
+}
+
+/// isReadModWriteImplicitKill - True if this is an implicit kill for a
+/// read/mod/write register, i.e. update partial register.
+static bool isReadModWriteImplicitKill(MachineInstr *MI, unsigned Reg) {
+ for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
+ MachineOperand& MO = MI->getOperand(i);
+ if (MO.isRegister() && MO.getReg() == Reg && MO.isImplicit() &&
+ MO.isDef() && !MO.isDead())
+ return true;
+ }
+ return false;
+}
+
+/// isReadModWriteImplicitDef - True if this is an implicit def for a
+/// read/mod/write register, i.e. update partial register.
+static bool isReadModWriteImplicitDef(MachineInstr *MI, unsigned Reg) {
+ for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
+ MachineOperand& MO = MI->getOperand(i);
+ if (MO.isRegister() && MO.getReg() == Reg && MO.isImplicit() &&
+ !MO.isDef() && MO.isKill())
+ return true;
+ }
+ return false;
+}
+
+
+void RABigBlock::AllocateBasicBlock(MachineBasicBlock &MBB) {
+ // loop over each instruction
+ MachineBasicBlock::iterator MII = MBB.begin();
+ const TargetInstrInfo &TII = *TM->getInstrInfo();
+
+ DEBUG(const BasicBlock *LBB = MBB.getBasicBlock();
+ if (LBB) DOUT << "\nStarting RegAlloc of BB: " << LBB->getName());
+
+ // If this is the first basic block in the machine function, add live-in
+ // registers as active.
+ if (&MBB == &*MF->begin()) {
+ for (MachineFunction::livein_iterator I = MF->livein_begin(),
+ E = MF->livein_end(); I != E; ++I) {
+ unsigned Reg = I->first;
+ MF->setPhysRegUsed(Reg);
+ PhysRegsUsed[Reg] = 0; // It is free and reserved now
+ for (const unsigned *AliasSet = RegInfo->getSubRegisters(Reg);
+ *AliasSet; ++AliasSet) {
+ if (PhysRegsUsed[*AliasSet] != -2) {
+ PhysRegsUsed[*AliasSet] = 0; // It is free and reserved now
+ MF->setPhysRegUsed(*AliasSet);
+ }
+ }
+ }
+ }
+
+ // Otherwise, sequentially allocate each instruction in the MBB.
+ MBBCurTime = -1;
+ while (MII != MBB.end()) {
+ MachineInstr *MI = MII++;
+ MBBCurTime++;
+ const TargetInstrDescriptor &TID = TII.get(MI->getOpcode());
+ DEBUG(DOUT << "\nTime=" << MBBCurTime << " Starting RegAlloc of: " << *MI;
+ DOUT << " Regs have values: ";
+ for (unsigned i = 0; i != RegInfo->getNumRegs(); ++i)
+ if (PhysRegsUsed[i] != -1 && PhysRegsUsed[i] != -2)
+ DOUT << "[" << RegInfo->getName(i)
+ << ",%reg" << PhysRegsUsed[i] << "] ";
+ DOUT << "\n");
+
+ SmallVector<unsigned, 8> Kills;
+ for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
+ MachineOperand& MO = MI->getOperand(i);
+ if (MO.isRegister() && MO.isKill()) {
+ if (!MO.isImplicit())
+ Kills.push_back(MO.getReg());
+ else if (!isReadModWriteImplicitKill(MI, MO.getReg()))
+ // These are extra physical register kills when a sub-register
+ // is defined (def of a sub-register is a read/mod/write of the
+ // larger registers). Ignore.
+ Kills.push_back(MO.getReg());
+ }
+ }
+
+ // Get the used operands into registers. This has the potential to spill
+ // incoming values if we are out of registers. Note that we completely
+ // ignore physical register uses here. We assume that if an explicit
+ // physical register is referenced by the instruction, that it is guaranteed
+ // to be live-in, or the input is badly hosed.
+ //
+ for (unsigned i = 0; i != MI->getNumOperands(); ++i) {
+ MachineOperand& MO = MI->getOperand(i);
+ // here we are looking for only used operands (never def&use)
+ if (MO.isRegister() && !MO.isDef() && MO.getReg() && !MO.isImplicit() &&
+ MRegisterInfo::isVirtualRegister(MO.getReg()))
+ MI = reloadVirtReg(MBB, MI, i);
+ }
+
+ // If this instruction is the last user of this register, kill the
+ // value, freeing the register being used, so it doesn't need to be
+ // spilled to memory.
+ //
+ for (unsigned i = 0, e = Kills.size(); i != e; ++i) {
+ unsigned VirtReg = Kills[i];
+ unsigned PhysReg = VirtReg;
+ if (MRegisterInfo::isVirtualRegister(VirtReg)) {
+ // If the virtual register was never materialized into a register, it
+ // might not be in the map, but it won't hurt to zero it out anyway.
+ unsigned &PhysRegSlot = getVirt2PhysRegMapSlot(VirtReg);
+ PhysReg = PhysRegSlot;
+ PhysRegSlot = 0;
+ } else if (PhysRegsUsed[PhysReg] == -2) {
+ // Unallocatable register dead, ignore.
+ continue;
+ } else {
+ assert(!PhysRegsUsed[PhysReg] || PhysRegsUsed[PhysReg] == -1 &&
+ "Silently clearing a virtual register?");
+ }
+
+ if (PhysReg) {
+ DOUT << " Last use of " << RegInfo->getName(PhysReg)
+ << "[%reg" << VirtReg <<"], removing it from live set\n";
+ removePhysReg(PhysReg);
+ for (const unsigned *AliasSet = RegInfo->getSubRegisters(PhysReg);
+ *AliasSet; ++AliasSet) {
+ if (PhysRegsUsed[*AliasSet] != -2) {
+ DOUT << " Last use of "
+ << RegInfo->getName(*AliasSet)
+ << "[%reg" << VirtReg <<"], removing it from live set\n";
+ removePhysReg(*AliasSet);
+ }
+ }
+ }
+ }
+
+ // Loop over all of the operands of the instruction, spilling registers that
+ // are defined, and marking explicit destinations in the PhysRegsUsed map.
+ for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
+ MachineOperand& MO = MI->getOperand(i);
+ if (MO.isRegister() && MO.isDef() && !MO.isImplicit() && MO.getReg() &&
+ MRegisterInfo::isPhysicalRegister(MO.getReg())) {
+ unsigned Reg = MO.getReg();
+ if (PhysRegsUsed[Reg] == -2) continue; // Something like ESP.
+ // These are extra physical register defs when a sub-register
+ // is defined (def of a sub-register is a read/mod/write of the
+ // larger registers). Ignore.
+ if (isReadModWriteImplicitDef(MI, MO.getReg())) continue;
+
+ MF->setPhysRegUsed(Reg);
+ spillPhysReg(MBB, MI, Reg, true); // Spill any existing value in reg
+ PhysRegsUsed[Reg] = 0; // It is free and reserved now
+ for (const unsigned *AliasSet = RegInfo->getSubRegisters(Reg);
+ *AliasSet; ++AliasSet) {
+ if (PhysRegsUsed[*AliasSet] != -2) {
+ PhysRegsUsed[*AliasSet] = 0; // It is free and reserved now
+ MF->setPhysRegUsed(*AliasSet);
+ }
+ }
+ }
+ }
+
+ // Loop over the implicit defs, spilling them as well.
+ if (TID.ImplicitDefs) {
+ for (const unsigned *ImplicitDefs = TID.ImplicitDefs;
+ *ImplicitDefs; ++ImplicitDefs) {
+ unsigned Reg = *ImplicitDefs;
+ if (PhysRegsUsed[Reg] != -2) {
+ spillPhysReg(MBB, MI, Reg, true);
+ PhysRegsUsed[Reg] = 0; // It is free and reserved now
+ }
+ MF->setPhysRegUsed(Reg);
+ for (const unsigned *AliasSet = RegInfo->getSubRegisters(Reg);
+ *AliasSet; ++AliasSet) {
+ if (PhysRegsUsed[*AliasSet] != -2) {
+ PhysRegsUsed[*AliasSet] = 0; // It is free and reserved now
+ MF->setPhysRegUsed(*AliasSet);
+ }
+ }
+ }
+ }
+
+ SmallVector<unsigned, 8> DeadDefs;
+ for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
+ MachineOperand& MO = MI->getOperand(i);
+ if (MO.isRegister() && MO.isDead())
+ DeadDefs.push_back(MO.getReg());
+ }
+
+ // Okay, we have allocated all of the source operands and spilled any values
+ // that would be destroyed by defs of this instruction. Loop over the
+ // explicit defs and assign them to a register, spilling incoming values if
+ // we need to scavenge a register.
+ //
+ for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
+ MachineOperand& MO = MI->getOperand(i);
+ if (MO.isRegister() && MO.isDef() && MO.getReg() &&
+ MRegisterInfo::isVirtualRegister(MO.getReg())) {
+ unsigned DestVirtReg = MO.getReg();
+ unsigned DestPhysReg;
+
+ // If DestVirtReg already has a value, use it.
+ if (!(DestPhysReg = getVirt2PhysRegMapSlot(DestVirtReg)))
+ DestPhysReg = chooseReg(MBB, MI, DestVirtReg);
+ MF->setPhysRegUsed(DestPhysReg);
+ markVirtRegModified(DestVirtReg);
+ MI->getOperand(i).setReg(DestPhysReg); // Assign the output register
+ }
+ }
+
+ // If this instruction defines any registers that are immediately dead,
+ // kill them now.
+ //
+ for (unsigned i = 0, e = DeadDefs.size(); i != e; ++i) {
+ unsigned VirtReg = DeadDefs[i];
+ unsigned PhysReg = VirtReg;
+ if (MRegisterInfo::isVirtualRegister(VirtReg)) {
+ unsigned &PhysRegSlot = getVirt2PhysRegMapSlot(VirtReg);
+ PhysReg = PhysRegSlot;
+ assert(PhysReg != 0);
+ PhysRegSlot = 0;
+ } else if (PhysRegsUsed[PhysReg] == -2) {
+ // Unallocatable register dead, ignore.
+ continue;
+ }
+
+ if (PhysReg) {
+ DOUT << " Register " << RegInfo->getName(PhysReg)
+ << " [%reg" << VirtReg
+ << "] is never used, removing it frame live list\n";
+ removePhysReg(PhysReg);
+ for (const unsigned *AliasSet = RegInfo->getAliasSet(PhysReg);
+ *AliasSet; ++AliasSet) {
+ if (PhysRegsUsed[*AliasSet] != -2) {
+ DOUT << " Register " << RegInfo->getName(*AliasSet)
+ << " [%reg" << *AliasSet
+ << "] is never used, removing it frame live list\n";
+ removePhysReg(*AliasSet);
+ }
+ }
+ }
+ }
+
+ // Finally, if this is a noop copy instruction, zap it.
+ unsigned SrcReg, DstReg;
+ if (TII.isMoveInstr(*MI, SrcReg, DstReg) && SrcReg == DstReg) {
+ LV->removeVirtualRegistersKilled(MI);
+ LV->removeVirtualRegistersDead(MI);
+ MBB.erase(MI);
+ }
+ }
+
+ MachineBasicBlock::iterator MI = MBB.getFirstTerminator();
+
+ // Spill all physical registers holding virtual registers now.
+ for (unsigned i = 0, e = RegInfo->getNumRegs(); i != e; ++i)
+ if (PhysRegsUsed[i] != -1 && PhysRegsUsed[i] != -2)
+ if (unsigned VirtReg = PhysRegsUsed[i])
+ spillVirtReg(MBB, MI, VirtReg, i);
+ else
+ removePhysReg(i);
+}
+
+/// runOnMachineFunction - Register allocate the whole function
+///
+bool RABigBlock::runOnMachineFunction(MachineFunction &Fn) {
+ DOUT << "Machine Function " << "\n";
+ MF = &Fn;
+ TM = &Fn.getTarget();
+ RegInfo = TM->getRegisterInfo();
+ LV = &getAnalysis<LiveVariables>();
+
+ PhysRegsUsed.assign(RegInfo->getNumRegs(), -1);
+
+ // At various places we want to efficiently check to see whether a register
+ // is allocatable. To handle this, we mark all unallocatable registers as
+ // being pinned down, permanently.
+ {
+ BitVector Allocable = RegInfo->getAllocatableSet(Fn);
+ for (unsigned i = 0, e = Allocable.size(); i != e; ++i)
+ if (!Allocable[i])
+ PhysRegsUsed[i] = -2; // Mark the reg unallocable.
+ }
+
+ // initialize the virtual->physical register map to have a 'null'
+ // mapping for all virtual registers
+ Virt2PhysRegMap.grow(MF->getSSARegMap()->getLastVirtReg());
+ StackSlotForVirtReg.grow(MF->getSSARegMap()->getLastVirtReg());
+ VirtRegModified.resize(MF->getSSARegMap()->getLastVirtReg() - MRegisterInfo::FirstVirtualRegister + 1,0);
+
+ // Loop over all of the basic blocks, eliminating virtual register references
+ for (MachineFunction::iterator MBB = Fn.begin(), MBBe = Fn.end();
+ MBB != MBBe; ++MBB) {
+ // fill out the read timetable
+ FillVRegReadTable(*MBB);
+ // use it to allocate the BB
+ AllocateBasicBlock(*MBB);
+ // clear it
+ VRegReadTable.clear();
+ }
+
+ StackSlotForVirtReg.clear();
+ PhysRegsUsed.clear();
+ VirtRegModified.clear();
+ Virt2PhysRegMap.clear();
+ return true;
+}
+
+FunctionPass *llvm::createBigBlockRegisterAllocator() {
+ return new RABigBlock();
+}
+
diff --git a/lib/CodeGen/RegAllocLinearScan.cpp b/lib/CodeGen/RegAllocLinearScan.cpp
new file mode 100644
index 0000000..01d43fd
--- /dev/null
+++ b/lib/CodeGen/RegAllocLinearScan.cpp
@@ -0,0 +1,828 @@
+//===-- RegAllocLinearScan.cpp - Linear Scan register allocator -----------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file was developed by the LLVM research group and is distributed under
+// the University of Illinois Open Source License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements a linear scan register allocator.
+//
+//===----------------------------------------------------------------------===//
+
+#define DEBUG_TYPE "regalloc"
+#include "llvm/CodeGen/LiveVariables.h"
+#include "llvm/CodeGen/LiveIntervalAnalysis.h"
+#include "PhysRegTracker.h"
+#include "VirtRegMap.h"
+#include "llvm/Function.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/MachineInstr.h"
+#include "llvm/CodeGen/Passes.h"
+#include "llvm/CodeGen/RegAllocRegistry.h"
+#include "llvm/CodeGen/SSARegMap.h"
+#include "llvm/Target/MRegisterInfo.h"
+#include "llvm/Target/TargetMachine.h"
+#include "llvm/ADT/EquivalenceClasses.h"
+#include "llvm/ADT/Statistic.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/Compiler.h"
+#include <algorithm>
+#include <set>
+#include <queue>
+#include <memory>
+#include <cmath>
+using namespace llvm;
+
+STATISTIC(NumIters , "Number of iterations performed");
+STATISTIC(NumBacktracks, "Number of times we had to backtrack");
+
+static RegisterRegAlloc
+linearscanRegAlloc("linearscan", " linear scan register allocator",
+ createLinearScanRegisterAllocator);
+
+namespace {
+ static unsigned numIterations = 0;
+ static unsigned numIntervals = 0;
+
+ struct VISIBILITY_HIDDEN RALinScan : public MachineFunctionPass {
+ static char ID;
+ RALinScan() : MachineFunctionPass((intptr_t)&ID) {}
+
+ typedef std::pair<LiveInterval*, LiveInterval::iterator> IntervalPtr;
+ typedef std::vector<IntervalPtr> IntervalPtrs;
+ private:
+ /// RelatedRegClasses - This structure is built the first time a function is
+ /// compiled, and keeps track of which register classes have registers that
+ /// belong to multiple classes or have aliases that are in other classes.
+ EquivalenceClasses<const TargetRegisterClass*> RelatedRegClasses;
+ std::map<unsigned, const TargetRegisterClass*> OneClassForEachPhysReg;
+
+ MachineFunction* mf_;
+ const TargetMachine* tm_;
+ const MRegisterInfo* mri_;
+ LiveIntervals* li_;
+
+ /// handled_ - Intervals are added to the handled_ set in the order of their
+ /// start value. This is uses for backtracking.
+ std::vector<LiveInterval*> handled_;
+
+ /// fixed_ - Intervals that correspond to machine registers.
+ ///
+ IntervalPtrs fixed_;
+
+ /// active_ - Intervals that are currently being processed, and which have a
+ /// live range active for the current point.
+ IntervalPtrs active_;
+
+ /// inactive_ - Intervals that are currently being processed, but which have
+ /// a hold at the current point.
+ IntervalPtrs inactive_;
+
+ typedef std::priority_queue<LiveInterval*,
+ std::vector<LiveInterval*>,
+ greater_ptr<LiveInterval> > IntervalHeap;
+ IntervalHeap unhandled_;
+ std::auto_ptr<PhysRegTracker> prt_;
+ std::auto_ptr<VirtRegMap> vrm_;
+ std::auto_ptr<Spiller> spiller_;
+
+ public:
+ virtual const char* getPassName() const {
+ return "Linear Scan Register Allocator";
+ }
+
+ virtual void getAnalysisUsage(AnalysisUsage &AU) const {
+ AU.addRequired<LiveIntervals>();
+ AU.addRequiredID(SimpleRegisterCoalescingID);
+ MachineFunctionPass::getAnalysisUsage(AU);
+ }
+
+ /// runOnMachineFunction - register allocate the whole function
+ bool runOnMachineFunction(MachineFunction&);
+
+ private:
+ /// linearScan - the linear scan algorithm
+ void linearScan();
+
+ /// initIntervalSets - initialize the interval sets.
+ ///
+ void initIntervalSets();
+
+ /// processActiveIntervals - expire old intervals and move non-overlapping
+ /// ones to the inactive list.
+ void processActiveIntervals(unsigned CurPoint);
+
+ /// processInactiveIntervals - expire old intervals and move overlapping
+ /// ones to the active list.
+ void processInactiveIntervals(unsigned CurPoint);
+
+ /// assignRegOrStackSlotAtInterval - assign a register if one
+ /// is available, or spill.
+ void assignRegOrStackSlotAtInterval(LiveInterval* cur);
+
+ ///
+ /// register handling helpers
+ ///
+
+ /// getFreePhysReg - return a free physical register for this virtual
+ /// register interval if we have one, otherwise return 0.
+ unsigned getFreePhysReg(LiveInterval* cur);
+
+ /// assignVirt2StackSlot - assigns this virtual register to a
+ /// stack slot. returns the stack slot
+ int assignVirt2StackSlot(unsigned virtReg);
+
+ void ComputeRelatedRegClasses();
+
+ template <typename ItTy>
+ void printIntervals(const char* const str, ItTy i, ItTy e) const {
+ if (str) DOUT << str << " intervals:\n";
+ for (; i != e; ++i) {
+ DOUT << "\t" << *i->first << " -> ";
+ unsigned reg = i->first->reg;
+ if (MRegisterInfo::isVirtualRegister(reg)) {
+ reg = vrm_->getPhys(reg);
+ }
+ DOUT << mri_->getName(reg) << '\n';
+ }
+ }
+ };
+ char RALinScan::ID = 0;
+}
+
+void RALinScan::ComputeRelatedRegClasses() {
+ const MRegisterInfo &MRI = *mri_;
+
+ // First pass, add all reg classes to the union, and determine at least one
+ // reg class that each register is in.
+ bool HasAliases = false;
+ for (MRegisterInfo::regclass_iterator RCI = MRI.regclass_begin(),
+ E = MRI.regclass_end(); RCI != E; ++RCI) {
+ RelatedRegClasses.insert(*RCI);
+ for (TargetRegisterClass::iterator I = (*RCI)->begin(), E = (*RCI)->end();
+ I != E; ++I) {
+ HasAliases = HasAliases || *MRI.getAliasSet(*I) != 0;
+
+ const TargetRegisterClass *&PRC = OneClassForEachPhysReg[*I];
+ if (PRC) {
+ // Already processed this register. Just make sure we know that
+ // multiple register classes share a register.
+ RelatedRegClasses.unionSets(PRC, *RCI);
+ } else {
+ PRC = *RCI;
+ }
+ }
+ }
+
+ // Second pass, now that we know conservatively what register classes each reg
+ // belongs to, add info about aliases. We don't need to do this for targets
+ // without register aliases.
+ if (HasAliases)
+ for (std::map<unsigned, const TargetRegisterClass*>::iterator
+ I = OneClassForEachPhysReg.begin(), E = OneClassForEachPhysReg.end();
+ I != E; ++I)
+ for (const unsigned *AS = MRI.getAliasSet(I->first); *AS; ++AS)
+ RelatedRegClasses.unionSets(I->second, OneClassForEachPhysReg[*AS]);
+}
+
+bool RALinScan::runOnMachineFunction(MachineFunction &fn) {
+ mf_ = &fn;
+ tm_ = &fn.getTarget();
+ mri_ = tm_->getRegisterInfo();
+ li_ = &getAnalysis<LiveIntervals>();
+
+ // If this is the first function compiled, compute the related reg classes.
+ if (RelatedRegClasses.empty())
+ ComputeRelatedRegClasses();
+
+ if (!prt_.get()) prt_.reset(new PhysRegTracker(*mri_));
+ vrm_.reset(new VirtRegMap(*mf_));
+ if (!spiller_.get()) spiller_.reset(createSpiller());
+
+ initIntervalSets();
+
+ linearScan();
+
+ // Rewrite spill code and update the PhysRegsUsed set.
+ spiller_->runOnMachineFunction(*mf_, *vrm_);
+
+ vrm_.reset(); // Free the VirtRegMap
+
+
+ while (!unhandled_.empty()) unhandled_.pop();
+ fixed_.clear();
+ active_.clear();
+ inactive_.clear();
+ handled_.clear();
+
+ return true;
+}
+
+/// initIntervalSets - initialize the interval sets.
+///
+void RALinScan::initIntervalSets()
+{
+ assert(unhandled_.empty() && fixed_.empty() &&
+ active_.empty() && inactive_.empty() &&
+ "interval sets should be empty on initialization");
+
+ for (LiveIntervals::iterator i = li_->begin(), e = li_->end(); i != e; ++i) {
+ if (MRegisterInfo::isPhysicalRegister(i->second.reg)) {
+ mf_->setPhysRegUsed(i->second.reg);
+ fixed_.push_back(std::make_pair(&i->second, i->second.begin()));
+ } else
+ unhandled_.push(&i->second);
+ }
+}
+
+void RALinScan::linearScan()
+{
+ // linear scan algorithm
+ DOUT << "********** LINEAR SCAN **********\n";
+ DOUT << "********** Function: " << mf_->getFunction()->getName() << '\n';
+
+ // DEBUG(printIntervals("unhandled", unhandled_.begin(), unhandled_.end()));
+ DEBUG(printIntervals("fixed", fixed_.begin(), fixed_.end()));
+ DEBUG(printIntervals("active", active_.begin(), active_.end()));
+ DEBUG(printIntervals("inactive", inactive_.begin(), inactive_.end()));
+
+ while (!unhandled_.empty()) {
+ // pick the interval with the earliest start point
+ LiveInterval* cur = unhandled_.top();
+ unhandled_.pop();
+ ++numIterations;
+ DOUT << "\n*** CURRENT ***: " << *cur << '\n';
+
+ processActiveIntervals(cur->beginNumber());
+ processInactiveIntervals(cur->beginNumber());
+
+ assert(MRegisterInfo::isVirtualRegister(cur->reg) &&
+ "Can only allocate virtual registers!");
+
+ // Allocating a virtual register. try to find a free
+ // physical register or spill an interval (possibly this one) in order to
+ // assign it one.
+ assignRegOrStackSlotAtInterval(cur);
+
+ DEBUG(printIntervals("active", active_.begin(), active_.end()));
+ DEBUG(printIntervals("inactive", inactive_.begin(), inactive_.end()));
+ }
+ numIntervals += li_->getNumIntervals();
+ NumIters += numIterations;
+
+ // expire any remaining active intervals
+ for (IntervalPtrs::reverse_iterator
+ i = active_.rbegin(); i != active_.rend(); ) {
+ unsigned reg = i->first->reg;
+ DOUT << "\tinterval " << *i->first << " expired\n";
+ assert(MRegisterInfo::isVirtualRegister(reg) &&
+ "Can only allocate virtual registers!");
+ reg = vrm_->getPhys(reg);
+ prt_->delRegUse(reg);
+ i = IntervalPtrs::reverse_iterator(active_.erase(i.base()-1));
+ }
+
+ // expire any remaining inactive intervals
+ for (IntervalPtrs::reverse_iterator
+ i = inactive_.rbegin(); i != inactive_.rend(); ) {
+ DOUT << "\tinterval " << *i->first << " expired\n";
+ i = IntervalPtrs::reverse_iterator(inactive_.erase(i.base()-1));
+ }
+
+ // A brute force way of adding live-ins to every BB.
+ MachineFunction::iterator MBB = mf_->begin();
+ ++MBB; // Skip entry MBB.
+ for (MachineFunction::iterator E = mf_->end(); MBB != E; ++MBB) {
+ unsigned StartIdx = li_->getMBBStartIdx(MBB->getNumber());
+ for (IntervalPtrs::iterator i = fixed_.begin(), e = fixed_.end();
+ i != e; ++i)
+ if (i->first->liveAt(StartIdx))
+ MBB->addLiveIn(i->first->reg);
+
+ for (unsigned i = 0, e = handled_.size(); i != e; ++i) {
+ LiveInterval *HI = handled_[i];
+ unsigned Reg = HI->reg;
+ if (!vrm_->hasStackSlot(Reg) && HI->liveAt(StartIdx)) {
+ assert(MRegisterInfo::isVirtualRegister(Reg));
+ Reg = vrm_->getPhys(Reg);
+ MBB->addLiveIn(Reg);
+ }
+ }
+ }
+
+ DOUT << *vrm_;
+}
+
+/// processActiveIntervals - expire old intervals and move non-overlapping ones
+/// to the inactive list.
+void RALinScan::processActiveIntervals(unsigned CurPoint)
+{
+ DOUT << "\tprocessing active intervals:\n";
+
+ for (unsigned i = 0, e = active_.size(); i != e; ++i) {
+ LiveInterval *Interval = active_[i].first;
+ LiveInterval::iterator IntervalPos = active_[i].second;
+ unsigned reg = Interval->reg;
+
+ IntervalPos = Interval->advanceTo(IntervalPos, CurPoint);
+
+ if (IntervalPos == Interval->end()) { // Remove expired intervals.
+ DOUT << "\t\tinterval " << *Interval << " expired\n";
+ assert(MRegisterInfo::isVirtualRegister(reg) &&
+ "Can only allocate virtual registers!");
+ reg = vrm_->getPhys(reg);
+ prt_->delRegUse(reg);
+
+ // Pop off the end of the list.
+ active_[i] = active_.back();
+ active_.pop_back();
+ --i; --e;
+
+ } else if (IntervalPos->start > CurPoint) {
+ // Move inactive intervals to inactive list.
+ DOUT << "\t\tinterval " << *Interval << " inactive\n";
+ assert(MRegisterInfo::isVirtualRegister(reg) &&
+ "Can only allocate virtual registers!");
+ reg = vrm_->getPhys(reg);
+ prt_->delRegUse(reg);
+ // add to inactive.
+ inactive_.push_back(std::make_pair(Interval, IntervalPos));
+
+ // Pop off the end of the list.
+ active_[i] = active_.back();
+ active_.pop_back();
+ --i; --e;
+ } else {
+ // Otherwise, just update the iterator position.
+ active_[i].second = IntervalPos;
+ }
+ }
+}
+
+/// processInactiveIntervals - expire old intervals and move overlapping
+/// ones to the active list.
+void RALinScan::processInactiveIntervals(unsigned CurPoint)
+{
+ DOUT << "\tprocessing inactive intervals:\n";
+
+ for (unsigned i = 0, e = inactive_.size(); i != e; ++i) {
+ LiveInterval *Interval = inactive_[i].first;
+ LiveInterval::iterator IntervalPos = inactive_[i].second;
+ unsigned reg = Interval->reg;
+
+ IntervalPos = Interval->advanceTo(IntervalPos, CurPoint);
+
+ if (IntervalPos == Interval->end()) { // remove expired intervals.
+ DOUT << "\t\tinterval " << *Interval << " expired\n";
+
+ // Pop off the end of the list.
+ inactive_[i] = inactive_.back();
+ inactive_.pop_back();
+ --i; --e;
+ } else if (IntervalPos->start <= CurPoint) {
+ // move re-activated intervals in active list
+ DOUT << "\t\tinterval " << *Interval << " active\n";
+ assert(MRegisterInfo::isVirtualRegister(reg) &&
+ "Can only allocate virtual registers!");
+ reg = vrm_->getPhys(reg);
+ prt_->addRegUse(reg);
+ // add to active
+ active_.push_back(std::make_pair(Interval, IntervalPos));
+
+ // Pop off the end of the list.
+ inactive_[i] = inactive_.back();
+ inactive_.pop_back();
+ --i; --e;
+ } else {
+ // Otherwise, just update the iterator position.
+ inactive_[i].second = IntervalPos;
+ }
+ }
+}
+
+/// updateSpillWeights - updates the spill weights of the specifed physical
+/// register and its weight.
+static void updateSpillWeights(std::vector<float> &Weights,
+ unsigned reg, float weight,
+ const MRegisterInfo *MRI) {
+ Weights[reg] += weight;
+ for (const unsigned* as = MRI->getAliasSet(reg); *as; ++as)
+ Weights[*as] += weight;
+}
+
+static
+RALinScan::IntervalPtrs::iterator
+FindIntervalInVector(RALinScan::IntervalPtrs &IP, LiveInterval *LI) {
+ for (RALinScan::IntervalPtrs::iterator I = IP.begin(), E = IP.end();
+ I != E; ++I)
+ if (I->first == LI) return I;
+ return IP.end();
+}
+
+static void RevertVectorIteratorsTo(RALinScan::IntervalPtrs &V, unsigned Point){
+ for (unsigned i = 0, e = V.size(); i != e; ++i) {
+ RALinScan::IntervalPtr &IP = V[i];
+ LiveInterval::iterator I = std::upper_bound(IP.first->begin(),
+ IP.second, Point);
+ if (I != IP.first->begin()) --I;
+ IP.second = I;
+ }
+}
+
+/// assignRegOrStackSlotAtInterval - assign a register if one is available, or
+/// spill.
+void RALinScan::assignRegOrStackSlotAtInterval(LiveInterval* cur)
+{
+ DOUT << "\tallocating current interval: ";
+
+ PhysRegTracker backupPrt = *prt_;
+
+ std::vector<std::pair<unsigned, float> > SpillWeightsToAdd;
+ unsigned StartPosition = cur->beginNumber();
+ const TargetRegisterClass *RC = mf_->getSSARegMap()->getRegClass(cur->reg);
+ const TargetRegisterClass *RCLeader = RelatedRegClasses.getLeaderValue(RC);
+
+ // for every interval in inactive we overlap with, mark the
+ // register as not free and update spill weights.
+ for (IntervalPtrs::const_iterator i = inactive_.begin(),
+ e = inactive_.end(); i != e; ++i) {
+ unsigned Reg = i->first->reg;
+ assert(MRegisterInfo::isVirtualRegister(Reg) &&
+ "Can only allocate virtual registers!");
+ const TargetRegisterClass *RegRC = mf_->getSSARegMap()->getRegClass(Reg);
+ // If this is not in a related reg class to the register we're allocating,
+ // don't check it.
+ if (RelatedRegClasses.getLeaderValue(RegRC) == RCLeader &&
+ cur->overlapsFrom(*i->first, i->second-1)) {
+ Reg = vrm_->getPhys(Reg);
+ prt_->addRegUse(Reg);
+ SpillWeightsToAdd.push_back(std::make_pair(Reg, i->first->weight));
+ }
+ }
+
+ // Speculatively check to see if we can get a register right now. If not,
+ // we know we won't be able to by adding more constraints. If so, we can
+ // check to see if it is valid. Doing an exhaustive search of the fixed_ list
+ // is very bad (it contains all callee clobbered registers for any functions
+ // with a call), so we want to avoid doing that if possible.
+ unsigned physReg = getFreePhysReg(cur);
+ if (physReg) {
+ // We got a register. However, if it's in the fixed_ list, we might
+ // conflict with it. Check to see if we conflict with it or any of its
+ // aliases.
+ std::set<unsigned> RegAliases;
+ for (const unsigned *AS = mri_->getAliasSet(physReg); *AS; ++AS)
+ RegAliases.insert(*AS);
+
+ bool ConflictsWithFixed = false;
+ for (unsigned i = 0, e = fixed_.size(); i != e; ++i) {
+ IntervalPtr &IP = fixed_[i];
+ if (physReg == IP.first->reg || RegAliases.count(IP.first->reg)) {
+ // Okay, this reg is on the fixed list. Check to see if we actually
+ // conflict.
+ LiveInterval *I = IP.first;
+ if (I->endNumber() > StartPosition) {
+ LiveInterval::iterator II = I->advanceTo(IP.second, StartPosition);
+ IP.second = II;
+ if (II != I->begin() && II->start > StartPosition)
+ --II;
+ if (cur->overlapsFrom(*I, II)) {
+ ConflictsWithFixed = true;
+ break;
+ }
+ }
+ }
+ }
+
+ // Okay, the register picked by our speculative getFreePhysReg call turned
+ // out to be in use. Actually add all of the conflicting fixed registers to
+ // prt so we can do an accurate query.
+ if (ConflictsWithFixed) {
+ // For every interval in fixed we overlap with, mark the register as not
+ // free and update spill weights.
+ for (unsigned i = 0, e = fixed_.size(); i != e; ++i) {
+ IntervalPtr &IP = fixed_[i];
+ LiveInterval *I = IP.first;
+
+ const TargetRegisterClass *RegRC = OneClassForEachPhysReg[I->reg];
+ if (RelatedRegClasses.getLeaderValue(RegRC) == RCLeader &&
+ I->endNumber() > StartPosition) {
+ LiveInterval::iterator II = I->advanceTo(IP.second, StartPosition);
+ IP.second = II;
+ if (II != I->begin() && II->start > StartPosition)
+ --II;
+ if (cur->overlapsFrom(*I, II)) {
+ unsigned reg = I->reg;
+ prt_->addRegUse(reg);
+ SpillWeightsToAdd.push_back(std::make_pair(reg, I->weight));
+ }
+ }
+ }
+
+ // Using the newly updated prt_ object, which includes conflicts in the
+ // future, see if there are any registers available.
+ physReg = getFreePhysReg(cur);
+ }
+ }
+
+ // Restore the physical register tracker, removing information about the
+ // future.
+ *prt_ = backupPrt;
+
+ // if we find a free register, we are done: assign this virtual to
+ // the free physical register and add this interval to the active
+ // list.
+ if (physReg) {
+ DOUT << mri_->getName(physReg) << '\n';
+ vrm_->assignVirt2Phys(cur->reg, physReg);
+ prt_->addRegUse(physReg);
+ active_.push_back(std::make_pair(cur, cur->begin()));
+ handled_.push_back(cur);
+ return;
+ }
+ DOUT << "no free registers\n";
+
+ // Compile the spill weights into an array that is better for scanning.
+ std::vector<float> SpillWeights(mri_->getNumRegs(), 0.0);
+ for (std::vector<std::pair<unsigned, float> >::iterator
+ I = SpillWeightsToAdd.begin(), E = SpillWeightsToAdd.end(); I != E; ++I)
+ updateSpillWeights(SpillWeights, I->first, I->second, mri_);
+
+ // for each interval in active, update spill weights.
+ for (IntervalPtrs::const_iterator i = active_.begin(), e = active_.end();
+ i != e; ++i) {
+ unsigned reg = i->first->reg;
+ assert(MRegisterInfo::isVirtualRegister(reg) &&
+ "Can only allocate virtual registers!");
+ reg = vrm_->getPhys(reg);
+ updateSpillWeights(SpillWeights, reg, i->first->weight, mri_);
+ }
+
+ DOUT << "\tassigning stack slot at interval "<< *cur << ":\n";
+
+ // Find a register to spill.
+ float minWeight = HUGE_VALF;
+ unsigned minReg = cur->preference; // Try the preferred register first.
+
+ if (!minReg || SpillWeights[minReg] == HUGE_VALF)
+ for (TargetRegisterClass::iterator i = RC->allocation_order_begin(*mf_),
+ e = RC->allocation_order_end(*mf_); i != e; ++i) {
+ unsigned reg = *i;
+ if (minWeight > SpillWeights[reg]) {
+ minWeight = SpillWeights[reg];
+ minReg = reg;
+ }
+ }
+
+ // If we didn't find a register that is spillable, try aliases?
+ if (!minReg) {
+ for (TargetRegisterClass::iterator i = RC->allocation_order_begin(*mf_),
+ e = RC->allocation_order_end(*mf_); i != e; ++i) {
+ unsigned reg = *i;
+ // No need to worry about if the alias register size < regsize of RC.
+ // We are going to spill all registers that alias it anyway.
+ for (const unsigned* as = mri_->getAliasSet(reg); *as; ++as) {
+ if (minWeight > SpillWeights[*as]) {
+ minWeight = SpillWeights[*as];
+ minReg = *as;
+ }
+ }
+ }
+
+ // All registers must have inf weight. Just grab one!
+ if (!minReg)
+ minReg = *RC->allocation_order_begin(*mf_);
+ }
+
+ DOUT << "\t\tregister with min weight: "
+ << mri_->getName(minReg) << " (" << minWeight << ")\n";
+
+ // if the current has the minimum weight, we need to spill it and
+ // add any added intervals back to unhandled, and restart
+ // linearscan.
+ if (cur->weight != HUGE_VALF && cur->weight <= minWeight) {
+ DOUT << "\t\t\tspilling(c): " << *cur << '\n';
+ // if the current interval is re-materializable, remember so and don't
+ // assign it a spill slot.
+ if (cur->remat)
+ vrm_->setVirtIsReMaterialized(cur->reg, cur->remat);
+ int slot = cur->remat ? vrm_->assignVirtReMatId(cur->reg)
+ : vrm_->assignVirt2StackSlot(cur->reg);
+ std::vector<LiveInterval*> added =
+ li_->addIntervalsForSpills(*cur, *vrm_, slot);
+ if (added.empty())
+ return; // Early exit if all spills were folded.
+
+ // Merge added with unhandled. Note that we know that
+ // addIntervalsForSpills returns intervals sorted by their starting
+ // point.
+ for (unsigned i = 0, e = added.size(); i != e; ++i)
+ unhandled_.push(added[i]);
+ return;
+ }
+
+ ++NumBacktracks;
+
+ // push the current interval back to unhandled since we are going
+ // to re-run at least this iteration. Since we didn't modify it it
+ // should go back right in the front of the list
+ unhandled_.push(cur);
+
+ // otherwise we spill all intervals aliasing the register with
+ // minimum weight, rollback to the interval with the earliest
+ // start point and let the linear scan algorithm run again
+ std::vector<LiveInterval*> added;
+ assert(MRegisterInfo::isPhysicalRegister(minReg) &&
+ "did not choose a register to spill?");
+ BitVector toSpill(mri_->getNumRegs());
+
+ // We are going to spill minReg and all its aliases.
+ toSpill[minReg] = true;
+ for (const unsigned* as = mri_->getAliasSet(minReg); *as; ++as)
+ toSpill[*as] = true;
+
+ // the earliest start of a spilled interval indicates up to where
+ // in handled we need to roll back
+ unsigned earliestStart = cur->beginNumber();
+
+ // set of spilled vregs (used later to rollback properly)
+ std::set<unsigned> spilled;
+
+ // spill live intervals of virtual regs mapped to the physical register we
+ // want to clear (and its aliases). We only spill those that overlap with the
+ // current interval as the rest do not affect its allocation. we also keep
+ // track of the earliest start of all spilled live intervals since this will
+ // mark our rollback point.
+ for (IntervalPtrs::iterator i = active_.begin(); i != active_.end(); ++i) {
+ unsigned reg = i->first->reg;
+ if (//MRegisterInfo::isVirtualRegister(reg) &&
+ toSpill[vrm_->getPhys(reg)] &&
+ cur->overlapsFrom(*i->first, i->second)) {
+ DOUT << "\t\t\tspilling(a): " << *i->first << '\n';
+ earliestStart = std::min(earliestStart, i->first->beginNumber());
+ if (i->first->remat)
+ vrm_->setVirtIsReMaterialized(reg, i->first->remat);
+ int slot = i->first->remat ? vrm_->assignVirtReMatId(reg)
+ : vrm_->assignVirt2StackSlot(reg);
+ std::vector<LiveInterval*> newIs =
+ li_->addIntervalsForSpills(*i->first, *vrm_, slot);
+ std::copy(newIs.begin(), newIs.end(), std::back_inserter(added));
+ spilled.insert(reg);
+ }
+ }
+ for (IntervalPtrs::iterator i = inactive_.begin(); i != inactive_.end(); ++i){
+ unsigned reg = i->first->reg;
+ if (//MRegisterInfo::isVirtualRegister(reg) &&
+ toSpill[vrm_->getPhys(reg)] &&
+ cur->overlapsFrom(*i->first, i->second-1)) {
+ DOUT << "\t\t\tspilling(i): " << *i->first << '\n';
+ earliestStart = std::min(earliestStart, i->first->beginNumber());
+ if (i->first->remat)
+ vrm_->setVirtIsReMaterialized(reg, i->first->remat);
+ int slot = i->first->remat ? vrm_->assignVirtReMatId(reg)
+ : vrm_->assignVirt2StackSlot(reg);
+ std::vector<LiveInterval*> newIs =
+ li_->addIntervalsForSpills(*i->first, *vrm_, slot);
+ std::copy(newIs.begin(), newIs.end(), std::back_inserter(added));
+ spilled.insert(reg);
+ }
+ }
+
+ DOUT << "\t\trolling back to: " << earliestStart << '\n';
+
+ // Scan handled in reverse order up to the earliest start of a
+ // spilled live interval and undo each one, restoring the state of
+ // unhandled.
+ while (!handled_.empty()) {
+ LiveInterval* i = handled_.back();
+ // If this interval starts before t we are done.
+ if (i->beginNumber() < earliestStart)
+ break;
+ DOUT << "\t\t\tundo changes for: " << *i << '\n';
+ handled_.pop_back();
+
+ // When undoing a live interval allocation we must know if it is active or
+ // inactive to properly update the PhysRegTracker and the VirtRegMap.
+ IntervalPtrs::iterator it;
+ if ((it = FindIntervalInVector(active_, i)) != active_.end()) {
+ active_.erase(it);
+ assert(!MRegisterInfo::isPhysicalRegister(i->reg));
+ if (!spilled.count(i->reg))
+ unhandled_.push(i);
+ prt_->delRegUse(vrm_->getPhys(i->reg));
+ vrm_->clearVirt(i->reg);
+ } else if ((it = FindIntervalInVector(inactive_, i)) != inactive_.end()) {
+ inactive_.erase(it);
+ assert(!MRegisterInfo::isPhysicalRegister(i->reg));
+ if (!spilled.count(i->reg))
+ unhandled_.push(i);
+ vrm_->clearVirt(i->reg);
+ } else {
+ assert(MRegisterInfo::isVirtualRegister(i->reg) &&
+ "Can only allocate virtual registers!");
+ vrm_->clearVirt(i->reg);
+ unhandled_.push(i);
+ }
+ }
+
+ // Rewind the iterators in the active, inactive, and fixed lists back to the
+ // point we reverted to.
+ RevertVectorIteratorsTo(active_, earliestStart);
+ RevertVectorIteratorsTo(inactive_, earliestStart);
+ RevertVectorIteratorsTo(fixed_, earliestStart);
+
+ // scan the rest and undo each interval that expired after t and
+ // insert it in active (the next iteration of the algorithm will
+ // put it in inactive if required)
+ for (unsigned i = 0, e = handled_.size(); i != e; ++i) {
+ LiveInterval *HI = handled_[i];
+ if (!HI->expiredAt(earliestStart) &&
+ HI->expiredAt(cur->beginNumber())) {
+ DOUT << "\t\t\tundo changes for: " << *HI << '\n';
+ active_.push_back(std::make_pair(HI, HI->begin()));
+ assert(!MRegisterInfo::isPhysicalRegister(HI->reg));
+ prt_->addRegUse(vrm_->getPhys(HI->reg));
+ }
+ }
+
+ // merge added with unhandled
+ for (unsigned i = 0, e = added.size(); i != e; ++i)
+ unhandled_.push(added[i]);
+}
+
+/// getFreePhysReg - return a free physical register for this virtual register
+/// interval if we have one, otherwise return 0.
+unsigned RALinScan::getFreePhysReg(LiveInterval *cur) {
+ std::vector<unsigned> inactiveCounts(mri_->getNumRegs(), 0);
+ unsigned MaxInactiveCount = 0;
+
+ const TargetRegisterClass *RC = mf_->getSSARegMap()->getRegClass(cur->reg);
+ const TargetRegisterClass *RCLeader = RelatedRegClasses.getLeaderValue(RC);
+
+ for (IntervalPtrs::iterator i = inactive_.begin(), e = inactive_.end();
+ i != e; ++i) {
+ unsigned reg = i->first->reg;
+ assert(MRegisterInfo::isVirtualRegister(reg) &&
+ "Can only allocate virtual registers!");
+
+ // If this is not in a related reg class to the register we're allocating,
+ // don't check it.
+ const TargetRegisterClass *RegRC = mf_->getSSARegMap()->getRegClass(reg);
+ if (RelatedRegClasses.getLeaderValue(RegRC) == RCLeader) {
+ reg = vrm_->getPhys(reg);
+ ++inactiveCounts[reg];
+ MaxInactiveCount = std::max(MaxInactiveCount, inactiveCounts[reg]);
+ }
+ }
+
+ unsigned FreeReg = 0;
+ unsigned FreeRegInactiveCount = 0;
+
+ // If copy coalescer has assigned a "preferred" register, check if it's
+ // available first.
+ if (cur->preference)
+ if (prt_->isRegAvail(cur->preference)) {
+ DOUT << "\t\tassigned the preferred register: "
+ << mri_->getName(cur->preference) << "\n";
+ return cur->preference;
+ } else
+ DOUT << "\t\tunable to assign the preferred register: "
+ << mri_->getName(cur->preference) << "\n";
+
+ // Scan for the first available register.
+ TargetRegisterClass::iterator I = RC->allocation_order_begin(*mf_);
+ TargetRegisterClass::iterator E = RC->allocation_order_end(*mf_);
+ for (; I != E; ++I)
+ if (prt_->isRegAvail(*I)) {
+ FreeReg = *I;
+ FreeRegInactiveCount = inactiveCounts[FreeReg];
+ break;
+ }
+
+ // If there are no free regs, or if this reg has the max inactive count,
+ // return this register.
+ if (FreeReg == 0 || FreeRegInactiveCount == MaxInactiveCount) return FreeReg;
+
+ // Continue scanning the registers, looking for the one with the highest
+ // inactive count. Alkis found that this reduced register pressure very
+ // slightly on X86 (in rev 1.94 of this file), though this should probably be
+ // reevaluated now.
+ for (; I != E; ++I) {
+ unsigned Reg = *I;
+ if (prt_->isRegAvail(Reg) && FreeRegInactiveCount < inactiveCounts[Reg]) {
+ FreeReg = Reg;
+ FreeRegInactiveCount = inactiveCounts[Reg];
+ if (FreeRegInactiveCount == MaxInactiveCount)
+ break; // We found the one with the max inactive count.
+ }
+ }
+
+ return FreeReg;
+}
+
+FunctionPass* llvm::createLinearScanRegisterAllocator() {
+ return new RALinScan();
+}
diff --git a/lib/CodeGen/RegAllocLocal.cpp b/lib/CodeGen/RegAllocLocal.cpp
new file mode 100644
index 0000000..6454900
--- /dev/null
+++ b/lib/CodeGen/RegAllocLocal.cpp
@@ -0,0 +1,830 @@
+//===-- RegAllocLocal.cpp - A BasicBlock generic register allocator -------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file was developed by the LLVM research group and is distributed under
+// the University of Illinois Open Source License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This register allocator allocates registers to a basic block at a time,
+// attempting to keep values in registers and reusing registers as appropriate.
+//
+//===----------------------------------------------------------------------===//
+
+#define DEBUG_TYPE "regalloc"
+#include "llvm/BasicBlock.h"
+#include "llvm/CodeGen/Passes.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/MachineInstr.h"
+#include "llvm/CodeGen/SSARegMap.h"
+#include "llvm/CodeGen/MachineFrameInfo.h"
+#include "llvm/CodeGen/LiveVariables.h"
+#include "llvm/CodeGen/RegAllocRegistry.h"
+#include "llvm/Target/TargetInstrInfo.h"
+#include "llvm/Target/TargetMachine.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/Compiler.h"
+#include "llvm/ADT/IndexedMap.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/Statistic.h"
+#include <algorithm>
+using namespace llvm;
+
+STATISTIC(NumStores, "Number of stores added");
+STATISTIC(NumLoads , "Number of loads added");
+STATISTIC(NumFolded, "Number of loads/stores folded into instructions");
+
+namespace {
+ static RegisterRegAlloc
+ localRegAlloc("local", " local register allocator",
+ createLocalRegisterAllocator);
+
+
+ class VISIBILITY_HIDDEN RALocal : public MachineFunctionPass {
+ public:
+ static char ID;
+ RALocal() : MachineFunctionPass((intptr_t)&ID) {}
+ private:
+ const TargetMachine *TM;
+ MachineFunction *MF;
+ const MRegisterInfo *RegInfo;
+ LiveVariables *LV;
+
+ // StackSlotForVirtReg - Maps virtual regs to the frame index where these
+ // values are spilled.
+ std::map<unsigned, int> StackSlotForVirtReg;
+
+ // Virt2PhysRegMap - This map contains entries for each virtual register
+ // that is currently available in a physical register.
+ IndexedMap<unsigned, VirtReg2IndexFunctor> Virt2PhysRegMap;
+
+ unsigned &getVirt2PhysRegMapSlot(unsigned VirtReg) {
+ return Virt2PhysRegMap[VirtReg];
+ }
+
+ // PhysRegsUsed - This array is effectively a map, containing entries for
+ // each physical register that currently has a value (ie, it is in
+ // Virt2PhysRegMap). The value mapped to is the virtual register
+ // corresponding to the physical register (the inverse of the
+ // Virt2PhysRegMap), or 0. The value is set to 0 if this register is pinned
+ // because it is used by a future instruction, and to -2 if it is not
+ // allocatable. If the entry for a physical register is -1, then the
+ // physical register is "not in the map".
+ //
+ std::vector<int> PhysRegsUsed;
+
+ // PhysRegsUseOrder - This contains a list of the physical registers that
+ // currently have a virtual register value in them. This list provides an
+ // ordering of registers, imposing a reallocation order. This list is only
+ // used if all registers are allocated and we have to spill one, in which
+ // case we spill the least recently used register. Entries at the front of
+ // the list are the least recently used registers, entries at the back are
+ // the most recently used.
+ //
+ std::vector<unsigned> PhysRegsUseOrder;
+
+ // VirtRegModified - This bitset contains information about which virtual
+ // registers need to be spilled back to memory when their registers are
+ // scavenged. If a virtual register has simply been rematerialized, there
+ // is no reason to spill it to memory when we need the register back.
+ //
+ std::vector<bool> VirtRegModified;
+
+ void markVirtRegModified(unsigned Reg, bool Val = true) {
+ assert(MRegisterInfo::isVirtualRegister(Reg) && "Illegal VirtReg!");
+ Reg -= MRegisterInfo::FirstVirtualRegister;
+ if (VirtRegModified.size() <= Reg) VirtRegModified.resize(Reg+1);
+ VirtRegModified[Reg] = Val;
+ }
+
+ bool isVirtRegModified(unsigned Reg) const {
+ assert(MRegisterInfo::isVirtualRegister(Reg) && "Illegal VirtReg!");
+ assert(Reg - MRegisterInfo::FirstVirtualRegister < VirtRegModified.size()
+ && "Illegal virtual register!");
+ return VirtRegModified[Reg - MRegisterInfo::FirstVirtualRegister];
+ }
+
+ void AddToPhysRegsUseOrder(unsigned Reg) {
+ std::vector<unsigned>::iterator It =
+ std::find(PhysRegsUseOrder.begin(), PhysRegsUseOrder.end(), Reg);
+ if (It != PhysRegsUseOrder.end())
+ PhysRegsUseOrder.erase(It);
+ PhysRegsUseOrder.push_back(Reg);
+ }
+
+ void MarkPhysRegRecentlyUsed(unsigned Reg) {
+ if (PhysRegsUseOrder.empty() ||
+ PhysRegsUseOrder.back() == Reg) return; // Already most recently used
+
+ for (unsigned i = PhysRegsUseOrder.size(); i != 0; --i)
+ if (areRegsEqual(Reg, PhysRegsUseOrder[i-1])) {
+ unsigned RegMatch = PhysRegsUseOrder[i-1]; // remove from middle
+ PhysRegsUseOrder.erase(PhysRegsUseOrder.begin()+i-1);
+ // Add it to the end of the list
+ PhysRegsUseOrder.push_back(RegMatch);
+ if (RegMatch == Reg)
+ return; // Found an exact match, exit early
+ }
+ }
+
+ public:
+ virtual const char *getPassName() const {
+ return "Local Register Allocator";
+ }
+
+ virtual void getAnalysisUsage(AnalysisUsage &AU) const {
+ AU.addRequired<LiveVariables>();
+ AU.addRequiredID(PHIEliminationID);
+ AU.addRequiredID(TwoAddressInstructionPassID);
+ MachineFunctionPass::getAnalysisUsage(AU);
+ }
+
+ private:
+ /// runOnMachineFunction - Register allocate the whole function
+ bool runOnMachineFunction(MachineFunction &Fn);
+
+ /// AllocateBasicBlock - Register allocate the specified basic block.
+ void AllocateBasicBlock(MachineBasicBlock &MBB);
+
+
+ /// areRegsEqual - This method returns true if the specified registers are
+ /// related to each other. To do this, it checks to see if they are equal
+ /// or if the first register is in the alias set of the second register.
+ ///
+ bool areRegsEqual(unsigned R1, unsigned R2) const {
+ if (R1 == R2) return true;
+ for (const unsigned *AliasSet = RegInfo->getAliasSet(R2);
+ *AliasSet; ++AliasSet) {
+ if (*AliasSet == R1) return true;
+ }
+ return false;
+ }
+
+ /// getStackSpaceFor - This returns the frame index of the specified virtual
+ /// register on the stack, allocating space if necessary.
+ int getStackSpaceFor(unsigned VirtReg, const TargetRegisterClass *RC);
+
+ /// removePhysReg - This method marks the specified physical register as no
+ /// longer being in use.
+ ///
+ void removePhysReg(unsigned PhysReg);
+
+ /// spillVirtReg - This method spills the value specified by PhysReg into
+ /// the virtual register slot specified by VirtReg. It then updates the RA
+ /// data structures to indicate the fact that PhysReg is now available.
+ ///
+ void spillVirtReg(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
+ unsigned VirtReg, unsigned PhysReg);
+
+ /// spillPhysReg - This method spills the specified physical register into
+ /// the virtual register slot associated with it. If OnlyVirtRegs is set to
+ /// true, then the request is ignored if the physical register does not
+ /// contain a virtual register.
+ ///
+ void spillPhysReg(MachineBasicBlock &MBB, MachineInstr *I,
+ unsigned PhysReg, bool OnlyVirtRegs = false);
+
+ /// assignVirtToPhysReg - This method updates local state so that we know
+ /// that PhysReg is the proper container for VirtReg now. The physical
+ /// register must not be used for anything else when this is called.
+ ///
+ void assignVirtToPhysReg(unsigned VirtReg, unsigned PhysReg);
+
+ /// isPhysRegAvailable - Return true if the specified physical register is
+ /// free and available for use. This also includes checking to see if
+ /// aliased registers are all free...
+ ///
+ bool isPhysRegAvailable(unsigned PhysReg) const;
+
+ /// getFreeReg - Look to see if there is a free register available in the
+ /// specified register class. If not, return 0.
+ ///
+ unsigned getFreeReg(const TargetRegisterClass *RC);
+
+ /// getReg - Find a physical register to hold the specified virtual
+ /// register. If all compatible physical registers are used, this method
+ /// spills the last used virtual register to the stack, and uses that
+ /// register.
+ ///
+ unsigned getReg(MachineBasicBlock &MBB, MachineInstr *MI,
+ unsigned VirtReg);
+
+ /// reloadVirtReg - This method transforms the specified specified virtual
+ /// register use to refer to a physical register. This method may do this
+ /// in one of several ways: if the register is available in a physical
+ /// register already, it uses that physical register. If the value is not
+ /// in a physical register, and if there are physical registers available,
+ /// it loads it into a register. If register pressure is high, and it is
+ /// possible, it tries to fold the load of the virtual register into the
+ /// instruction itself. It avoids doing this if register pressure is low to
+ /// improve the chance that subsequent instructions can use the reloaded
+ /// value. This method returns the modified instruction.
+ ///
+ MachineInstr *reloadVirtReg(MachineBasicBlock &MBB, MachineInstr *MI,
+ unsigned OpNum);
+
+
+ void reloadPhysReg(MachineBasicBlock &MBB, MachineBasicBlock::iterator &I,
+ unsigned PhysReg);
+ };
+ char RALocal::ID = 0;
+}
+
+/// getStackSpaceFor - This allocates space for the specified virtual register
+/// to be held on the stack.
+int RALocal::getStackSpaceFor(unsigned VirtReg, const TargetRegisterClass *RC) {
+ // Find the location Reg would belong...
+ std::map<unsigned, int>::iterator I =StackSlotForVirtReg.lower_bound(VirtReg);
+
+ if (I != StackSlotForVirtReg.end() && I->first == VirtReg)
+ return I->second; // Already has space allocated?
+
+ // Allocate a new stack object for this spill location...
+ int FrameIdx = MF->getFrameInfo()->CreateStackObject(RC->getSize(),
+ RC->getAlignment());
+
+ // Assign the slot...
+ StackSlotForVirtReg.insert(I, std::make_pair(VirtReg, FrameIdx));
+ return FrameIdx;
+}
+
+
+/// removePhysReg - This method marks the specified physical register as no
+/// longer being in use.
+///
+void RALocal::removePhysReg(unsigned PhysReg) {
+ PhysRegsUsed[PhysReg] = -1; // PhyReg no longer used
+
+ std::vector<unsigned>::iterator It =
+ std::find(PhysRegsUseOrder.begin(), PhysRegsUseOrder.end(), PhysReg);
+ if (It != PhysRegsUseOrder.end())
+ PhysRegsUseOrder.erase(It);
+}
+
+
+/// spillVirtReg - This method spills the value specified by PhysReg into the
+/// virtual register slot specified by VirtReg. It then updates the RA data
+/// structures to indicate the fact that PhysReg is now available.
+///
+void RALocal::spillVirtReg(MachineBasicBlock &MBB,
+ MachineBasicBlock::iterator I,
+ unsigned VirtReg, unsigned PhysReg) {
+ assert(VirtReg && "Spilling a physical register is illegal!"
+ " Must not have appropriate kill for the register or use exists beyond"
+ " the intended one.");
+ DOUT << " Spilling register " << RegInfo->getName(PhysReg)
+ << " containing %reg" << VirtReg;
+ if (!isVirtRegModified(VirtReg))
+ DOUT << " which has not been modified, so no store necessary!";
+
+ // Otherwise, there is a virtual register corresponding to this physical
+ // register. We only need to spill it into its stack slot if it has been
+ // modified.
+ if (isVirtRegModified(VirtReg)) {
+ const TargetRegisterClass *RC = MF->getSSARegMap()->getRegClass(VirtReg);
+ int FrameIndex = getStackSpaceFor(VirtReg, RC);
+ DOUT << " to stack slot #" << FrameIndex;
+ RegInfo->storeRegToStackSlot(MBB, I, PhysReg, FrameIndex, RC);
+ ++NumStores; // Update statistics
+ }
+
+ getVirt2PhysRegMapSlot(VirtReg) = 0; // VirtReg no longer available
+
+ DOUT << "\n";
+ removePhysReg(PhysReg);
+}
+
+
+/// spillPhysReg - This method spills the specified physical register into the
+/// virtual register slot associated with it. If OnlyVirtRegs is set to true,
+/// then the request is ignored if the physical register does not contain a
+/// virtual register.
+///
+void RALocal::spillPhysReg(MachineBasicBlock &MBB, MachineInstr *I,
+ unsigned PhysReg, bool OnlyVirtRegs) {
+ if (PhysRegsUsed[PhysReg] != -1) { // Only spill it if it's used!
+ assert(PhysRegsUsed[PhysReg] != -2 && "Non allocable reg used!");
+ if (PhysRegsUsed[PhysReg] || !OnlyVirtRegs)
+ spillVirtReg(MBB, I, PhysRegsUsed[PhysReg], PhysReg);
+ } else {
+ // If the selected register aliases any other registers, we must make
+ // sure that one of the aliases isn't alive.
+ for (const unsigned *AliasSet = RegInfo->getAliasSet(PhysReg);
+ *AliasSet; ++AliasSet)
+ if (PhysRegsUsed[*AliasSet] != -1 && // Spill aliased register.
+ PhysRegsUsed[*AliasSet] != -2) // If allocatable.
+ if (PhysRegsUsed[*AliasSet])
+ spillVirtReg(MBB, I, PhysRegsUsed[*AliasSet], *AliasSet);
+ }
+}
+
+
+/// assignVirtToPhysReg - This method updates local state so that we know
+/// that PhysReg is the proper container for VirtReg now. The physical
+/// register must not be used for anything else when this is called.
+///
+void RALocal::assignVirtToPhysReg(unsigned VirtReg, unsigned PhysReg) {
+ assert(PhysRegsUsed[PhysReg] == -1 && "Phys reg already assigned!");
+ // Update information to note the fact that this register was just used, and
+ // it holds VirtReg.
+ PhysRegsUsed[PhysReg] = VirtReg;
+ getVirt2PhysRegMapSlot(VirtReg) = PhysReg;
+ AddToPhysRegsUseOrder(PhysReg); // New use of PhysReg
+}
+
+
+/// isPhysRegAvailable - Return true if the specified physical register is free
+/// and available for use. This also includes checking to see if aliased
+/// registers are all free...
+///
+bool RALocal::isPhysRegAvailable(unsigned PhysReg) const {
+ if (PhysRegsUsed[PhysReg] != -1) return false;
+
+ // If the selected register aliases any other allocated registers, it is
+ // not free!
+ for (const unsigned *AliasSet = RegInfo->getAliasSet(PhysReg);
+ *AliasSet; ++AliasSet)
+ if (PhysRegsUsed[*AliasSet] != -1) // Aliased register in use?
+ return false; // Can't use this reg then.
+ return true;
+}
+
+
+/// getFreeReg - Look to see if there is a free register available in the
+/// specified register class. If not, return 0.
+///
+unsigned RALocal::getFreeReg(const TargetRegisterClass *RC) {
+ // Get iterators defining the range of registers that are valid to allocate in
+ // this class, which also specifies the preferred allocation order.
+ TargetRegisterClass::iterator RI = RC->allocation_order_begin(*MF);
+ TargetRegisterClass::iterator RE = RC->allocation_order_end(*MF);
+
+ for (; RI != RE; ++RI)
+ if (isPhysRegAvailable(*RI)) { // Is reg unused?
+ assert(*RI != 0 && "Cannot use register!");
+ return *RI; // Found an unused register!
+ }
+ return 0;
+}
+
+
+/// getReg - Find a physical register to hold the specified virtual
+/// register. If all compatible physical registers are used, this method spills
+/// the last used virtual register to the stack, and uses that register.
+///
+unsigned RALocal::getReg(MachineBasicBlock &MBB, MachineInstr *I,
+ unsigned VirtReg) {
+ const TargetRegisterClass *RC = MF->getSSARegMap()->getRegClass(VirtReg);
+
+ // First check to see if we have a free register of the requested type...
+ unsigned PhysReg = getFreeReg(RC);
+
+ // If we didn't find an unused register, scavenge one now!
+ if (PhysReg == 0) {
+ assert(!PhysRegsUseOrder.empty() && "No allocated registers??");
+
+ // Loop over all of the preallocated registers from the least recently used
+ // to the most recently used. When we find one that is capable of holding
+ // our register, use it.
+ for (unsigned i = 0; PhysReg == 0; ++i) {
+ assert(i != PhysRegsUseOrder.size() &&
+ "Couldn't find a register of the appropriate class!");
+
+ unsigned R = PhysRegsUseOrder[i];
+
+ // We can only use this register if it holds a virtual register (ie, it
+ // can be spilled). Do not use it if it is an explicitly allocated
+ // physical register!
+ assert(PhysRegsUsed[R] != -1 &&
+ "PhysReg in PhysRegsUseOrder, but is not allocated?");
+ if (PhysRegsUsed[R] && PhysRegsUsed[R] != -2) {
+ // If the current register is compatible, use it.
+ if (RC->contains(R)) {
+ PhysReg = R;
+ break;
+ } else {
+ // If one of the registers aliased to the current register is
+ // compatible, use it.
+ for (const unsigned *AliasIt = RegInfo->getAliasSet(R);
+ *AliasIt; ++AliasIt) {
+ if (RC->contains(*AliasIt) &&
+ // If this is pinned down for some reason, don't use it. For
+ // example, if CL is pinned, and we run across CH, don't use
+ // CH as justification for using scavenging ECX (which will
+ // fail).
+ PhysRegsUsed[*AliasIt] != 0 &&
+
+ // Make sure the register is allocatable. Don't allocate SIL on
+ // x86-32.
+ PhysRegsUsed[*AliasIt] != -2) {
+ PhysReg = *AliasIt; // Take an aliased register
+ break;
+ }
+ }
+ }
+ }
+ }
+
+ assert(PhysReg && "Physical register not assigned!?!?");
+
+ // At this point PhysRegsUseOrder[i] is the least recently used register of
+ // compatible register class. Spill it to memory and reap its remains.
+ spillPhysReg(MBB, I, PhysReg);
+ }
+
+ // Now that we know which register we need to assign this to, do it now!
+ assignVirtToPhysReg(VirtReg, PhysReg);
+ return PhysReg;
+}
+
+
+/// reloadVirtReg - This method transforms the specified specified virtual
+/// register use to refer to a physical register. This method may do this in
+/// one of several ways: if the register is available in a physical register
+/// already, it uses that physical register. If the value is not in a physical
+/// register, and if there are physical registers available, it loads it into a
+/// register. If register pressure is high, and it is possible, it tries to
+/// fold the load of the virtual register into the instruction itself. It
+/// avoids doing this if register pressure is low to improve the chance that
+/// subsequent instructions can use the reloaded value. This method returns the
+/// modified instruction.
+///
+MachineInstr *RALocal::reloadVirtReg(MachineBasicBlock &MBB, MachineInstr *MI,
+ unsigned OpNum) {
+ unsigned VirtReg = MI->getOperand(OpNum).getReg();
+
+ // If the virtual register is already available, just update the instruction
+ // and return.
+ if (unsigned PR = getVirt2PhysRegMapSlot(VirtReg)) {
+ MarkPhysRegRecentlyUsed(PR); // Already have this value available!
+ MI->getOperand(OpNum).setReg(PR); // Assign the input register
+ return MI;
+ }
+
+ // Otherwise, we need to fold it into the current instruction, or reload it.
+ // If we have registers available to hold the value, use them.
+ const TargetRegisterClass *RC = MF->getSSARegMap()->getRegClass(VirtReg);
+ unsigned PhysReg = getFreeReg(RC);
+ int FrameIndex = getStackSpaceFor(VirtReg, RC);
+
+ if (PhysReg) { // Register is available, allocate it!
+ assignVirtToPhysReg(VirtReg, PhysReg);
+ } else { // No registers available.
+ // If we can fold this spill into this instruction, do so now.
+ if (MachineInstr* FMI = RegInfo->foldMemoryOperand(MI, OpNum, FrameIndex)){
+ ++NumFolded;
+ // Since we changed the address of MI, make sure to update live variables
+ // to know that the new instruction has the properties of the old one.
+ LV->instructionChanged(MI, FMI);
+ return MBB.insert(MBB.erase(MI), FMI);
+ }
+
+ // It looks like we can't fold this virtual register load into this
+ // instruction. Force some poor hapless value out of the register file to
+ // make room for the new register, and reload it.
+ PhysReg = getReg(MBB, MI, VirtReg);
+ }
+
+ markVirtRegModified(VirtReg, false); // Note that this reg was just reloaded
+
+ DOUT << " Reloading %reg" << VirtReg << " into "
+ << RegInfo->getName(PhysReg) << "\n";
+
+ // Add move instruction(s)
+ RegInfo->loadRegFromStackSlot(MBB, MI, PhysReg, FrameIndex, RC);
+ ++NumLoads; // Update statistics
+
+ MF->setPhysRegUsed(PhysReg);
+ MI->getOperand(OpNum).setReg(PhysReg); // Assign the input register
+ return MI;
+}
+
+/// isReadModWriteImplicitKill - True if this is an implicit kill for a
+/// read/mod/write register, i.e. update partial register.
+static bool isReadModWriteImplicitKill(MachineInstr *MI, unsigned Reg) {
+ for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
+ MachineOperand& MO = MI->getOperand(i);
+ if (MO.isRegister() && MO.getReg() == Reg && MO.isImplicit() &&
+ MO.isDef() && !MO.isDead())
+ return true;
+ }
+ return false;
+}
+
+/// isReadModWriteImplicitDef - True if this is an implicit def for a
+/// read/mod/write register, i.e. update partial register.
+static bool isReadModWriteImplicitDef(MachineInstr *MI, unsigned Reg) {
+ for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
+ MachineOperand& MO = MI->getOperand(i);
+ if (MO.isRegister() && MO.getReg() == Reg && MO.isImplicit() &&
+ !MO.isDef() && MO.isKill())
+ return true;
+ }
+ return false;
+}
+
+void RALocal::AllocateBasicBlock(MachineBasicBlock &MBB) {
+ // loop over each instruction
+ MachineBasicBlock::iterator MII = MBB.begin();
+ const TargetInstrInfo &TII = *TM->getInstrInfo();
+
+ DEBUG(const BasicBlock *LBB = MBB.getBasicBlock();
+ if (LBB) DOUT << "\nStarting RegAlloc of BB: " << LBB->getName());
+
+ // If this is the first basic block in the machine function, add live-in
+ // registers as active.
+ if (&MBB == &*MF->begin()) {
+ for (MachineFunction::livein_iterator I = MF->livein_begin(),
+ E = MF->livein_end(); I != E; ++I) {
+ unsigned Reg = I->first;
+ MF->setPhysRegUsed(Reg);
+ PhysRegsUsed[Reg] = 0; // It is free and reserved now
+ AddToPhysRegsUseOrder(Reg);
+ for (const unsigned *AliasSet = RegInfo->getSubRegisters(Reg);
+ *AliasSet; ++AliasSet) {
+ if (PhysRegsUsed[*AliasSet] != -2) {
+ AddToPhysRegsUseOrder(*AliasSet);
+ PhysRegsUsed[*AliasSet] = 0; // It is free and reserved now
+ MF->setPhysRegUsed(*AliasSet);
+ }
+ }
+ }
+ }
+
+ // Otherwise, sequentially allocate each instruction in the MBB.
+ while (MII != MBB.end()) {
+ MachineInstr *MI = MII++;
+ const TargetInstrDescriptor &TID = TII.get(MI->getOpcode());
+ DEBUG(DOUT << "\nStarting RegAlloc of: " << *MI;
+ DOUT << " Regs have values: ";
+ for (unsigned i = 0; i != RegInfo->getNumRegs(); ++i)
+ if (PhysRegsUsed[i] != -1 && PhysRegsUsed[i] != -2)
+ DOUT << "[" << RegInfo->getName(i)
+ << ",%reg" << PhysRegsUsed[i] << "] ";
+ DOUT << "\n");
+
+ // Loop over the implicit uses, making sure that they are at the head of the
+ // use order list, so they don't get reallocated.
+ if (TID.ImplicitUses) {
+ for (const unsigned *ImplicitUses = TID.ImplicitUses;
+ *ImplicitUses; ++ImplicitUses)
+ MarkPhysRegRecentlyUsed(*ImplicitUses);
+ }
+
+ SmallVector<unsigned, 8> Kills;
+ for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
+ MachineOperand& MO = MI->getOperand(i);
+ if (MO.isRegister() && MO.isKill()) {
+ if (!MO.isImplicit())
+ Kills.push_back(MO.getReg());
+ else if (!isReadModWriteImplicitKill(MI, MO.getReg()))
+ // These are extra physical register kills when a sub-register
+ // is defined (def of a sub-register is a read/mod/write of the
+ // larger registers). Ignore.
+ Kills.push_back(MO.getReg());
+ }
+ }
+
+ // Get the used operands into registers. This has the potential to spill
+ // incoming values if we are out of registers. Note that we completely
+ // ignore physical register uses here. We assume that if an explicit
+ // physical register is referenced by the instruction, that it is guaranteed
+ // to be live-in, or the input is badly hosed.
+ //
+ for (unsigned i = 0; i != MI->getNumOperands(); ++i) {
+ MachineOperand& MO = MI->getOperand(i);
+ // here we are looking for only used operands (never def&use)
+ if (MO.isRegister() && !MO.isDef() && MO.getReg() && !MO.isImplicit() &&
+ MRegisterInfo::isVirtualRegister(MO.getReg()))
+ MI = reloadVirtReg(MBB, MI, i);
+ }
+
+ // If this instruction is the last user of this register, kill the
+ // value, freeing the register being used, so it doesn't need to be
+ // spilled to memory.
+ //
+ for (unsigned i = 0, e = Kills.size(); i != e; ++i) {
+ unsigned VirtReg = Kills[i];
+ unsigned PhysReg = VirtReg;
+ if (MRegisterInfo::isVirtualRegister(VirtReg)) {
+ // If the virtual register was never materialized into a register, it
+ // might not be in the map, but it won't hurt to zero it out anyway.
+ unsigned &PhysRegSlot = getVirt2PhysRegMapSlot(VirtReg);
+ PhysReg = PhysRegSlot;
+ PhysRegSlot = 0;
+ } else if (PhysRegsUsed[PhysReg] == -2) {
+ // Unallocatable register dead, ignore.
+ continue;
+ } else {
+ assert(!PhysRegsUsed[PhysReg] || PhysRegsUsed[PhysReg] == -1 &&
+ "Silently clearing a virtual register?");
+ }
+
+ if (PhysReg) {
+ DOUT << " Last use of " << RegInfo->getName(PhysReg)
+ << "[%reg" << VirtReg <<"], removing it from live set\n";
+ removePhysReg(PhysReg);
+ for (const unsigned *AliasSet = RegInfo->getSubRegisters(PhysReg);
+ *AliasSet; ++AliasSet) {
+ if (PhysRegsUsed[*AliasSet] != -2) {
+ DOUT << " Last use of "
+ << RegInfo->getName(*AliasSet)
+ << "[%reg" << VirtReg <<"], removing it from live set\n";
+ removePhysReg(*AliasSet);
+ }
+ }
+ }
+ }
+
+ // Loop over all of the operands of the instruction, spilling registers that
+ // are defined, and marking explicit destinations in the PhysRegsUsed map.
+ for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
+ MachineOperand& MO = MI->getOperand(i);
+ if (MO.isRegister() && MO.isDef() && !MO.isImplicit() && MO.getReg() &&
+ MRegisterInfo::isPhysicalRegister(MO.getReg())) {
+ unsigned Reg = MO.getReg();
+ if (PhysRegsUsed[Reg] == -2) continue; // Something like ESP.
+ // These are extra physical register defs when a sub-register
+ // is defined (def of a sub-register is a read/mod/write of the
+ // larger registers). Ignore.
+ if (isReadModWriteImplicitDef(MI, MO.getReg())) continue;
+
+ MF->setPhysRegUsed(Reg);
+ spillPhysReg(MBB, MI, Reg, true); // Spill any existing value in reg
+ PhysRegsUsed[Reg] = 0; // It is free and reserved now
+ AddToPhysRegsUseOrder(Reg);
+
+ for (const unsigned *AliasSet = RegInfo->getSubRegisters(Reg);
+ *AliasSet; ++AliasSet) {
+ if (PhysRegsUsed[*AliasSet] != -2) {
+ MF->setPhysRegUsed(*AliasSet);
+ PhysRegsUsed[*AliasSet] = 0; // It is free and reserved now
+ AddToPhysRegsUseOrder(*AliasSet);
+ }
+ }
+ }
+ }
+
+ // Loop over the implicit defs, spilling them as well.
+ if (TID.ImplicitDefs) {
+ for (const unsigned *ImplicitDefs = TID.ImplicitDefs;
+ *ImplicitDefs; ++ImplicitDefs) {
+ unsigned Reg = *ImplicitDefs;
+ if (PhysRegsUsed[Reg] != -2) {
+ spillPhysReg(MBB, MI, Reg, true);
+ AddToPhysRegsUseOrder(Reg);
+ PhysRegsUsed[Reg] = 0; // It is free and reserved now
+ }
+ MF->setPhysRegUsed(Reg);
+ for (const unsigned *AliasSet = RegInfo->getSubRegisters(Reg);
+ *AliasSet; ++AliasSet) {
+ if (PhysRegsUsed[*AliasSet] != -2) {
+ AddToPhysRegsUseOrder(*AliasSet);
+ PhysRegsUsed[*AliasSet] = 0; // It is free and reserved now
+ MF->setPhysRegUsed(*AliasSet);
+ }
+ }
+ }
+ }
+
+ SmallVector<unsigned, 8> DeadDefs;
+ for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
+ MachineOperand& MO = MI->getOperand(i);
+ if (MO.isRegister() && MO.isDead())
+ DeadDefs.push_back(MO.getReg());
+ }
+
+ // Okay, we have allocated all of the source operands and spilled any values
+ // that would be destroyed by defs of this instruction. Loop over the
+ // explicit defs and assign them to a register, spilling incoming values if
+ // we need to scavenge a register.
+ //
+ for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
+ MachineOperand& MO = MI->getOperand(i);
+ if (MO.isRegister() && MO.isDef() && MO.getReg() &&
+ MRegisterInfo::isVirtualRegister(MO.getReg())) {
+ unsigned DestVirtReg = MO.getReg();
+ unsigned DestPhysReg;
+
+ // If DestVirtReg already has a value, use it.
+ if (!(DestPhysReg = getVirt2PhysRegMapSlot(DestVirtReg)))
+ DestPhysReg = getReg(MBB, MI, DestVirtReg);
+ MF->setPhysRegUsed(DestPhysReg);
+ markVirtRegModified(DestVirtReg);
+ MI->getOperand(i).setReg(DestPhysReg); // Assign the output register
+ }
+ }
+
+ // If this instruction defines any registers that are immediately dead,
+ // kill them now.
+ //
+ for (unsigned i = 0, e = DeadDefs.size(); i != e; ++i) {
+ unsigned VirtReg = DeadDefs[i];
+ unsigned PhysReg = VirtReg;
+ if (MRegisterInfo::isVirtualRegister(VirtReg)) {
+ unsigned &PhysRegSlot = getVirt2PhysRegMapSlot(VirtReg);
+ PhysReg = PhysRegSlot;
+ assert(PhysReg != 0);
+ PhysRegSlot = 0;
+ } else if (PhysRegsUsed[PhysReg] == -2) {
+ // Unallocatable register dead, ignore.
+ continue;
+ }
+
+ if (PhysReg) {
+ DOUT << " Register " << RegInfo->getName(PhysReg)
+ << " [%reg" << VirtReg
+ << "] is never used, removing it frame live list\n";
+ removePhysReg(PhysReg);
+ for (const unsigned *AliasSet = RegInfo->getAliasSet(PhysReg);
+ *AliasSet; ++AliasSet) {
+ if (PhysRegsUsed[*AliasSet] != -2) {
+ DOUT << " Register " << RegInfo->getName(*AliasSet)
+ << " [%reg" << *AliasSet
+ << "] is never used, removing it frame live list\n";
+ removePhysReg(*AliasSet);
+ }
+ }
+ }
+ }
+
+ // Finally, if this is a noop copy instruction, zap it.
+ unsigned SrcReg, DstReg;
+ if (TII.isMoveInstr(*MI, SrcReg, DstReg) && SrcReg == DstReg) {
+ LV->removeVirtualRegistersKilled(MI);
+ LV->removeVirtualRegistersDead(MI);
+ MBB.erase(MI);
+ }
+ }
+
+ MachineBasicBlock::iterator MI = MBB.getFirstTerminator();
+
+ // Spill all physical registers holding virtual registers now.
+ for (unsigned i = 0, e = RegInfo->getNumRegs(); i != e; ++i)
+ if (PhysRegsUsed[i] != -1 && PhysRegsUsed[i] != -2)
+ if (unsigned VirtReg = PhysRegsUsed[i])
+ spillVirtReg(MBB, MI, VirtReg, i);
+ else
+ removePhysReg(i);
+
+#if 0
+ // This checking code is very expensive.
+ bool AllOk = true;
+ for (unsigned i = MRegisterInfo::FirstVirtualRegister,
+ e = MF->getSSARegMap()->getLastVirtReg(); i <= e; ++i)
+ if (unsigned PR = Virt2PhysRegMap[i]) {
+ cerr << "Register still mapped: " << i << " -> " << PR << "\n";
+ AllOk = false;
+ }
+ assert(AllOk && "Virtual registers still in phys regs?");
+#endif
+
+ // Clear any physical register which appear live at the end of the basic
+ // block, but which do not hold any virtual registers. e.g., the stack
+ // pointer.
+ PhysRegsUseOrder.clear();
+}
+
+
+/// runOnMachineFunction - Register allocate the whole function
+///
+bool RALocal::runOnMachineFunction(MachineFunction &Fn) {
+ DOUT << "Machine Function " << "\n";
+ MF = &Fn;
+ TM = &Fn.getTarget();
+ RegInfo = TM->getRegisterInfo();
+ LV = &getAnalysis<LiveVariables>();
+
+ PhysRegsUsed.assign(RegInfo->getNumRegs(), -1);
+
+ // At various places we want to efficiently check to see whether a register
+ // is allocatable. To handle this, we mark all unallocatable registers as
+ // being pinned down, permanently.
+ {
+ BitVector Allocable = RegInfo->getAllocatableSet(Fn);
+ for (unsigned i = 0, e = Allocable.size(); i != e; ++i)
+ if (!Allocable[i])
+ PhysRegsUsed[i] = -2; // Mark the reg unallocable.
+ }
+
+ // initialize the virtual->physical register map to have a 'null'
+ // mapping for all virtual registers
+ Virt2PhysRegMap.grow(MF->getSSARegMap()->getLastVirtReg());
+
+ // Loop over all of the basic blocks, eliminating virtual register references
+ for (MachineFunction::iterator MBB = Fn.begin(), MBBe = Fn.end();
+ MBB != MBBe; ++MBB)
+ AllocateBasicBlock(*MBB);
+
+ StackSlotForVirtReg.clear();
+ PhysRegsUsed.clear();
+ VirtRegModified.clear();
+ Virt2PhysRegMap.clear();
+ return true;
+}
+
+FunctionPass *llvm::createLocalRegisterAllocator() {
+ return new RALocal();
+}
diff --git a/lib/CodeGen/RegAllocSimple.cpp b/lib/CodeGen/RegAllocSimple.cpp
new file mode 100644
index 0000000..f49dd4c
--- /dev/null
+++ b/lib/CodeGen/RegAllocSimple.cpp
@@ -0,0 +1,253 @@
+//===-- RegAllocSimple.cpp - A simple generic register allocator ----------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file was developed by the LLVM research group and is distributed under
+// the University of Illinois Open Source License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements a simple register allocator. *Very* simple: It immediate
+// spills every value right after it is computed, and it reloads all used
+// operands from the spill area to temporary registers before each instruction.
+// It does not keep values in registers across instructions.
+//
+//===----------------------------------------------------------------------===//
+
+#define DEBUG_TYPE "regalloc"
+#include "llvm/CodeGen/Passes.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/MachineInstr.h"
+#include "llvm/CodeGen/SSARegMap.h"
+#include "llvm/CodeGen/MachineFrameInfo.h"
+#include "llvm/CodeGen/RegAllocRegistry.h"
+#include "llvm/Target/TargetInstrInfo.h"
+#include "llvm/Target/TargetMachine.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/Compiler.h"
+#include "llvm/ADT/Statistic.h"
+#include "llvm/ADT/STLExtras.h"
+using namespace llvm;
+
+STATISTIC(NumStores, "Number of stores added");
+STATISTIC(NumLoads , "Number of loads added");
+
+namespace {
+ static RegisterRegAlloc
+ simpleRegAlloc("simple", " simple register allocator",
+ createSimpleRegisterAllocator);
+
+ class VISIBILITY_HIDDEN RegAllocSimple : public MachineFunctionPass {
+ public:
+ static char ID;
+ RegAllocSimple() : MachineFunctionPass((intptr_t)&ID) {}
+ private:
+ MachineFunction *MF;
+ const TargetMachine *TM;
+ const MRegisterInfo *RegInfo;
+
+ // StackSlotForVirtReg - Maps SSA Regs => frame index on the stack where
+ // these values are spilled
+ std::map<unsigned, int> StackSlotForVirtReg;
+
+ // RegsUsed - Keep track of what registers are currently in use. This is a
+ // bitset.
+ std::vector<bool> RegsUsed;
+
+ // RegClassIdx - Maps RegClass => which index we can take a register
+ // from. Since this is a simple register allocator, when we need a register
+ // of a certain class, we just take the next available one.
+ std::map<const TargetRegisterClass*, unsigned> RegClassIdx;
+
+ public:
+ virtual const char *getPassName() const {
+ return "Simple Register Allocator";
+ }
+
+ /// runOnMachineFunction - Register allocate the whole function
+ bool runOnMachineFunction(MachineFunction &Fn);
+
+ virtual void getAnalysisUsage(AnalysisUsage &AU) const {
+ AU.addRequiredID(PHIEliminationID); // Eliminate PHI nodes
+ MachineFunctionPass::getAnalysisUsage(AU);
+ }
+ private:
+ /// AllocateBasicBlock - Register allocate the specified basic block.
+ void AllocateBasicBlock(MachineBasicBlock &MBB);
+
+ /// getStackSpaceFor - This returns the offset of the specified virtual
+ /// register on the stack, allocating space if necessary.
+ int getStackSpaceFor(unsigned VirtReg, const TargetRegisterClass *RC);
+
+ /// Given a virtual register, return a compatible physical register that is
+ /// currently unused.
+ ///
+ /// Side effect: marks that register as being used until manually cleared
+ ///
+ unsigned getFreeReg(unsigned virtualReg);
+
+ /// Moves value from memory into that register
+ unsigned reloadVirtReg(MachineBasicBlock &MBB,
+ MachineBasicBlock::iterator I, unsigned VirtReg);
+
+ /// Saves reg value on the stack (maps virtual register to stack value)
+ void spillVirtReg(MachineBasicBlock &MBB, MachineBasicBlock::iterator I,
+ unsigned VirtReg, unsigned PhysReg);
+ };
+ char RegAllocSimple::ID = 0;
+}
+
+/// getStackSpaceFor - This allocates space for the specified virtual
+/// register to be held on the stack.
+int RegAllocSimple::getStackSpaceFor(unsigned VirtReg,
+ const TargetRegisterClass *RC) {
+ // Find the location VirtReg would belong...
+ std::map<unsigned, int>::iterator I =
+ StackSlotForVirtReg.lower_bound(VirtReg);
+
+ if (I != StackSlotForVirtReg.end() && I->first == VirtReg)
+ return I->second; // Already has space allocated?
+
+ // Allocate a new stack object for this spill location...
+ int FrameIdx = MF->getFrameInfo()->CreateStackObject(RC->getSize(),
+ RC->getAlignment());
+
+ // Assign the slot...
+ StackSlotForVirtReg.insert(I, std::make_pair(VirtReg, FrameIdx));
+
+ return FrameIdx;
+}
+
+unsigned RegAllocSimple::getFreeReg(unsigned virtualReg) {
+ const TargetRegisterClass* RC = MF->getSSARegMap()->getRegClass(virtualReg);
+ TargetRegisterClass::iterator RI = RC->allocation_order_begin(*MF);
+ TargetRegisterClass::iterator RE = RC->allocation_order_end(*MF);
+
+ while (1) {
+ unsigned regIdx = RegClassIdx[RC]++;
+ assert(RI+regIdx != RE && "Not enough registers!");
+ unsigned PhysReg = *(RI+regIdx);
+
+ if (!RegsUsed[PhysReg]) {
+ MF->setPhysRegUsed(PhysReg);
+ return PhysReg;
+ }
+ }
+}
+
+unsigned RegAllocSimple::reloadVirtReg(MachineBasicBlock &MBB,
+ MachineBasicBlock::iterator I,
+ unsigned VirtReg) {
+ const TargetRegisterClass* RC = MF->getSSARegMap()->getRegClass(VirtReg);
+ int FrameIdx = getStackSpaceFor(VirtReg, RC);
+ unsigned PhysReg = getFreeReg(VirtReg);
+
+ // Add move instruction(s)
+ ++NumLoads;
+ RegInfo->loadRegFromStackSlot(MBB, I, PhysReg, FrameIdx, RC);
+ return PhysReg;
+}
+
+void RegAllocSimple::spillVirtReg(MachineBasicBlock &MBB,
+ MachineBasicBlock::iterator I,
+ unsigned VirtReg, unsigned PhysReg) {
+ const TargetRegisterClass* RC = MF->getSSARegMap()->getRegClass(VirtReg);
+ int FrameIdx = getStackSpaceFor(VirtReg, RC);
+
+ // Add move instruction(s)
+ ++NumStores;
+ RegInfo->storeRegToStackSlot(MBB, I, PhysReg, FrameIdx, RC);
+}
+
+
+void RegAllocSimple::AllocateBasicBlock(MachineBasicBlock &MBB) {
+ // loop over each instruction
+ for (MachineBasicBlock::iterator MI = MBB.begin(); MI != MBB.end(); ++MI) {
+ // Made to combat the incorrect allocation of r2 = add r1, r1
+ std::map<unsigned, unsigned> Virt2PhysRegMap;
+
+ RegsUsed.resize(RegInfo->getNumRegs());
+
+ // This is a preliminary pass that will invalidate any registers that are
+ // used by the instruction (including implicit uses).
+ unsigned Opcode = MI->getOpcode();
+ const TargetInstrDescriptor &Desc = TM->getInstrInfo()->get(Opcode);
+ const unsigned *Regs;
+ if (Desc.ImplicitUses) {
+ for (Regs = Desc.ImplicitUses; *Regs; ++Regs)
+ RegsUsed[*Regs] = true;
+ }
+
+ if (Desc.ImplicitDefs) {
+ for (Regs = Desc.ImplicitDefs; *Regs; ++Regs) {
+ RegsUsed[*Regs] = true;
+ MF->setPhysRegUsed(*Regs);
+ }
+ }
+
+ // Loop over uses, move from memory into registers.
+ for (int i = MI->getNumOperands() - 1; i >= 0; --i) {
+ MachineOperand &op = MI->getOperand(i);
+
+ if (op.isRegister() && op.getReg() &&
+ MRegisterInfo::isVirtualRegister(op.getReg())) {
+ unsigned virtualReg = (unsigned) op.getReg();
+ DOUT << "op: " << op << "\n";
+ DOUT << "\t inst[" << i << "]: ";
+ DEBUG(MI->print(*cerr.stream(), TM));
+
+ // make sure the same virtual register maps to the same physical
+ // register in any given instruction
+ unsigned physReg = Virt2PhysRegMap[virtualReg];
+ if (physReg == 0) {
+ if (op.isDef()) {
+ int TiedOp = MI->getInstrDescriptor()->findTiedToSrcOperand(i);
+ if (TiedOp == -1) {
+ physReg = getFreeReg(virtualReg);
+ } else {
+ // must be same register number as the source operand that is
+ // tied to. This maps a = b + c into b = b + c, and saves b into
+ // a's spot.
+ assert(MI->getOperand(TiedOp).isRegister() &&
+ MI->getOperand(TiedOp).getReg() &&
+ MI->getOperand(TiedOp).isUse() &&
+ "Two address instruction invalid!");
+
+ physReg = MI->getOperand(TiedOp).getReg();
+ }
+ spillVirtReg(MBB, next(MI), virtualReg, physReg);
+ } else {
+ physReg = reloadVirtReg(MBB, MI, virtualReg);
+ Virt2PhysRegMap[virtualReg] = physReg;
+ }
+ }
+ MI->getOperand(i).setReg(physReg);
+ DOUT << "virt: " << virtualReg << ", phys: " << op.getReg() << "\n";
+ }
+ }
+ RegClassIdx.clear();
+ RegsUsed.clear();
+ }
+}
+
+
+/// runOnMachineFunction - Register allocate the whole function
+///
+bool RegAllocSimple::runOnMachineFunction(MachineFunction &Fn) {
+ DOUT << "Machine Function\n";
+ MF = &Fn;
+ TM = &MF->getTarget();
+ RegInfo = TM->getRegisterInfo();
+
+ // Loop over all of the basic blocks, eliminating virtual register references
+ for (MachineFunction::iterator MBB = Fn.begin(), MBBe = Fn.end();
+ MBB != MBBe; ++MBB)
+ AllocateBasicBlock(*MBB);
+
+ StackSlotForVirtReg.clear();
+ return true;
+}
+
+FunctionPass *llvm::createSimpleRegisterAllocator() {
+ return new RegAllocSimple();
+}
diff --git a/lib/CodeGen/RegisterScavenging.cpp b/lib/CodeGen/RegisterScavenging.cpp
new file mode 100644
index 0000000..ae40e58
--- /dev/null
+++ b/lib/CodeGen/RegisterScavenging.cpp
@@ -0,0 +1,292 @@
+//===-- RegisterScavenging.cpp - Machine register scavenging --------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file was developed by the Evan Cheng and is distributed under the
+// University of Illinois Open Source License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements the machine register scavenger. It can provide
+// information such as unused register at any point in a machine basic block.
+// It also provides a mechanism to make registers availbale by evicting them
+// to spill slots.
+//
+//===----------------------------------------------------------------------===//
+
+#define DEBUG_TYPE "reg-scavenging"
+#include "llvm/CodeGen/RegisterScavenging.h"
+#include "llvm/CodeGen/MachineFunction.h"
+#include "llvm/CodeGen/MachineBasicBlock.h"
+#include "llvm/CodeGen/MachineInstr.h"
+#include "llvm/Target/MRegisterInfo.h"
+#include "llvm/Target/TargetInstrInfo.h"
+#include "llvm/Target/TargetMachine.h"
+#include "llvm/ADT/STLExtras.h"
+using namespace llvm;
+
+void RegScavenger::enterBasicBlock(MachineBasicBlock *mbb) {
+ const MachineFunction &MF = *mbb->getParent();
+ const TargetMachine &TM = MF.getTarget();
+ TII = TM.getInstrInfo();
+ RegInfo = TM.getRegisterInfo();
+
+ assert((NumPhysRegs == 0 || NumPhysRegs == RegInfo->getNumRegs()) &&
+ "Target changed?");
+
+ if (!MBB) {
+ NumPhysRegs = RegInfo->getNumRegs();
+ RegsAvailable.resize(NumPhysRegs);
+
+ // Create reserved registers bitvector.
+ ReservedRegs = RegInfo->getReservedRegs(MF);
+
+ // Create callee-saved registers bitvector.
+ CalleeSavedRegs.resize(NumPhysRegs);
+ const unsigned *CSRegs = RegInfo->getCalleeSavedRegs();
+ if (CSRegs != NULL)
+ for (unsigned i = 0; CSRegs[i]; ++i)
+ CalleeSavedRegs.set(CSRegs[i]);
+ }
+
+ MBB = mbb;
+ ScavengedReg = 0;
+ ScavengedRC = NULL;
+
+ // All registers started out unused.
+ RegsAvailable.set();
+
+ // Reserved registers are always used.
+ RegsAvailable ^= ReservedRegs;
+
+ // Live-in registers are in use.
+ if (!MBB->livein_empty())
+ for (MachineBasicBlock::const_livein_iterator I = MBB->livein_begin(),
+ E = MBB->livein_end(); I != E; ++I)
+ setUsed(*I);
+
+ Tracking = false;
+}
+
+void RegScavenger::restoreScavengedReg() {
+ if (!ScavengedReg)
+ return;
+
+ RegInfo->loadRegFromStackSlot(*MBB, MBBI, ScavengedReg,
+ ScavengingFrameIndex, ScavengedRC);
+ MachineBasicBlock::iterator II = prior(MBBI);
+ RegInfo->eliminateFrameIndex(II, 0, this);
+ setUsed(ScavengedReg);
+ ScavengedReg = 0;
+ ScavengedRC = NULL;
+}
+
+void RegScavenger::forward() {
+ // Move ptr forward.
+ if (!Tracking) {
+ MBBI = MBB->begin();
+ Tracking = true;
+ } else {
+ assert(MBBI != MBB->end() && "Already at the end of the basic block!");
+ MBBI = next(MBBI);
+ }
+
+ MachineInstr *MI = MBBI;
+
+ // Reaching a terminator instruction. Restore a scavenged register (which
+ // must be life out.
+ if (TII->isTerminatorInstr(MI->getOpcode()))
+ restoreScavengedReg();
+
+ // Process uses first.
+ BitVector ChangedRegs(NumPhysRegs);
+ for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
+ const MachineOperand &MO = MI->getOperand(i);
+ if (!MO.isReg() || !MO.isUse())
+ continue;
+ unsigned Reg = MO.getReg();
+ if (Reg == 0)
+ continue;
+ if (!isUsed(Reg)) {
+ // Register has been scavenged. Restore it!
+ if (Reg != ScavengedReg)
+ assert(false && "Using an undefined register!");
+ else
+ restoreScavengedReg();
+ }
+ if (MO.isKill() && !isReserved(Reg))
+ ChangedRegs.set(Reg);
+ }
+ // Change states of all registers after all the uses are processed to guard
+ // against multiple uses.
+ setUnused(ChangedRegs);
+
+ // Process defs.
+ const TargetInstrDescriptor *TID = MI->getInstrDescriptor();
+ for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
+ const MachineOperand &MO = MI->getOperand(i);
+ if (!MO.isReg() || !MO.isDef())
+ continue;
+ unsigned Reg = MO.getReg();
+ // If it's dead upon def, then it is now free.
+ if (MO.isDead()) {
+ setUnused(Reg);
+ continue;
+ }
+ // Skip two-address destination operand.
+ if (TID->findTiedToSrcOperand(i) != -1) {
+ assert(isUsed(Reg) && "Using an undefined register!");
+ continue;
+ }
+ assert((isUnused(Reg) || isReserved(Reg)) &&
+ "Re-defining a live register!");
+ setUsed(Reg);
+ }
+}
+
+void RegScavenger::backward() {
+ assert(Tracking && "Not tracking states!");
+ assert(MBBI != MBB->begin() && "Already at start of basic block!");
+ // Move ptr backward.
+ MBBI = prior(MBBI);
+
+ MachineInstr *MI = MBBI;
+ // Process defs first.
+ const TargetInstrDescriptor *TID = MI->getInstrDescriptor();
+ for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
+ const MachineOperand &MO = MI->getOperand(i);
+ if (!MO.isReg() || !MO.isDef())
+ continue;
+ // Skip two-address destination operand.
+ if (TID->findTiedToSrcOperand(i) != -1)
+ continue;
+ unsigned Reg = MO.getReg();
+ assert(isUsed(Reg));
+ if (!isReserved(Reg))
+ setUnused(Reg);
+ }
+
+ // Process uses.
+ BitVector ChangedRegs(NumPhysRegs);
+ for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
+ const MachineOperand &MO = MI->getOperand(i);
+ if (!MO.isReg() || !MO.isUse())
+ continue;
+ unsigned Reg = MO.getReg();
+ if (Reg == 0)
+ continue;
+ assert(isUnused(Reg) || isReserved(Reg));
+ ChangedRegs.set(Reg);
+ }
+ setUsed(ChangedRegs);
+}
+
+void RegScavenger::getRegsUsed(BitVector &used, bool includeReserved) {
+ if (includeReserved)
+ used = ~RegsAvailable;
+ else
+ used = ~RegsAvailable & ~ReservedRegs;
+}
+
+/// CreateRegClassMask - Set the bits that represent the registers in the
+/// TargetRegisterClass.
+static void CreateRegClassMask(const TargetRegisterClass *RC, BitVector &Mask) {
+ for (TargetRegisterClass::iterator I = RC->begin(), E = RC->end(); I != E;
+ ++I)
+ Mask.set(*I);
+}
+
+unsigned RegScavenger::FindUnusedReg(const TargetRegisterClass *RegClass,
+ const BitVector &Candidates) const {
+ // Mask off the registers which are not in the TargetRegisterClass.
+ BitVector RegsAvailableCopy(NumPhysRegs, false);
+ CreateRegClassMask(RegClass, RegsAvailableCopy);
+ RegsAvailableCopy &= RegsAvailable;
+
+ // Restrict the search to candidates.
+ RegsAvailableCopy &= Candidates;
+
+ // Returns the first unused (bit is set) register, or 0 is none is found.
+ int Reg = RegsAvailableCopy.find_first();
+ return (Reg == -1) ? 0 : Reg;
+}
+
+unsigned RegScavenger::FindUnusedReg(const TargetRegisterClass *RegClass,
+ bool ExCalleeSaved) const {
+ // Mask off the registers which are not in the TargetRegisterClass.
+ BitVector RegsAvailableCopy(NumPhysRegs, false);
+ CreateRegClassMask(RegClass, RegsAvailableCopy);
+ RegsAvailableCopy &= RegsAvailable;
+
+ // If looking for a non-callee-saved register, mask off all the callee-saved
+ // registers.
+ if (ExCalleeSaved)
+ RegsAvailableCopy &= ~CalleeSavedRegs;
+
+ // Returns the first unused (bit is set) register, or 0 is none is found.
+ int Reg = RegsAvailableCopy.find_first();
+ return (Reg == -1) ? 0 : Reg;
+}
+
+/// calcDistanceToUse - Calculate the distance to the first use of the
+/// specified register.
+static unsigned calcDistanceToUse(MachineBasicBlock *MBB,
+ MachineBasicBlock::iterator I, unsigned Reg) {
+ unsigned Dist = 0;
+ I = next(I);
+ while (I != MBB->end()) {
+ Dist++;
+ if (I->findRegisterUseOperandIdx(Reg) != -1)
+ return Dist;
+ I = next(I);
+ }
+ return Dist + 1;
+}
+
+unsigned RegScavenger::scavengeRegister(const TargetRegisterClass *RC,
+ MachineBasicBlock::iterator I,
+ int SPAdj) {
+ assert(ScavengingFrameIndex >= 0 &&
+ "Cannot scavenge a register without an emergency spill slot!");
+
+ // Mask off the registers which are not in the TargetRegisterClass.
+ BitVector Candidates(NumPhysRegs, false);
+ CreateRegClassMask(RC, Candidates);
+ Candidates ^= ReservedRegs; // Do not include reserved registers.
+
+ // Exclude all the registers being used by the instruction.
+ for (unsigned i = 0, e = I->getNumOperands(); i != e; ++i) {
+ MachineOperand &MO = I->getOperand(i);
+ if (MO.isReg())
+ Candidates.reset(MO.getReg());
+ }
+
+ // Find the register whose use is furtherest aaway.
+ unsigned SReg = 0;
+ unsigned MaxDist = 0;
+ int Reg = Candidates.find_first();
+ while (Reg != -1) {
+ unsigned Dist = calcDistanceToUse(MBB, I, Reg);
+ if (Dist >= MaxDist) {
+ MaxDist = Dist;
+ SReg = Reg;
+ }
+ Reg = Candidates.find_next(Reg);
+ }
+
+ if (ScavengedReg != 0) {
+ // First restore previously scavenged register.
+ RegInfo->loadRegFromStackSlot(*MBB, I, ScavengedReg,
+ ScavengingFrameIndex, ScavengedRC);
+ MachineBasicBlock::iterator II = prior(I);
+ RegInfo->eliminateFrameIndex(II, SPAdj, this);
+ }
+
+ RegInfo->storeRegToStackSlot(*MBB, I, SReg, ScavengingFrameIndex, RC);
+ MachineBasicBlock::iterator II = prior(I);
+ RegInfo->eliminateFrameIndex(II, SPAdj, this);
+ ScavengedReg = SReg;
+ ScavengedRC = RC;
+
+ return SReg;
+}
diff --git a/lib/CodeGen/SelectionDAG/CallingConvLower.cpp b/lib/CodeGen/SelectionDAG/CallingConvLower.cpp
new file mode 100644
index 0000000..defbe34
--- /dev/null
+++ b/lib/CodeGen/SelectionDAG/CallingConvLower.cpp
@@ -0,0 +1,102 @@
+//===-- llvm/CallingConvLower.cpp - Calling Conventions -------------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file was developed by Chris Lattner and is distributed under
+// the University of Illinois Open Source License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements the CCState class, used for lowering and implementing
+// calling conventions.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/CodeGen/CallingConvLower.h"
+#include "llvm/CodeGen/SelectionDAGNodes.h"
+#include "llvm/Target/MRegisterInfo.h"
+#include "llvm/Target/TargetMachine.h"
+using namespace llvm;
+
+CCState::CCState(unsigned CC, bool isVarArg, const TargetMachine &tm,
+ SmallVector<CCValAssign, 16> &locs)
+ : CallingConv(CC), IsVarArg(isVarArg), TM(tm),
+ MRI(*TM.getRegisterInfo()), Locs(locs) {
+ // No stack is used.
+ StackOffset = 0;
+
+ UsedRegs.resize(MRI.getNumRegs());
+}
+
+
+/// MarkAllocated - Mark a register and all of its aliases as allocated.
+void CCState::MarkAllocated(unsigned Reg) {
+ UsedRegs[Reg/32] |= 1 << (Reg&31);
+
+ if (const unsigned *RegAliases = MRI.getAliasSet(Reg))
+ for (; (Reg = *RegAliases); ++RegAliases)
+ UsedRegs[Reg/32] |= 1 << (Reg&31);
+}
+
+/// AnalyzeFormalArguments - Analyze an ISD::FORMAL_ARGUMENTS node,
+/// incorporating info about the formals into this state.
+void CCState::AnalyzeFormalArguments(SDNode *TheArgs, CCAssignFn Fn) {
+ unsigned NumArgs = TheArgs->getNumValues()-1;
+
+ for (unsigned i = 0; i != NumArgs; ++i) {
+ MVT::ValueType ArgVT = TheArgs->getValueType(i);
+ SDOperand FlagOp = TheArgs->getOperand(3+i);
+ unsigned ArgFlags = cast<ConstantSDNode>(FlagOp)->getValue();
+ if (Fn(i, ArgVT, ArgVT, CCValAssign::Full, ArgFlags, *this)) {
+ cerr << "Formal argument #" << i << " has unhandled type "
+ << MVT::getValueTypeString(ArgVT) << "\n";
+ abort();
+ }
+ }
+}
+
+/// AnalyzeReturn - Analyze the returned values of an ISD::RET node,
+/// incorporating info about the result values into this state.
+void CCState::AnalyzeReturn(SDNode *TheRet, CCAssignFn Fn) {
+ // Determine which register each value should be copied into.
+ for (unsigned i = 0, e = TheRet->getNumOperands() / 2; i != e; ++i) {
+ MVT::ValueType VT = TheRet->getOperand(i*2+1).getValueType();
+ if (Fn(i, VT, VT, CCValAssign::Full,
+ cast<ConstantSDNode>(TheRet->getOperand(i*2+2))->getValue(), *this)){
+ cerr << "Return operand #" << i << " has unhandled type "
+ << MVT::getValueTypeString(VT) << "\n";
+ abort();
+ }
+ }
+}
+
+
+/// AnalyzeCallOperands - Analyze an ISD::CALL node, incorporating info
+/// about the passed values into this state.
+void CCState::AnalyzeCallOperands(SDNode *TheCall, CCAssignFn Fn) {
+ unsigned NumOps = (TheCall->getNumOperands() - 5) / 2;
+ for (unsigned i = 0; i != NumOps; ++i) {
+ MVT::ValueType ArgVT = TheCall->getOperand(5+2*i).getValueType();
+ SDOperand FlagOp = TheCall->getOperand(5+2*i+1);
+ unsigned ArgFlags =cast<ConstantSDNode>(FlagOp)->getValue();
+ if (Fn(i, ArgVT, ArgVT, CCValAssign::Full, ArgFlags, *this)) {
+ cerr << "Call operand #" << i << " has unhandled type "
+ << MVT::getValueTypeString(ArgVT) << "\n";
+ abort();
+ }
+ }
+}
+
+/// AnalyzeCallResult - Analyze the return values of an ISD::CALL node,
+/// incorporating info about the passed values into this state.
+void CCState::AnalyzeCallResult(SDNode *TheCall, CCAssignFn Fn) {
+ for (unsigned i = 0, e = TheCall->getNumValues() - 1; i != e; ++i) {
+ MVT::ValueType VT = TheCall->getValueType(i);
+ if (Fn(i, VT, VT, CCValAssign::Full, 0, *this)) {
+ cerr << "Call result #" << i << " has unhandled type "
+ << MVT::getValueTypeString(VT) << "\n";
+ abort();
+ }
+ }
+}
+
diff --git a/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
new file mode 100644
index 0000000..22c6e6b
--- /dev/null
+++ b/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -0,0 +1,4749 @@
+//===-- DAGCombiner.cpp - Implement a DAG node combiner -------------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file was developed by Nate Begeman and is distributed under the
+// University of Illinois Open Source License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This pass combines dag nodes to form fewer, simpler DAG nodes. It can be run
+// both before and after the DAG is legalized.
+//
+// FIXME: Missing folds
+// sdiv, udiv, srem, urem (X, const) where X is an integer can be expanded into
+// a sequence of multiplies, shifts, and adds. This should be controlled by
+// some kind of hint from the target that int div is expensive.
+// various folds of mulh[s,u] by constants such as -1, powers of 2, etc.
+//
+// FIXME: select C, pow2, pow2 -> something smart
+// FIXME: trunc(select X, Y, Z) -> select X, trunc(Y), trunc(Z)
+// FIXME: Dead stores -> nuke
+// FIXME: shr X, (and Y,31) -> shr X, Y (TRICKY!)
+// FIXME: mul (x, const) -> shifts + adds
+// FIXME: undef values
+// FIXME: divide by zero is currently left unfolded. do we want to turn this
+// into an undef?
+// FIXME: select ne (select cc, 1, 0), 0, true, false -> select cc, true, false
+//
+//===----------------------------------------------------------------------===//
+
+#define DEBUG_TYPE "dagcombine"
+#include "llvm/CodeGen/SelectionDAG.h"
+#include "llvm/Analysis/AliasAnalysis.h"
+#include "llvm/Target/TargetData.h"
+#include "llvm/Target/TargetLowering.h"
+#include "llvm/Target/TargetMachine.h"
+#include "llvm/Target/TargetOptions.h"
+#include "llvm/ADT/SmallPtrSet.h"
+#include "llvm/ADT/Statistic.h"
+#include "llvm/Support/Compiler.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/MathExtras.h"
+#include <algorithm>
+using namespace llvm;
+
+STATISTIC(NodesCombined , "Number of dag nodes combined");
+STATISTIC(PreIndexedNodes , "Number of pre-indexed nodes created");
+STATISTIC(PostIndexedNodes, "Number of post-indexed nodes created");
+
+namespace {
+#ifndef NDEBUG
+ static cl::opt<bool>
+ ViewDAGCombine1("view-dag-combine1-dags", cl::Hidden,
+ cl::desc("Pop up a window to show dags before the first "
+ "dag combine pass"));
+ static cl::opt<bool>
+ ViewDAGCombine2("view-dag-combine2-dags", cl::Hidden,
+ cl::desc("Pop up a window to show dags before the second "
+ "dag combine pass"));
+#else
+ static const bool ViewDAGCombine1 = false;
+ static const bool ViewDAGCombine2 = false;
+#endif
+
+ static cl::opt<bool>
+ CombinerAA("combiner-alias-analysis", cl::Hidden,
+ cl::desc("Turn on alias analysis during testing"));
+
+ static cl::opt<bool>
+ CombinerGlobalAA("combiner-global-alias-analysis", cl::Hidden,
+ cl::desc("Include global information in alias analysis"));
+
+//------------------------------ DAGCombiner ---------------------------------//
+
+ class VISIBILITY_HIDDEN DAGCombiner {
+ SelectionDAG &DAG;
+ TargetLowering &TLI;
+ bool AfterLegalize;
+
+ // Worklist of all of the nodes that need to be simplified.
+ std::vector<SDNode*> WorkList;
+
+ // AA - Used for DAG load/store alias analysis.
+ AliasAnalysis &AA;
+
+ /// AddUsersToWorkList - When an instruction is simplified, add all users of
+ /// the instruction to the work lists because they might get more simplified
+ /// now.
+ ///
+ void AddUsersToWorkList(SDNode *N) {
+ for (SDNode::use_iterator UI = N->use_begin(), UE = N->use_end();
+ UI != UE; ++UI)
+ AddToWorkList(*UI);
+ }
+
+ /// removeFromWorkList - remove all instances of N from the worklist.
+ ///
+ void removeFromWorkList(SDNode *N) {
+ WorkList.erase(std::remove(WorkList.begin(), WorkList.end(), N),
+ WorkList.end());
+ }
+
+ public:
+ /// AddToWorkList - Add to the work list making sure it's instance is at the
+ /// the back (next to be processed.)
+ void AddToWorkList(SDNode *N) {
+ removeFromWorkList(N);
+ WorkList.push_back(N);
+ }
+
+ SDOperand CombineTo(SDNode *N, const SDOperand *To, unsigned NumTo,
+ bool AddTo = true) {
+ assert(N->getNumValues() == NumTo && "Broken CombineTo call!");
+ ++NodesCombined;
+ DOUT << "\nReplacing.1 "; DEBUG(N->dump(&DAG));
+ DOUT << "\nWith: "; DEBUG(To[0].Val->dump(&DAG));
+ DOUT << " and " << NumTo-1 << " other values\n";
+ std::vector<SDNode*> NowDead;
+ DAG.ReplaceAllUsesWith(N, To, &NowDead);
+
+ if (AddTo) {
+ // Push the new nodes and any users onto the worklist
+ for (unsigned i = 0, e = NumTo; i != e; ++i) {
+ AddToWorkList(To[i].Val);
+ AddUsersToWorkList(To[i].Val);
+ }
+ }
+
+ // Nodes can be reintroduced into the worklist. Make sure we do not
+ // process a node that has been replaced.
+ removeFromWorkList(N);
+ for (unsigned i = 0, e = NowDead.size(); i != e; ++i)
+ removeFromWorkList(NowDead[i]);
+
+ // Finally, since the node is now dead, remove it from the graph.
+ DAG.DeleteNode(N);
+ return SDOperand(N, 0);
+ }
+
+ SDOperand CombineTo(SDNode *N, SDOperand Res, bool AddTo = true) {
+ return CombineTo(N, &Res, 1, AddTo);
+ }
+
+ SDOperand CombineTo(SDNode *N, SDOperand Res0, SDOperand Res1,
+ bool AddTo = true) {
+ SDOperand To[] = { Res0, Res1 };
+ return CombineTo(N, To, 2, AddTo);
+ }
+ private:
+
+ /// SimplifyDemandedBits - Check the specified integer node value to see if
+ /// it can be simplified or if things it uses can be simplified by bit
+ /// propagation. If so, return true.
+ bool SimplifyDemandedBits(SDOperand Op) {
+ TargetLowering::TargetLoweringOpt TLO(DAG);
+ uint64_t KnownZero, KnownOne;
+ uint64_t Demanded = MVT::getIntVTBitMask(Op.getValueType());
+ if (!TLI.SimplifyDemandedBits(Op, Demanded, KnownZero, KnownOne, TLO))
+ return false;
+
+ // Revisit the node.
+ AddToWorkList(Op.Val);
+
+ // Replace the old value with the new one.
+ ++NodesCombined;
+ DOUT << "\nReplacing.2 "; DEBUG(TLO.Old.Val->dump(&DAG));
+ DOUT << "\nWith: "; DEBUG(TLO.New.Val->dump(&DAG));
+ DOUT << '\n';
+
+ std::vector<SDNode*> NowDead;
+ DAG.ReplaceAllUsesOfValueWith(TLO.Old, TLO.New, NowDead);
+
+ // Push the new node and any (possibly new) users onto the worklist.
+ AddToWorkList(TLO.New.Val);
+ AddUsersToWorkList(TLO.New.Val);
+
+ // Nodes can end up on the worklist more than once. Make sure we do
+ // not process a node that has been replaced.
+ for (unsigned i = 0, e = NowDead.size(); i != e; ++i)
+ removeFromWorkList(NowDead[i]);
+
+ // Finally, if the node is now dead, remove it from the graph. The node
+ // may not be dead if the replacement process recursively simplified to
+ // something else needing this node.
+ if (TLO.Old.Val->use_empty()) {
+ removeFromWorkList(TLO.Old.Val);
+
+ // If the operands of this node are only used by the node, they will now
+ // be dead. Make sure to visit them first to delete dead nodes early.
+ for (unsigned i = 0, e = TLO.Old.Val->getNumOperands(); i != e; ++i)
+ if (TLO.Old.Val->getOperand(i).Val->hasOneUse())
+ AddToWorkList(TLO.Old.Val->getOperand(i).Val);
+
+ DAG.DeleteNode(TLO.Old.Val);
+ }
+ return true;
+ }
+
+ bool CombineToPreIndexedLoadStore(SDNode *N);
+ bool CombineToPostIndexedLoadStore(SDNode *N);
+
+
+ /// visit - call the node-specific routine that knows how to fold each
+ /// particular type of node.
+ SDOperand visit(SDNode *N);
+
+ // Visitation implementation - Implement dag node combining for different
+ // node types. The semantics are as follows:
+ // Return Value:
+ // SDOperand.Val == 0 - No change was made
+ // SDOperand.Val == N - N was replaced, is dead, and is already handled.
+ // otherwise - N should be replaced by the returned Operand.
+ //
+ SDOperand visitTokenFactor(SDNode *N);
+ SDOperand visitADD(SDNode *N);
+ SDOperand visitSUB(SDNode *N);
+ SDOperand visitADDC(SDNode *N);
+ SDOperand visitADDE(SDNode *N);
+ SDOperand visitMUL(SDNode *N);
+ SDOperand visitSDIV(SDNode *N);
+ SDOperand visitUDIV(SDNode *N);
+ SDOperand visitSREM(SDNode *N);
+ SDOperand visitUREM(SDNode *N);
+ SDOperand visitMULHU(SDNode *N);
+ SDOperand visitMULHS(SDNode *N);
+ SDOperand visitAND(SDNode *N);
+ SDOperand visitOR(SDNode *N);
+ SDOperand visitXOR(SDNode *N);
+ SDOperand SimplifyVBinOp(SDNode *N);
+ SDOperand visitSHL(SDNode *N);
+ SDOperand visitSRA(SDNode *N);
+ SDOperand visitSRL(SDNode *N);
+ SDOperand visitCTLZ(SDNode *N);
+ SDOperand visitCTTZ(SDNode *N);
+ SDOperand visitCTPOP(SDNode *N);
+ SDOperand visitSELECT(SDNode *N);
+ SDOperand visitSELECT_CC(SDNode *N);
+ SDOperand visitSETCC(SDNode *N);
+ SDOperand visitSIGN_EXTEND(SDNode *N);
+ SDOperand visitZERO_EXTEND(SDNode *N);
+ SDOperand visitANY_EXTEND(SDNode *N);
+ SDOperand visitSIGN_EXTEND_INREG(SDNode *N);
+ SDOperand visitTRUNCATE(SDNode *N);
+ SDOperand visitBIT_CONVERT(SDNode *N);
+ SDOperand visitFADD(SDNode *N);
+ SDOperand visitFSUB(SDNode *N);
+ SDOperand visitFMUL(SDNode *N);
+ SDOperand visitFDIV(SDNode *N);
+ SDOperand visitFREM(SDNode *N);
+ SDOperand visitFCOPYSIGN(SDNode *N);
+ SDOperand visitSINT_TO_FP(SDNode *N);
+ SDOperand visitUINT_TO_FP(SDNode *N);
+ SDOperand visitFP_TO_SINT(SDNode *N);
+ SDOperand visitFP_TO_UINT(SDNode *N);
+ SDOperand visitFP_ROUND(SDNode *N);
+ SDOperand visitFP_ROUND_INREG(SDNode *N);
+ SDOperand visitFP_EXTEND(SDNode *N);
+ SDOperand visitFNEG(SDNode *N);
+ SDOperand visitFABS(SDNode *N);
+ SDOperand visitBRCOND(SDNode *N);
+ SDOperand visitBR_CC(SDNode *N);
+ SDOperand visitLOAD(SDNode *N);
+ SDOperand visitSTORE(SDNode *N);
+ SDOperand visitINSERT_VECTOR_ELT(SDNode *N);
+ SDOperand visitBUILD_VECTOR(SDNode *N);
+ SDOperand visitCONCAT_VECTORS(SDNode *N);
+ SDOperand visitVECTOR_SHUFFLE(SDNode *N);
+
+ SDOperand XformToShuffleWithZero(SDNode *N);
+ SDOperand ReassociateOps(unsigned Opc, SDOperand LHS, SDOperand RHS);
+
+ bool SimplifySelectOps(SDNode *SELECT, SDOperand LHS, SDOperand RHS);
+ SDOperand SimplifyBinOpWithSameOpcodeHands(SDNode *N);
+ SDOperand SimplifySelect(SDOperand N0, SDOperand N1, SDOperand N2);
+ SDOperand SimplifySelectCC(SDOperand N0, SDOperand N1, SDOperand N2,
+ SDOperand N3, ISD::CondCode CC,
+ bool NotExtCompare = false);
+ SDOperand SimplifySetCC(MVT::ValueType VT, SDOperand N0, SDOperand N1,
+ ISD::CondCode Cond, bool foldBooleans = true);
+ SDOperand ConstantFoldBIT_CONVERTofBUILD_VECTOR(SDNode *, MVT::ValueType);
+ SDOperand BuildSDIV(SDNode *N);
+ SDOperand BuildUDIV(SDNode *N);
+ SDNode *MatchRotate(SDOperand LHS, SDOperand RHS);
+ SDOperand ReduceLoadWidth(SDNode *N);
+
+ /// GatherAllAliases - Walk up chain skipping non-aliasing memory nodes,
+ /// looking for aliasing nodes and adding them to the Aliases vector.
+ void GatherAllAliases(SDNode *N, SDOperand OriginalChain,
+ SmallVector<SDOperand, 8> &Aliases);
+
+ /// isAlias - Return true if there is any possibility that the two addresses
+ /// overlap.
+ bool isAlias(SDOperand Ptr1, int64_t Size1,
+ const Value *SrcValue1, int SrcValueOffset1,
+ SDOperand Ptr2, int64_t Size2,
+ const Value *SrcValue2, int SrcValueOffset2);
+
+ /// FindAliasInfo - Extracts the relevant alias information from the memory
+ /// node. Returns true if the operand was a load.
+ bool FindAliasInfo(SDNode *N,
+ SDOperand &Ptr, int64_t &Size,
+ const Value *&SrcValue, int &SrcValueOffset);
+
+ /// FindBetterChain - Walk up chain skipping non-aliasing memory nodes,
+ /// looking for a better chain (aliasing node.)
+ SDOperand FindBetterChain(SDNode *N, SDOperand Chain);
+
+public:
+ DAGCombiner(SelectionDAG &D, AliasAnalysis &A)
+ : DAG(D),
+ TLI(D.getTargetLoweringInfo()),
+ AfterLegalize(false),
+ AA(A) {}
+
+ /// Run - runs the dag combiner on all nodes in the work list
+ void Run(bool RunningAfterLegalize);
+ };
+}
+
+//===----------------------------------------------------------------------===//
+// TargetLowering::DAGCombinerInfo implementation
+//===----------------------------------------------------------------------===//
+
+void TargetLowering::DAGCombinerInfo::AddToWorklist(SDNode *N) {
+ ((DAGCombiner*)DC)->AddToWorkList(N);
+}
+
+SDOperand TargetLowering::DAGCombinerInfo::
+CombineTo(SDNode *N, const std::vector<SDOperand> &To) {
+ return ((DAGCombiner*)DC)->CombineTo(N, &To[0], To.size());
+}
+
+SDOperand TargetLowering::DAGCombinerInfo::
+CombineTo(SDNode *N, SDOperand Res) {
+ return ((DAGCombiner*)DC)->CombineTo(N, Res);
+}
+
+
+SDOperand TargetLowering::DAGCombinerInfo::
+CombineTo(SDNode *N, SDOperand Res0, SDOperand Res1) {
+ return ((DAGCombiner*)DC)->CombineTo(N, Res0, Res1);
+}
+
+
+//===----------------------------------------------------------------------===//
+// Helper Functions
+//===----------------------------------------------------------------------===//
+
+/// isNegatibleForFree - Return 1 if we can compute the negated form of the
+/// specified expression for the same cost as the expression itself, or 2 if we
+/// can compute the negated form more cheaply than the expression itself.
+static char isNegatibleForFree(SDOperand Op, unsigned Depth = 0) {
+ // fneg is removable even if it has multiple uses.
+ if (Op.getOpcode() == ISD::FNEG) return 2;
+
+ // Don't allow anything with multiple uses.
+ if (!Op.hasOneUse()) return 0;
+
+ // Don't recurse exponentially.
+ if (Depth > 6) return 0;
+
+ switch (Op.getOpcode()) {
+ default: return false;
+ case ISD::ConstantFP:
+ return 1;
+ case ISD::FADD:
+ // FIXME: determine better conditions for this xform.
+ if (!UnsafeFPMath) return 0;
+
+ // -(A+B) -> -A - B
+ if (char V = isNegatibleForFree(Op.getOperand(0), Depth+1))
+ return V;
+ // -(A+B) -> -B - A
+ return isNegatibleForFree(Op.getOperand(1), Depth+1);
+ case ISD::FSUB:
+ // We can't turn -(A-B) into B-A when we honor signed zeros.
+ if (!UnsafeFPMath) return 0;
+
+ // -(A-B) -> B-A
+ return 1;
+
+ case ISD::FMUL:
+ case ISD::FDIV:
+ if (HonorSignDependentRoundingFPMath()) return 0;
+
+ // -(X*Y) -> (-X * Y) or (X*-Y)
+ if (char V = isNegatibleForFree(Op.getOperand(0), Depth+1))
+ return V;
+
+ return isNegatibleForFree(Op.getOperand(1), Depth+1);
+
+ case ISD::FP_EXTEND:
+ case ISD::FP_ROUND:
+ case ISD::FSIN:
+ return isNegatibleForFree(Op.getOperand(0), Depth+1);
+ }
+}
+
+/// GetNegatedExpression - If isNegatibleForFree returns true, this function
+/// returns the newly negated expression.
+static SDOperand GetNegatedExpression(SDOperand Op, SelectionDAG &DAG,
+ unsigned Depth = 0) {
+ // fneg is removable even if it has multiple uses.
+ if (Op.getOpcode() == ISD::FNEG) return Op.getOperand(0);
+
+ // Don't allow anything with multiple uses.
+ assert(Op.hasOneUse() && "Unknown reuse!");
+
+ assert(Depth <= 6 && "GetNegatedExpression doesn't match isNegatibleForFree");
+ switch (Op.getOpcode()) {
+ default: assert(0 && "Unknown code");
+ case ISD::ConstantFP:
+ return DAG.getConstantFP(-cast<ConstantFPSDNode>(Op)->getValue(),
+ Op.getValueType());
+ case ISD::FADD:
+ // FIXME: determine better conditions for this xform.
+ assert(UnsafeFPMath);
+
+ // -(A+B) -> -A - B
+ if (isNegatibleForFree(Op.getOperand(0), Depth+1))
+ return DAG.getNode(ISD::FSUB, Op.getValueType(),
+ GetNegatedExpression(Op.getOperand(0), DAG, Depth+1),
+ Op.getOperand(1));
+ // -(A+B) -> -B - A
+ return DAG.getNode(ISD::FSUB, Op.getValueType(),
+ GetNegatedExpression(Op.getOperand(1), DAG, Depth+1),
+ Op.getOperand(0));
+ case ISD::FSUB:
+ // We can't turn -(A-B) into B-A when we honor signed zeros.
+ assert(UnsafeFPMath);
+
+ // -(0-B) -> B
+ if (ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(Op.getOperand(0)))
+ if (N0CFP->getValue() == 0.0)
+ return Op.getOperand(1);
+
+ // -(A-B) -> B-A
+ return DAG.getNode(ISD::FSUB, Op.getValueType(), Op.getOperand(1),
+ Op.getOperand(0));
+
+ case ISD::FMUL:
+ case ISD::FDIV:
+ assert(!HonorSignDependentRoundingFPMath());
+
+ // -(X*Y) -> -X * Y
+ if (isNegatibleForFree(Op.getOperand(0), Depth+1))
+ return DAG.getNode(Op.getOpcode(), Op.getValueType(),
+ GetNegatedExpression(Op.getOperand(0), DAG, Depth+1),
+ Op.getOperand(1));
+
+ // -(X*Y) -> X * -Y
+ return DAG.getNode(Op.getOpcode(), Op.getValueType(),
+ Op.getOperand(0),
+ GetNegatedExpression(Op.getOperand(1), DAG, Depth+1));
+
+ case ISD::FP_EXTEND:
+ case ISD::FP_ROUND:
+ case ISD::FSIN:
+ return DAG.getNode(Op.getOpcode(), Op.getValueType(),
+ GetNegatedExpression(Op.getOperand(0), DAG, Depth+1));
+ }
+}
+
+
+// isSetCCEquivalent - Return true if this node is a setcc, or is a select_cc
+// that selects between the values 1 and 0, making it equivalent to a setcc.
+// Also, set the incoming LHS, RHS, and CC references to the appropriate
+// nodes based on the type of node we are checking. This simplifies life a
+// bit for the callers.
+static bool isSetCCEquivalent(SDOperand N, SDOperand &LHS, SDOperand &RHS,
+ SDOperand &CC) {
+ if (N.getOpcode() == ISD::SETCC) {
+ LHS = N.getOperand(0);
+ RHS = N.getOperand(1);
+ CC = N.getOperand(2);
+ return true;
+ }
+ if (N.getOpcode() == ISD::SELECT_CC &&
+ N.getOperand(2).getOpcode() == ISD::Constant &&
+ N.getOperand(3).getOpcode() == ISD::Constant &&
+ cast<ConstantSDNode>(N.getOperand(2))->getValue() == 1 &&
+ cast<ConstantSDNode>(N.getOperand(3))->isNullValue()) {
+ LHS = N.getOperand(0);
+ RHS = N.getOperand(1);
+ CC = N.getOperand(4);
+ return true;
+ }
+ return false;
+}
+
+// isOneUseSetCC - Return true if this is a SetCC-equivalent operation with only
+// one use. If this is true, it allows the users to invert the operation for
+// free when it is profitable to do so.
+static bool isOneUseSetCC(SDOperand N) {
+ SDOperand N0, N1, N2;
+ if (isSetCCEquivalent(N, N0, N1, N2) && N.Val->hasOneUse())
+ return true;
+ return false;
+}
+
+SDOperand DAGCombiner::ReassociateOps(unsigned Opc, SDOperand N0, SDOperand N1){
+ MVT::ValueType VT = N0.getValueType();
+ // reassoc. (op (op x, c1), y) -> (op (op x, y), c1) iff x+c1 has one use
+ // reassoc. (op (op x, c1), c2) -> (op x, (op c1, c2))
+ if (N0.getOpcode() == Opc && isa<ConstantSDNode>(N0.getOperand(1))) {
+ if (isa<ConstantSDNode>(N1)) {
+ SDOperand OpNode = DAG.getNode(Opc, VT, N0.getOperand(1), N1);
+ AddToWorkList(OpNode.Val);
+ return DAG.getNode(Opc, VT, OpNode, N0.getOperand(0));
+ } else if (N0.hasOneUse()) {
+ SDOperand OpNode = DAG.getNode(Opc, VT, N0.getOperand(0), N1);
+ AddToWorkList(OpNode.Val);
+ return DAG.getNode(Opc, VT, OpNode, N0.getOperand(1));
+ }
+ }
+ // reassoc. (op y, (op x, c1)) -> (op (op x, y), c1) iff x+c1 has one use
+ // reassoc. (op c2, (op x, c1)) -> (op x, (op c1, c2))
+ if (N1.getOpcode() == Opc && isa<ConstantSDNode>(N1.getOperand(1))) {
+ if (isa<ConstantSDNode>(N0)) {
+ SDOperand OpNode = DAG.getNode(Opc, VT, N1.getOperand(1), N0);
+ AddToWorkList(OpNode.Val);
+ return DAG.getNode(Opc, VT, OpNode, N1.getOperand(0));
+ } else if (N1.hasOneUse()) {
+ SDOperand OpNode = DAG.getNode(Opc, VT, N1.getOperand(0), N0);
+ AddToWorkList(OpNode.Val);
+ return DAG.getNode(Opc, VT, OpNode, N1.getOperand(1));
+ }
+ }
+ return SDOperand();
+}
+
+//===----------------------------------------------------------------------===//
+// Main DAG Combiner implementation
+//===----------------------------------------------------------------------===//
+
+void DAGCombiner::Run(bool RunningAfterLegalize) {
+ // set the instance variable, so that the various visit routines may use it.
+ AfterLegalize = RunningAfterLegalize;
+
+ // Add all the dag nodes to the worklist.
+ for (SelectionDAG::allnodes_iterator I = DAG.allnodes_begin(),
+ E = DAG.allnodes_end(); I != E; ++I)
+ WorkList.push_back(I);
+
+ // Create a dummy node (which is not added to allnodes), that adds a reference
+ // to the root node, preventing it from being deleted, and tracking any
+ // changes of the root.
+ HandleSDNode Dummy(DAG.getRoot());
+
+ // The root of the dag may dangle to deleted nodes until the dag combiner is
+ // done. Set it to null to avoid confusion.
+ DAG.setRoot(SDOperand());
+
+ /// DagCombineInfo - Expose the DAG combiner to the target combiner impls.
+ TargetLowering::DAGCombinerInfo
+ DagCombineInfo(DAG, !RunningAfterLegalize, false, this);
+
+ // while the worklist isn't empty, inspect the node on the end of it and
+ // try and combine it.
+ while (!WorkList.empty()) {
+ SDNode *N = WorkList.back();
+ WorkList.pop_back();
+
+ // If N has no uses, it is dead. Make sure to revisit all N's operands once
+ // N is deleted from the DAG, since they too may now be dead or may have a
+ // reduced number of uses, allowing other xforms.
+ if (N->use_empty() && N != &Dummy) {
+ for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i)
+ AddToWorkList(N->getOperand(i).Val);
+
+ DAG.DeleteNode(N);
+ continue;
+ }
+
+ SDOperand RV = visit(N);
+
+ // If nothing happened, try a target-specific DAG combine.
+ if (RV.Val == 0) {
+ assert(N->getOpcode() != ISD::DELETED_NODE &&
+ "Node was deleted but visit returned NULL!");
+ if (N->getOpcode() >= ISD::BUILTIN_OP_END ||
+ TLI.hasTargetDAGCombine((ISD::NodeType)N->getOpcode()))
+ RV = TLI.PerformDAGCombine(N, DagCombineInfo);
+ }
+
+ if (RV.Val) {
+ ++NodesCombined;
+ // If we get back the same node we passed in, rather than a new node or
+ // zero, we know that the node must have defined multiple values and
+ // CombineTo was used. Since CombineTo takes care of the worklist
+ // mechanics for us, we have no work to do in this case.
+ if (RV.Val != N) {
+ assert(N->getOpcode() != ISD::DELETED_NODE &&
+ RV.Val->getOpcode() != ISD::DELETED_NODE &&
+ "Node was deleted but visit returned new node!");
+
+ DOUT << "\nReplacing.3 "; DEBUG(N->dump(&DAG));
+ DOUT << "\nWith: "; DEBUG(RV.Val->dump(&DAG));
+ DOUT << '\n';
+ std::vector<SDNode*> NowDead;
+ if (N->getNumValues() == RV.Val->getNumValues())
+ DAG.ReplaceAllUsesWith(N, RV.Val, &NowDead);
+ else {
+ assert(N->getValueType(0) == RV.getValueType() && "Type mismatch");
+ SDOperand OpV = RV;
+ DAG.ReplaceAllUsesWith(N, &OpV, &NowDead);
+ }
+
+ // Push the new node and any users onto the worklist
+ AddToWorkList(RV.Val);
+ AddUsersToWorkList(RV.Val);
+
+ // Nodes can be reintroduced into the worklist. Make sure we do not
+ // process a node that has been replaced.
+ removeFromWorkList(N);
+ for (unsigned i = 0, e = NowDead.size(); i != e; ++i)
+ removeFromWorkList(NowDead[i]);
+
+ // Finally, since the node is now dead, remove it from the graph.
+ DAG.DeleteNode(N);
+ }
+ }
+ }
+
+ // If the root changed (e.g. it was a dead load, update the root).
+ DAG.setRoot(Dummy.getValue());
+}
+
+SDOperand DAGCombiner::visit(SDNode *N) {
+ switch(N->getOpcode()) {
+ default: break;
+ case ISD::TokenFactor: return visitTokenFactor(N);
+ case ISD::ADD: return visitADD(N);
+ case ISD::SUB: return visitSUB(N);
+ case ISD::ADDC: return visitADDC(N);
+ case ISD::ADDE: return visitADDE(N);
+ case ISD::MUL: return visitMUL(N);
+ case ISD::SDIV: return visitSDIV(N);
+ case ISD::UDIV: return visitUDIV(N);
+ case ISD::SREM: return visitSREM(N);
+ case ISD::UREM: return visitUREM(N);
+ case ISD::MULHU: return visitMULHU(N);
+ case ISD::MULHS: return visitMULHS(N);
+ case ISD::AND: return visitAND(N);
+ case ISD::OR: return visitOR(N);
+ case ISD::XOR: return visitXOR(N);
+ case ISD::SHL: return visitSHL(N);
+ case ISD::SRA: return visitSRA(N);
+ case ISD::SRL: return visitSRL(N);
+ case ISD::CTLZ: return visitCTLZ(N);
+ case ISD::CTTZ: return visitCTTZ(N);
+ case ISD::CTPOP: return visitCTPOP(N);
+ case ISD::SELECT: return visitSELECT(N);
+ case ISD::SELECT_CC: return visitSELECT_CC(N);
+ case ISD::SETCC: return visitSETCC(N);
+ case ISD::SIGN_EXTEND: return visitSIGN_EXTEND(N);
+ case ISD::ZERO_EXTEND: return visitZERO_EXTEND(N);
+ case ISD::ANY_EXTEND: return visitANY_EXTEND(N);
+ case ISD::SIGN_EXTEND_INREG: return visitSIGN_EXTEND_INREG(N);
+ case ISD::TRUNCATE: return visitTRUNCATE(N);
+ case ISD::BIT_CONVERT: return visitBIT_CONVERT(N);
+ case ISD::FADD: return visitFADD(N);
+ case ISD::FSUB: return visitFSUB(N);
+ case ISD::FMUL: return visitFMUL(N);
+ case ISD::FDIV: return visitFDIV(N);
+ case ISD::FREM: return visitFREM(N);
+ case ISD::FCOPYSIGN: return visitFCOPYSIGN(N);
+ case ISD::SINT_TO_FP: return visitSINT_TO_FP(N);
+ case ISD::UINT_TO_FP: return visitUINT_TO_FP(N);
+ case ISD::FP_TO_SINT: return visitFP_TO_SINT(N);
+ case ISD::FP_TO_UINT: return visitFP_TO_UINT(N);
+ case ISD::FP_ROUND: return visitFP_ROUND(N);
+ case ISD::FP_ROUND_INREG: return visitFP_ROUND_INREG(N);
+ case ISD::FP_EXTEND: return visitFP_EXTEND(N);
+ case ISD::FNEG: return visitFNEG(N);
+ case ISD::FABS: return visitFABS(N);
+ case ISD::BRCOND: return visitBRCOND(N);
+ case ISD::BR_CC: return visitBR_CC(N);
+ case ISD::LOAD: return visitLOAD(N);
+ case ISD::STORE: return visitSTORE(N);
+ case ISD::INSERT_VECTOR_ELT: return visitINSERT_VECTOR_ELT(N);
+ case ISD::BUILD_VECTOR: return visitBUILD_VECTOR(N);
+ case ISD::CONCAT_VECTORS: return visitCONCAT_VECTORS(N);
+ case ISD::VECTOR_SHUFFLE: return visitVECTOR_SHUFFLE(N);
+ }
+ return SDOperand();
+}
+
+/// getInputChainForNode - Given a node, return its input chain if it has one,
+/// otherwise return a null sd operand.
+static SDOperand getInputChainForNode(SDNode *N) {
+ if (unsigned NumOps = N->getNumOperands()) {
+ if (N->getOperand(0).getValueType() == MVT::Other)
+ return N->getOperand(0);
+ else if (N->getOperand(NumOps-1).getValueType() == MVT::Other)
+ return N->getOperand(NumOps-1);
+ for (unsigned i = 1; i < NumOps-1; ++i)
+ if (N->getOperand(i).getValueType() == MVT::Other)
+ return N->getOperand(i);
+ }
+ return SDOperand(0, 0);
+}
+
+SDOperand DAGCombiner::visitTokenFactor(SDNode *N) {
+ // If N has two operands, where one has an input chain equal to the other,
+ // the 'other' chain is redundant.
+ if (N->getNumOperands() == 2) {
+ if (getInputChainForNode(N->getOperand(0).Val) == N->getOperand(1))
+ return N->getOperand(0);
+ if (getInputChainForNode(N->getOperand(1).Val) == N->getOperand(0))
+ return N->getOperand(1);
+ }
+
+ SmallVector<SDNode *, 8> TFs; // List of token factors to visit.
+ SmallVector<SDOperand, 8> Ops; // Ops for replacing token factor.
+ SmallPtrSet<SDNode*, 16> SeenOps;
+ bool Changed = false; // If we should replace this token factor.
+
+ // Start out with this token factor.
+ TFs.push_back(N);
+
+ // Iterate through token factors. The TFs grows when new token factors are
+ // encountered.
+ for (unsigned i = 0; i < TFs.size(); ++i) {
+ SDNode *TF = TFs[i];
+
+ // Check each of the operands.
+ for (unsigned i = 0, ie = TF->getNumOperands(); i != ie; ++i) {
+ SDOperand Op = TF->getOperand(i);
+
+ switch (Op.getOpcode()) {
+ case ISD::EntryToken:
+ // Entry tokens don't need to be added to the list. They are
+ // rededundant.
+ Changed = true;
+ break;
+
+ case ISD::TokenFactor:
+ if ((CombinerAA || Op.hasOneUse()) &&
+ std::find(TFs.begin(), TFs.end(), Op.Val) == TFs.end()) {
+ // Queue up for processing.
+ TFs.push_back(Op.Val);
+ // Clean up in case the token factor is removed.
+ AddToWorkList(Op.Val);
+ Changed = true;
+ break;
+ }
+ // Fall thru
+
+ default:
+ // Only add if it isn't already in the list.
+ if (SeenOps.insert(Op.Val))
+ Ops.push_back(Op);
+ else
+ Changed = true;
+ break;
+ }
+ }
+ }
+
+ SDOperand Result;
+
+ // If we've change things around then replace token factor.
+ if (Changed) {
+ if (Ops.size() == 0) {
+ // The entry token is the only possible outcome.
+ Result = DAG.getEntryNode();
+ } else {
+ // New and improved token factor.
+ Result = DAG.getNode(ISD::TokenFactor, MVT::Other, &Ops[0], Ops.size());
+ }
+
+ // Don't add users to work list.
+ return CombineTo(N, Result, false);
+ }
+
+ return Result;
+}
+
+static
+SDOperand combineShlAddConstant(SDOperand N0, SDOperand N1, SelectionDAG &DAG) {
+ MVT::ValueType VT = N0.getValueType();
+ SDOperand N00 = N0.getOperand(0);
+ SDOperand N01 = N0.getOperand(1);
+ ConstantSDNode *N01C = dyn_cast<ConstantSDNode>(N01);
+ if (N01C && N00.getOpcode() == ISD::ADD && N00.Val->hasOneUse() &&
+ isa<ConstantSDNode>(N00.getOperand(1))) {
+ N0 = DAG.getNode(ISD::ADD, VT,
+ DAG.getNode(ISD::SHL, VT, N00.getOperand(0), N01),
+ DAG.getNode(ISD::SHL, VT, N00.getOperand(1), N01));
+ return DAG.getNode(ISD::ADD, VT, N0, N1);
+ }
+ return SDOperand();
+}
+
+static
+SDOperand combineSelectAndUse(SDNode *N, SDOperand Slct, SDOperand OtherOp,
+ SelectionDAG &DAG) {
+ MVT::ValueType VT = N->getValueType(0);
+ unsigned Opc = N->getOpcode();
+ bool isSlctCC = Slct.getOpcode() == ISD::SELECT_CC;
+ SDOperand LHS = isSlctCC ? Slct.getOperand(2) : Slct.getOperand(1);
+ SDOperand RHS = isSlctCC ? Slct.getOperand(3) : Slct.getOperand(2);
+ ISD::CondCode CC = ISD::SETCC_INVALID;
+ if (isSlctCC)
+ CC = cast<CondCodeSDNode>(Slct.getOperand(4))->get();
+ else {
+ SDOperand CCOp = Slct.getOperand(0);
+ if (CCOp.getOpcode() == ISD::SETCC)
+ CC = cast<CondCodeSDNode>(CCOp.getOperand(2))->get();
+ }
+
+ bool DoXform = false;
+ bool InvCC = false;
+ assert ((Opc == ISD::ADD || (Opc == ISD::SUB && Slct == N->getOperand(1))) &&
+ "Bad input!");
+ if (LHS.getOpcode() == ISD::Constant &&
+ cast<ConstantSDNode>(LHS)->isNullValue())
+ DoXform = true;
+ else if (CC != ISD::SETCC_INVALID &&
+ RHS.getOpcode() == ISD::Constant &&
+ cast<ConstantSDNode>(RHS)->isNullValue()) {
+ std::swap(LHS, RHS);
+ bool isInt = MVT::isInteger(isSlctCC ? Slct.getOperand(0).getValueType()
+ : Slct.getOperand(0).getOperand(0).getValueType());
+ CC = ISD::getSetCCInverse(CC, isInt);
+ DoXform = true;
+ InvCC = true;
+ }
+
+ if (DoXform) {
+ SDOperand Result = DAG.getNode(Opc, VT, OtherOp, RHS);
+ if (isSlctCC)
+ return DAG.getSelectCC(OtherOp, Result,
+ Slct.getOperand(0), Slct.getOperand(1), CC);
+ SDOperand CCOp = Slct.getOperand(0);
+ if (InvCC)
+ CCOp = DAG.getSetCC(CCOp.getValueType(), CCOp.getOperand(0),
+ CCOp.getOperand(1), CC);
+ return DAG.getNode(ISD::SELECT, VT, CCOp, OtherOp, Result);
+ }
+ return SDOperand();
+}
+
+SDOperand DAGCombiner::visitADD(SDNode *N) {
+ SDOperand N0 = N->getOperand(0);
+ SDOperand N1 = N->getOperand(1);
+ ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
+ ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
+ MVT::ValueType VT = N0.getValueType();
+
+ // fold vector ops
+ if (MVT::isVector(VT)) {
+ SDOperand FoldedVOp = SimplifyVBinOp(N);
+ if (FoldedVOp.Val) return FoldedVOp;
+ }
+
+ // fold (add x, undef) -> undef
+ if (N0.getOpcode() == ISD::UNDEF)
+ return N0;
+ if (N1.getOpcode() == ISD::UNDEF)
+ return N1;
+ // fold (add c1, c2) -> c1+c2
+ if (N0C && N1C)
+ return DAG.getNode(ISD::ADD, VT, N0, N1);
+ // canonicalize constant to RHS
+ if (N0C && !N1C)
+ return DAG.getNode(ISD::ADD, VT, N1, N0);
+ // fold (add x, 0) -> x
+ if (N1C && N1C->isNullValue())
+ return N0;
+ // fold ((c1-A)+c2) -> (c1+c2)-A
+ if (N1C && N0.getOpcode() == ISD::SUB)
+ if (ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0.getOperand(0)))
+ return DAG.getNode(ISD::SUB, VT,
+ DAG.getConstant(N1C->getValue()+N0C->getValue(), VT),
+ N0.getOperand(1));
+ // reassociate add
+ SDOperand RADD = ReassociateOps(ISD::ADD, N0, N1);
+ if (RADD.Val != 0)
+ return RADD;
+ // fold ((0-A) + B) -> B-A
+ if (N0.getOpcode() == ISD::SUB && isa<ConstantSDNode>(N0.getOperand(0)) &&
+ cast<ConstantSDNode>(N0.getOperand(0))->isNullValue())
+ return DAG.getNode(ISD::SUB, VT, N1, N0.getOperand(1));
+ // fold (A + (0-B)) -> A-B
+ if (N1.getOpcode() == ISD::SUB && isa<ConstantSDNode>(N1.getOperand(0)) &&
+ cast<ConstantSDNode>(N1.getOperand(0))->isNullValue())
+ return DAG.getNode(ISD::SUB, VT, N0, N1.getOperand(1));
+ // fold (A+(B-A)) -> B
+ if (N1.getOpcode() == ISD::SUB && N0 == N1.getOperand(1))
+ return N1.getOperand(0);
+
+ if (!MVT::isVector(VT) && SimplifyDemandedBits(SDOperand(N, 0)))
+ return SDOperand(N, 0);
+
+ // fold (a+b) -> (a|b) iff a and b share no bits.
+ if (MVT::isInteger(VT) && !MVT::isVector(VT)) {
+ uint64_t LHSZero, LHSOne;
+ uint64_t RHSZero, RHSOne;
+ uint64_t Mask = MVT::getIntVTBitMask(VT);
+ DAG.ComputeMaskedBits(N0, Mask, LHSZero, LHSOne);
+ if (LHSZero) {
+ DAG.ComputeMaskedBits(N1, Mask, RHSZero, RHSOne);
+
+ // If all possibly-set bits on the LHS are clear on the RHS, return an OR.
+ // If all possibly-set bits on the RHS are clear on the LHS, return an OR.
+ if ((RHSZero & (~LHSZero & Mask)) == (~LHSZero & Mask) ||
+ (LHSZero & (~RHSZero & Mask)) == (~RHSZero & Mask))
+ return DAG.getNode(ISD::OR, VT, N0, N1);
+ }
+ }
+
+ // fold (add (shl (add x, c1), c2), ) -> (add (add (shl x, c2), c1<<c2), )
+ if (N0.getOpcode() == ISD::SHL && N0.Val->hasOneUse()) {
+ SDOperand Result = combineShlAddConstant(N0, N1, DAG);
+ if (Result.Val) return Result;
+ }
+ if (N1.getOpcode() == ISD::SHL && N1.Val->hasOneUse()) {
+ SDOperand Result = combineShlAddConstant(N1, N0, DAG);
+ if (Result.Val) return Result;
+ }
+
+ // fold (add (select cc, 0, c), x) -> (select cc, x, (add, x, c))
+ if (N0.getOpcode() == ISD::SELECT && N0.Val->hasOneUse()) {
+ SDOperand Result = combineSelectAndUse(N, N0, N1, DAG);
+ if (Result.Val) return Result;
+ }
+ if (N1.getOpcode() == ISD::SELECT && N1.Val->hasOneUse()) {
+ SDOperand Result = combineSelectAndUse(N, N1, N0, DAG);
+ if (Result.Val) return Result;
+ }
+
+ return SDOperand();
+}
+
+SDOperand DAGCombiner::visitADDC(SDNode *N) {
+ SDOperand N0 = N->getOperand(0);
+ SDOperand N1 = N->getOperand(1);
+ ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
+ ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
+ MVT::ValueType VT = N0.getValueType();
+
+ // If the flag result is dead, turn this into an ADD.
+ if (N->hasNUsesOfValue(0, 1))
+ return CombineTo(N, DAG.getNode(ISD::ADD, VT, N1, N0),
+ DAG.getNode(ISD::CARRY_FALSE, MVT::Flag));
+
+ // canonicalize constant to RHS.
+ if (N0C && !N1C) {
+ SDOperand Ops[] = { N1, N0 };
+ return DAG.getNode(ISD::ADDC, N->getVTList(), Ops, 2);
+ }
+
+ // fold (addc x, 0) -> x + no carry out
+ if (N1C && N1C->isNullValue())
+ return CombineTo(N, N0, DAG.getNode(ISD::CARRY_FALSE, MVT::Flag));
+
+ // fold (addc a, b) -> (or a, b), CARRY_FALSE iff a and b share no bits.
+ uint64_t LHSZero, LHSOne;
+ uint64_t RHSZero, RHSOne;
+ uint64_t Mask = MVT::getIntVTBitMask(VT);
+ DAG.ComputeMaskedBits(N0, Mask, LHSZero, LHSOne);
+ if (LHSZero) {
+ DAG.ComputeMaskedBits(N1, Mask, RHSZero, RHSOne);
+
+ // If all possibly-set bits on the LHS are clear on the RHS, return an OR.
+ // If all possibly-set bits on the RHS are clear on the LHS, return an OR.
+ if ((RHSZero & (~LHSZero & Mask)) == (~LHSZero & Mask) ||
+ (LHSZero & (~RHSZero & Mask)) == (~RHSZero & Mask))
+ return CombineTo(N, DAG.getNode(ISD::OR, VT, N0, N1),
+ DAG.getNode(ISD::CARRY_FALSE, MVT::Flag));
+ }
+
+ return SDOperand();
+}
+
+SDOperand DAGCombiner::visitADDE(SDNode *N) {
+ SDOperand N0 = N->getOperand(0);
+ SDOperand N1 = N->getOperand(1);
+ SDOperand CarryIn = N->getOperand(2);
+ ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
+ ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
+ //MVT::ValueType VT = N0.getValueType();
+
+ // canonicalize constant to RHS
+ if (N0C && !N1C) {
+ SDOperand Ops[] = { N1, N0, CarryIn };
+ return DAG.getNode(ISD::ADDE, N->getVTList(), Ops, 3);
+ }
+
+ // fold (adde x, y, false) -> (addc x, y)
+ if (CarryIn.getOpcode() == ISD::CARRY_FALSE) {
+ SDOperand Ops[] = { N1, N0 };
+ return DAG.getNode(ISD::ADDC, N->getVTList(), Ops, 2);
+ }
+
+ return SDOperand();
+}
+
+
+
+SDOperand DAGCombiner::visitSUB(SDNode *N) {
+ SDOperand N0 = N->getOperand(0);
+ SDOperand N1 = N->getOperand(1);
+ ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0.Val);
+ ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.Val);
+ MVT::ValueType VT = N0.getValueType();
+
+ // fold vector ops
+ if (MVT::isVector(VT)) {
+ SDOperand FoldedVOp = SimplifyVBinOp(N);
+ if (FoldedVOp.Val) return FoldedVOp;
+ }
+
+ // fold (sub x, x) -> 0
+ if (N0 == N1)
+ return DAG.getConstant(0, N->getValueType(0));
+ // fold (sub c1, c2) -> c1-c2
+ if (N0C && N1C)
+ return DAG.getNode(ISD::SUB, VT, N0, N1);
+ // fold (sub x, c) -> (add x, -c)
+ if (N1C)
+ return DAG.getNode(ISD::ADD, VT, N0, DAG.getConstant(-N1C->getValue(), VT));
+ // fold (A+B)-A -> B
+ if (N0.getOpcode() == ISD::ADD && N0.getOperand(0) == N1)
+ return N0.getOperand(1);
+ // fold (A+B)-B -> A
+ if (N0.getOpcode() == ISD::ADD && N0.getOperand(1) == N1)
+ return N0.getOperand(0);
+ // fold (sub x, (select cc, 0, c)) -> (select cc, x, (sub, x, c))
+ if (N1.getOpcode() == ISD::SELECT && N1.Val->hasOneUse()) {
+ SDOperand Result = combineSelectAndUse(N, N1, N0, DAG);
+ if (Result.Val) return Result;
+ }
+ // If either operand of a sub is undef, the result is undef
+ if (N0.getOpcode() == ISD::UNDEF)
+ return N0;
+ if (N1.getOpcode() == ISD::UNDEF)
+ return N1;
+
+ return SDOperand();
+}
+
+SDOperand DAGCombiner::visitMUL(SDNode *N) {
+ SDOperand N0 = N->getOperand(0);
+ SDOperand N1 = N->getOperand(1);
+ ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
+ ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
+ MVT::ValueType VT = N0.getValueType();
+
+ // fold vector ops
+ if (MVT::isVector(VT)) {
+ SDOperand FoldedVOp = SimplifyVBinOp(N);
+ if (FoldedVOp.Val) return FoldedVOp;
+ }
+
+ // fold (mul x, undef) -> 0
+ if (N0.getOpcode() == ISD::UNDEF || N1.getOpcode() == ISD::UNDEF)
+ return DAG.getConstant(0, VT);
+ // fold (mul c1, c2) -> c1*c2
+ if (N0C && N1C)
+ return DAG.getNode(ISD::MUL, VT, N0, N1);
+ // canonicalize constant to RHS
+ if (N0C && !N1C)
+ return DAG.getNode(ISD::MUL, VT, N1, N0);
+ // fold (mul x, 0) -> 0
+ if (N1C && N1C->isNullValue())
+ return N1;
+ // fold (mul x, -1) -> 0-x
+ if (N1C && N1C->isAllOnesValue())
+ return DAG.getNode(ISD::SUB, VT, DAG.getConstant(0, VT), N0);
+ // fold (mul x, (1 << c)) -> x << c
+ if (N1C && isPowerOf2_64(N1C->getValue()))
+ return DAG.getNode(ISD::SHL, VT, N0,
+ DAG.getConstant(Log2_64(N1C->getValue()),
+ TLI.getShiftAmountTy()));
+ // fold (mul x, -(1 << c)) -> -(x << c) or (-x) << c
+ if (N1C && isPowerOf2_64(-N1C->getSignExtended())) {
+ // FIXME: If the input is something that is easily negated (e.g. a
+ // single-use add), we should put the negate there.
+ return DAG.getNode(ISD::SUB, VT, DAG.getConstant(0, VT),
+ DAG.getNode(ISD::SHL, VT, N0,
+ DAG.getConstant(Log2_64(-N1C->getSignExtended()),
+ TLI.getShiftAmountTy())));
+ }
+
+ // (mul (shl X, c1), c2) -> (mul X, c2 << c1)
+ if (N1C && N0.getOpcode() == ISD::SHL &&
+ isa<ConstantSDNode>(N0.getOperand(1))) {
+ SDOperand C3 = DAG.getNode(ISD::SHL, VT, N1, N0.getOperand(1));
+ AddToWorkList(C3.Val);
+ return DAG.getNode(ISD::MUL, VT, N0.getOperand(0), C3);
+ }
+
+ // Change (mul (shl X, C), Y) -> (shl (mul X, Y), C) when the shift has one
+ // use.
+ {
+ SDOperand Sh(0,0), Y(0,0);
+ // Check for both (mul (shl X, C), Y) and (mul Y, (shl X, C)).
+ if (N0.getOpcode() == ISD::SHL && isa<ConstantSDNode>(N0.getOperand(1)) &&
+ N0.Val->hasOneUse()) {
+ Sh = N0; Y = N1;
+ } else if (N1.getOpcode() == ISD::SHL &&
+ isa<ConstantSDNode>(N1.getOperand(1)) && N1.Val->hasOneUse()) {
+ Sh = N1; Y = N0;
+ }
+ if (Sh.Val) {
+ SDOperand Mul = DAG.getNode(ISD::MUL, VT, Sh.getOperand(0), Y);
+ return DAG.getNode(ISD::SHL, VT, Mul, Sh.getOperand(1));
+ }
+ }
+ // fold (mul (add x, c1), c2) -> (add (mul x, c2), c1*c2)
+ if (N1C && N0.getOpcode() == ISD::ADD && N0.Val->hasOneUse() &&
+ isa<ConstantSDNode>(N0.getOperand(1))) {
+ return DAG.getNode(ISD::ADD, VT,
+ DAG.getNode(ISD::MUL, VT, N0.getOperand(0), N1),
+ DAG.getNode(ISD::MUL, VT, N0.getOperand(1), N1));
+ }
+
+ // reassociate mul
+ SDOperand RMUL = ReassociateOps(ISD::MUL, N0, N1);
+ if (RMUL.Val != 0)
+ return RMUL;
+
+ return SDOperand();
+}
+
+SDOperand DAGCombiner::visitSDIV(SDNode *N) {
+ SDOperand N0 = N->getOperand(0);
+ SDOperand N1 = N->getOperand(1);
+ ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0.Val);
+ ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.Val);
+ MVT::ValueType VT = N->getValueType(0);
+
+ // fold vector ops
+ if (MVT::isVector(VT)) {
+ SDOperand FoldedVOp = SimplifyVBinOp(N);
+ if (FoldedVOp.Val) return FoldedVOp;
+ }
+
+ // fold (sdiv c1, c2) -> c1/c2
+ if (N0C && N1C && !N1C->isNullValue())
+ return DAG.getNode(ISD::SDIV, VT, N0, N1);
+ // fold (sdiv X, 1) -> X
+ if (N1C && N1C->getSignExtended() == 1LL)
+ return N0;
+ // fold (sdiv X, -1) -> 0-X
+ if (N1C && N1C->isAllOnesValue())
+ return DAG.getNode(ISD::SUB, VT, DAG.getConstant(0, VT), N0);
+ // If we know the sign bits of both operands are zero, strength reduce to a
+ // udiv instead. Handles (X&15) /s 4 -> X&15 >> 2
+ uint64_t SignBit = 1ULL << (MVT::getSizeInBits(VT)-1);
+ if (DAG.MaskedValueIsZero(N1, SignBit) &&
+ DAG.MaskedValueIsZero(N0, SignBit))
+ return DAG.getNode(ISD::UDIV, N1.getValueType(), N0, N1);
+ // fold (sdiv X, pow2) -> simple ops after legalize
+ if (N1C && N1C->getValue() && !TLI.isIntDivCheap() &&
+ (isPowerOf2_64(N1C->getSignExtended()) ||
+ isPowerOf2_64(-N1C->getSignExtended()))) {
+ // If dividing by powers of two is cheap, then don't perform the following
+ // fold.
+ if (TLI.isPow2DivCheap())
+ return SDOperand();
+ int64_t pow2 = N1C->getSignExtended();
+ int64_t abs2 = pow2 > 0 ? pow2 : -pow2;
+ unsigned lg2 = Log2_64(abs2);
+ // Splat the sign bit into the register
+ SDOperand SGN = DAG.getNode(ISD::SRA, VT, N0,
+ DAG.getConstant(MVT::getSizeInBits(VT)-1,
+ TLI.getShiftAmountTy()));
+ AddToWorkList(SGN.Val);
+ // Add (N0 < 0) ? abs2 - 1 : 0;
+ SDOperand SRL = DAG.getNode(ISD::SRL, VT, SGN,
+ DAG.getConstant(MVT::getSizeInBits(VT)-lg2,
+ TLI.getShiftAmountTy()));
+ SDOperand ADD = DAG.getNode(ISD::ADD, VT, N0, SRL);
+ AddToWorkList(SRL.Val);
+ AddToWorkList(ADD.Val); // Divide by pow2
+ SDOperand SRA = DAG.getNode(ISD::SRA, VT, ADD,
+ DAG.getConstant(lg2, TLI.getShiftAmountTy()));
+ // If we're dividing by a positive value, we're done. Otherwise, we must
+ // negate the result.
+ if (pow2 > 0)
+ return SRA;
+ AddToWorkList(SRA.Val);
+ return DAG.getNode(ISD::SUB, VT, DAG.getConstant(0, VT), SRA);
+ }
+ // if integer divide is expensive and we satisfy the requirements, emit an
+ // alternate sequence.
+ if (N1C && (N1C->getSignExtended() < -1 || N1C->getSignExtended() > 1) &&
+ !TLI.isIntDivCheap()) {
+ SDOperand Op = BuildSDIV(N);
+ if (Op.Val) return Op;
+ }
+
+ // undef / X -> 0
+ if (N0.getOpcode() == ISD::UNDEF)
+ return DAG.getConstant(0, VT);
+ // X / undef -> undef
+ if (N1.getOpcode() == ISD::UNDEF)
+ return N1;
+
+ return SDOperand();
+}
+
+SDOperand DAGCombiner::visitUDIV(SDNode *N) {
+ SDOperand N0 = N->getOperand(0);
+ SDOperand N1 = N->getOperand(1);
+ ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0.Val);
+ ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.Val);
+ MVT::ValueType VT = N->getValueType(0);
+
+ // fold vector ops
+ if (MVT::isVector(VT)) {
+ SDOperand FoldedVOp = SimplifyVBinOp(N);
+ if (FoldedVOp.Val) return FoldedVOp;
+ }
+
+ // fold (udiv c1, c2) -> c1/c2
+ if (N0C && N1C && !N1C->isNullValue())
+ return DAG.getNode(ISD::UDIV, VT, N0, N1);
+ // fold (udiv x, (1 << c)) -> x >>u c
+ if (N1C && isPowerOf2_64(N1C->getValue()))
+ return DAG.getNode(ISD::SRL, VT, N0,
+ DAG.getConstant(Log2_64(N1C->getValue()),
+ TLI.getShiftAmountTy()));
+ // fold (udiv x, (shl c, y)) -> x >>u (log2(c)+y) iff c is power of 2
+ if (N1.getOpcode() == ISD::SHL) {
+ if (ConstantSDNode *SHC = dyn_cast<ConstantSDNode>(N1.getOperand(0))) {
+ if (isPowerOf2_64(SHC->getValue())) {
+ MVT::ValueType ADDVT = N1.getOperand(1).getValueType();
+ SDOperand Add = DAG.getNode(ISD::ADD, ADDVT, N1.getOperand(1),
+ DAG.getConstant(Log2_64(SHC->getValue()),
+ ADDVT));
+ AddToWorkList(Add.Val);
+ return DAG.getNode(ISD::SRL, VT, N0, Add);
+ }
+ }
+ }
+ // fold (udiv x, c) -> alternate
+ if (N1C && N1C->getValue() && !TLI.isIntDivCheap()) {
+ SDOperand Op = BuildUDIV(N);
+ if (Op.Val) return Op;
+ }
+
+ // undef / X -> 0
+ if (N0.getOpcode() == ISD::UNDEF)
+ return DAG.getConstant(0, VT);
+ // X / undef -> undef
+ if (N1.getOpcode() == ISD::UNDEF)
+ return N1;
+
+ return SDOperand();
+}
+
+SDOperand DAGCombiner::visitSREM(SDNode *N) {
+ SDOperand N0 = N->getOperand(0);
+ SDOperand N1 = N->getOperand(1);
+ ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
+ ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
+ MVT::ValueType VT = N->getValueType(0);
+
+ // fold (srem c1, c2) -> c1%c2
+ if (N0C && N1C && !N1C->isNullValue())
+ return DAG.getNode(ISD::SREM, VT, N0, N1);
+ // If we know the sign bits of both operands are zero, strength reduce to a
+ // urem instead. Handles (X & 0x0FFFFFFF) %s 16 -> X&15
+ uint64_t SignBit = 1ULL << (MVT::getSizeInBits(VT)-1);
+ if (DAG.MaskedValueIsZero(N1, SignBit) &&
+ DAG.MaskedValueIsZero(N0, SignBit))
+ return DAG.getNode(ISD::UREM, VT, N0, N1);
+
+ // Unconditionally lower X%C -> X-X/C*C. This allows the X/C logic to hack on
+ // the remainder operation.
+ if (N1C && !N1C->isNullValue()) {
+ SDOperand Div = DAG.getNode(ISD::SDIV, VT, N0, N1);
+ SDOperand Mul = DAG.getNode(ISD::MUL, VT, Div, N1);
+ SDOperand Sub = DAG.getNode(ISD::SUB, VT, N0, Mul);
+ AddToWorkList(Div.Val);
+ AddToWorkList(Mul.Val);
+ return Sub;
+ }
+
+ // undef % X -> 0
+ if (N0.getOpcode() == ISD::UNDEF)
+ return DAG.getConstant(0, VT);
+ // X % undef -> undef
+ if (N1.getOpcode() == ISD::UNDEF)
+ return N1;
+
+ return SDOperand();
+}
+
+SDOperand DAGCombiner::visitUREM(SDNode *N) {
+ SDOperand N0 = N->getOperand(0);
+ SDOperand N1 = N->getOperand(1);
+ ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
+ ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
+ MVT::ValueType VT = N->getValueType(0);
+
+ // fold (urem c1, c2) -> c1%c2
+ if (N0C && N1C && !N1C->isNullValue())
+ return DAG.getNode(ISD::UREM, VT, N0, N1);
+ // fold (urem x, pow2) -> (and x, pow2-1)
+ if (N1C && !N1C->isNullValue() && isPowerOf2_64(N1C->getValue()))
+ return DAG.getNode(ISD::AND, VT, N0, DAG.getConstant(N1C->getValue()-1,VT));
+ // fold (urem x, (shl pow2, y)) -> (and x, (add (shl pow2, y), -1))
+ if (N1.getOpcode() == ISD::SHL) {
+ if (ConstantSDNode *SHC = dyn_cast<ConstantSDNode>(N1.getOperand(0))) {
+ if (isPowerOf2_64(SHC->getValue())) {
+ SDOperand Add = DAG.getNode(ISD::ADD, VT, N1,DAG.getConstant(~0ULL,VT));
+ AddToWorkList(Add.Val);
+ return DAG.getNode(ISD::AND, VT, N0, Add);
+ }
+ }
+ }
+
+ // Unconditionally lower X%C -> X-X/C*C. This allows the X/C logic to hack on
+ // the remainder operation.
+ if (N1C && !N1C->isNullValue()) {
+ SDOperand Div = DAG.getNode(ISD::UDIV, VT, N0, N1);
+ SDOperand Mul = DAG.getNode(ISD::MUL, VT, Div, N1);
+ SDOperand Sub = DAG.getNode(ISD::SUB, VT, N0, Mul);
+ AddToWorkList(Div.Val);
+ AddToWorkList(Mul.Val);
+ return Sub;
+ }
+
+ // undef % X -> 0
+ if (N0.getOpcode() == ISD::UNDEF)
+ return DAG.getConstant(0, VT);
+ // X % undef -> undef
+ if (N1.getOpcode() == ISD::UNDEF)
+ return N1;
+
+ return SDOperand();
+}
+
+SDOperand DAGCombiner::visitMULHS(SDNode *N) {
+ SDOperand N0 = N->getOperand(0);
+ SDOperand N1 = N->getOperand(1);
+ ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
+ MVT::ValueType VT = N->getValueType(0);
+
+ // fold (mulhs x, 0) -> 0
+ if (N1C && N1C->isNullValue())
+ return N1;
+ // fold (mulhs x, 1) -> (sra x, size(x)-1)
+ if (N1C && N1C->getValue() == 1)
+ return DAG.getNode(ISD::SRA, N0.getValueType(), N0,
+ DAG.getConstant(MVT::getSizeInBits(N0.getValueType())-1,
+ TLI.getShiftAmountTy()));
+ // fold (mulhs x, undef) -> 0
+ if (N0.getOpcode() == ISD::UNDEF || N1.getOpcode() == ISD::UNDEF)
+ return DAG.getConstant(0, VT);
+
+ return SDOperand();
+}
+
+SDOperand DAGCombiner::visitMULHU(SDNode *N) {
+ SDOperand N0 = N->getOperand(0);
+ SDOperand N1 = N->getOperand(1);
+ ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
+ MVT::ValueType VT = N->getValueType(0);
+
+ // fold (mulhu x, 0) -> 0
+ if (N1C && N1C->isNullValue())
+ return N1;
+ // fold (mulhu x, 1) -> 0
+ if (N1C && N1C->getValue() == 1)
+ return DAG.getConstant(0, N0.getValueType());
+ // fold (mulhu x, undef) -> 0
+ if (N0.getOpcode() == ISD::UNDEF || N1.getOpcode() == ISD::UNDEF)
+ return DAG.getConstant(0, VT);
+
+ return SDOperand();
+}
+
+/// SimplifyBinOpWithSameOpcodeHands - If this is a binary operator with
+/// two operands of the same opcode, try to simplify it.
+SDOperand DAGCombiner::SimplifyBinOpWithSameOpcodeHands(SDNode *N) {
+ SDOperand N0 = N->getOperand(0), N1 = N->getOperand(1);
+ MVT::ValueType VT = N0.getValueType();
+ assert(N0.getOpcode() == N1.getOpcode() && "Bad input!");
+
+ // For each of OP in AND/OR/XOR:
+ // fold (OP (zext x), (zext y)) -> (zext (OP x, y))
+ // fold (OP (sext x), (sext y)) -> (sext (OP x, y))
+ // fold (OP (aext x), (aext y)) -> (aext (OP x, y))
+ // fold (OP (trunc x), (trunc y)) -> (trunc (OP x, y))
+ if ((N0.getOpcode() == ISD::ZERO_EXTEND || N0.getOpcode() == ISD::ANY_EXTEND||
+ N0.getOpcode() == ISD::SIGN_EXTEND || N0.getOpcode() == ISD::TRUNCATE) &&
+ N0.getOperand(0).getValueType() == N1.getOperand(0).getValueType()) {
+ SDOperand ORNode = DAG.getNode(N->getOpcode(),
+ N0.getOperand(0).getValueType(),
+ N0.getOperand(0), N1.getOperand(0));
+ AddToWorkList(ORNode.Val);
+ return DAG.getNode(N0.getOpcode(), VT, ORNode);
+ }
+
+ // For each of OP in SHL/SRL/SRA/AND...
+ // fold (and (OP x, z), (OP y, z)) -> (OP (and x, y), z)
+ // fold (or (OP x, z), (OP y, z)) -> (OP (or x, y), z)
+ // fold (xor (OP x, z), (OP y, z)) -> (OP (xor x, y), z)
+ if ((N0.getOpcode() == ISD::SHL || N0.getOpcode() == ISD::SRL ||
+ N0.getOpcode() == ISD::SRA || N0.getOpcode() == ISD::AND) &&
+ N0.getOperand(1) == N1.getOperand(1)) {
+ SDOperand ORNode = DAG.getNode(N->getOpcode(),
+ N0.getOperand(0).getValueType(),
+ N0.getOperand(0), N1.getOperand(0));
+ AddToWorkList(ORNode.Val);
+ return DAG.getNode(N0.getOpcode(), VT, ORNode, N0.getOperand(1));
+ }
+
+ return SDOperand();
+}
+
+SDOperand DAGCombiner::visitAND(SDNode *N) {
+ SDOperand N0 = N->getOperand(0);
+ SDOperand N1 = N->getOperand(1);
+ SDOperand LL, LR, RL, RR, CC0, CC1;
+ ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
+ ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
+ MVT::ValueType VT = N1.getValueType();
+
+ // fold vector ops
+ if (MVT::isVector(VT)) {
+ SDOperand FoldedVOp = SimplifyVBinOp(N);
+ if (FoldedVOp.Val) return FoldedVOp;
+ }
+
+ // fold (and x, undef) -> 0
+ if (N0.getOpcode() == ISD::UNDEF || N1.getOpcode() == ISD::UNDEF)
+ return DAG.getConstant(0, VT);
+ // fold (and c1, c2) -> c1&c2
+ if (N0C && N1C)
+ return DAG.getNode(ISD::AND, VT, N0, N1);
+ // canonicalize constant to RHS
+ if (N0C && !N1C)
+ return DAG.getNode(ISD::AND, VT, N1, N0);
+ // fold (and x, -1) -> x
+ if (N1C && N1C->isAllOnesValue())
+ return N0;
+ // if (and x, c) is known to be zero, return 0
+ if (N1C && DAG.MaskedValueIsZero(SDOperand(N, 0), MVT::getIntVTBitMask(VT)))
+ return DAG.getConstant(0, VT);
+ // reassociate and
+ SDOperand RAND = ReassociateOps(ISD::AND, N0, N1);
+ if (RAND.Val != 0)
+ return RAND;
+ // fold (and (or x, 0xFFFF), 0xFF) -> 0xFF
+ if (N1C && N0.getOpcode() == ISD::OR)
+ if (ConstantSDNode *ORI = dyn_cast<ConstantSDNode>(N0.getOperand(1)))
+ if ((ORI->getValue() & N1C->getValue()) == N1C->getValue())
+ return N1;
+ // fold (and (any_ext V), c) -> (zero_ext V) if 'and' only clears top bits.
+ if (N1C && N0.getOpcode() == ISD::ANY_EXTEND) {
+ unsigned InMask = MVT::getIntVTBitMask(N0.getOperand(0).getValueType());
+ if (DAG.MaskedValueIsZero(N0.getOperand(0),
+ ~N1C->getValue() & InMask)) {
+ SDOperand Zext = DAG.getNode(ISD::ZERO_EXTEND, N0.getValueType(),
+ N0.getOperand(0));
+
+ // Replace uses of the AND with uses of the Zero extend node.
+ CombineTo(N, Zext);
+
+ // We actually want to replace all uses of the any_extend with the
+ // zero_extend, to avoid duplicating things. This will later cause this
+ // AND to be folded.
+ CombineTo(N0.Val, Zext);
+ return SDOperand(N, 0); // Return N so it doesn't get rechecked!
+ }
+ }
+ // fold (and (setcc x), (setcc y)) -> (setcc (and x, y))
+ if (isSetCCEquivalent(N0, LL, LR, CC0) && isSetCCEquivalent(N1, RL, RR, CC1)){
+ ISD::CondCode Op0 = cast<CondCodeSDNode>(CC0)->get();
+ ISD::CondCode Op1 = cast<CondCodeSDNode>(CC1)->get();
+
+ if (LR == RR && isa<ConstantSDNode>(LR) && Op0 == Op1 &&
+ MVT::isInteger(LL.getValueType())) {
+ // fold (X == 0) & (Y == 0) -> (X|Y == 0)
+ if (cast<ConstantSDNode>(LR)->getValue() == 0 && Op1 == ISD::SETEQ) {
+ SDOperand ORNode = DAG.getNode(ISD::OR, LR.getValueType(), LL, RL);
+ AddToWorkList(ORNode.Val);
+ return DAG.getSetCC(VT, ORNode, LR, Op1);
+ }
+ // fold (X == -1) & (Y == -1) -> (X&Y == -1)
+ if (cast<ConstantSDNode>(LR)->isAllOnesValue() && Op1 == ISD::SETEQ) {
+ SDOperand ANDNode = DAG.getNode(ISD::AND, LR.getValueType(), LL, RL);
+ AddToWorkList(ANDNode.Val);
+ return DAG.getSetCC(VT, ANDNode, LR, Op1);
+ }
+ // fold (X > -1) & (Y > -1) -> (X|Y > -1)
+ if (cast<ConstantSDNode>(LR)->isAllOnesValue() && Op1 == ISD::SETGT) {
+ SDOperand ORNode = DAG.getNode(ISD::OR, LR.getValueType(), LL, RL);
+ AddToWorkList(ORNode.Val);
+ return DAG.getSetCC(VT, ORNode, LR, Op1);
+ }
+ }
+ // canonicalize equivalent to ll == rl
+ if (LL == RR && LR == RL) {
+ Op1 = ISD::getSetCCSwappedOperands(Op1);
+ std::swap(RL, RR);
+ }
+ if (LL == RL && LR == RR) {
+ bool isInteger = MVT::isInteger(LL.getValueType());
+ ISD::CondCode Result = ISD::getSetCCAndOperation(Op0, Op1, isInteger);
+ if (Result != ISD::SETCC_INVALID)
+ return DAG.getSetCC(N0.getValueType(), LL, LR, Result);
+ }
+ }
+
+ // Simplify: and (op x...), (op y...) -> (op (and x, y))
+ if (N0.getOpcode() == N1.getOpcode()) {
+ SDOperand Tmp = SimplifyBinOpWithSameOpcodeHands(N);
+ if (Tmp.Val) return Tmp;
+ }
+
+ // fold (and (sign_extend_inreg x, i16 to i32), 1) -> (and x, 1)
+ // fold (and (sra)) -> (and (srl)) when possible.
+ if (!MVT::isVector(VT) &&
+ SimplifyDemandedBits(SDOperand(N, 0)))
+ return SDOperand(N, 0);
+ // fold (zext_inreg (extload x)) -> (zextload x)
+ if (ISD::isEXTLoad(N0.Val) && ISD::isUNINDEXEDLoad(N0.Val)) {
+ LoadSDNode *LN0 = cast<LoadSDNode>(N0);
+ MVT::ValueType EVT = LN0->getLoadedVT();
+ // If we zero all the possible extended bits, then we can turn this into
+ // a zextload if we are running before legalize or the operation is legal.
+ if (DAG.MaskedValueIsZero(N1, ~0ULL << MVT::getSizeInBits(EVT)) &&
+ (!AfterLegalize || TLI.isLoadXLegal(ISD::ZEXTLOAD, EVT))) {
+ SDOperand ExtLoad = DAG.getExtLoad(ISD::ZEXTLOAD, VT, LN0->getChain(),
+ LN0->getBasePtr(), LN0->getSrcValue(),
+ LN0->getSrcValueOffset(), EVT,
+ LN0->isVolatile(),
+ LN0->getAlignment());
+ AddToWorkList(N);
+ CombineTo(N0.Val, ExtLoad, ExtLoad.getValue(1));
+ return SDOperand(N, 0); // Return N so it doesn't get rechecked!
+ }
+ }
+ // fold (zext_inreg (sextload x)) -> (zextload x) iff load has one use
+ if (ISD::isSEXTLoad(N0.Val) && ISD::isUNINDEXEDLoad(N0.Val) &&
+ N0.hasOneUse()) {
+ LoadSDNode *LN0 = cast<LoadSDNode>(N0);
+ MVT::ValueType EVT = LN0->getLoadedVT();
+ // If we zero all the possible extended bits, then we can turn this into
+ // a zextload if we are running before legalize or the operation is legal.
+ if (DAG.MaskedValueIsZero(N1, ~0ULL << MVT::getSizeInBits(EVT)) &&
+ (!AfterLegalize || TLI.isLoadXLegal(ISD::ZEXTLOAD, EVT))) {
+ SDOperand ExtLoad = DAG.getExtLoad(ISD::ZEXTLOAD, VT, LN0->getChain(),
+ LN0->getBasePtr(), LN0->getSrcValue(),
+ LN0->getSrcValueOffset(), EVT,
+ LN0->isVolatile(),
+ LN0->getAlignment());
+ AddToWorkList(N);
+ CombineTo(N0.Val, ExtLoad, ExtLoad.getValue(1));
+ return SDOperand(N, 0); // Return N so it doesn't get rechecked!
+ }
+ }
+
+ // fold (and (load x), 255) -> (zextload x, i8)
+ // fold (and (extload x, i16), 255) -> (zextload x, i8)
+ if (N1C && N0.getOpcode() == ISD::LOAD) {
+ LoadSDNode *LN0 = cast<LoadSDNode>(N0);
+ if (LN0->getExtensionType() != ISD::SEXTLOAD &&
+ LN0->getAddressingMode() == ISD::UNINDEXED &&
+ N0.hasOneUse()) {
+ MVT::ValueType EVT, LoadedVT;
+ if (N1C->getValue() == 255)
+ EVT = MVT::i8;
+ else if (N1C->getValue() == 65535)
+ EVT = MVT::i16;
+ else if (N1C->getValue() == ~0U)
+ EVT = MVT::i32;
+ else
+ EVT = MVT::Other;
+
+ LoadedVT = LN0->getLoadedVT();
+ if (EVT != MVT::Other && LoadedVT > EVT &&
+ (!AfterLegalize || TLI.isLoadXLegal(ISD::ZEXTLOAD, EVT))) {
+ MVT::ValueType PtrType = N0.getOperand(1).getValueType();
+ // For big endian targets, we need to add an offset to the pointer to
+ // load the correct bytes. For little endian systems, we merely need to
+ // read fewer bytes from the same pointer.
+ unsigned PtrOff =
+ (MVT::getSizeInBits(LoadedVT) - MVT::getSizeInBits(EVT)) / 8;
+ SDOperand NewPtr = LN0->getBasePtr();
+ if (!TLI.isLittleEndian())
+ NewPtr = DAG.getNode(ISD::ADD, PtrType, NewPtr,
+ DAG.getConstant(PtrOff, PtrType));
+ AddToWorkList(NewPtr.Val);
+ SDOperand Load =
+ DAG.getExtLoad(ISD::ZEXTLOAD, VT, LN0->getChain(), NewPtr,
+ LN0->getSrcValue(), LN0->getSrcValueOffset(), EVT,
+ LN0->isVolatile(), LN0->getAlignment());
+ AddToWorkList(N);
+ CombineTo(N0.Val, Load, Load.getValue(1));
+ return SDOperand(N, 0); // Return N so it doesn't get rechecked!
+ }
+ }
+ }
+
+ return SDOperand();
+}
+
+SDOperand DAGCombiner::visitOR(SDNode *N) {
+ SDOperand N0 = N->getOperand(0);
+ SDOperand N1 = N->getOperand(1);
+ SDOperand LL, LR, RL, RR, CC0, CC1;
+ ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
+ ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
+ MVT::ValueType VT = N1.getValueType();
+ unsigned OpSizeInBits = MVT::getSizeInBits(VT);
+
+ // fold vector ops
+ if (MVT::isVector(VT)) {
+ SDOperand FoldedVOp = SimplifyVBinOp(N);
+ if (FoldedVOp.Val) return FoldedVOp;
+ }
+
+ // fold (or x, undef) -> -1
+ if (N0.getOpcode() == ISD::UNDEF || N1.getOpcode() == ISD::UNDEF)
+ return DAG.getConstant(~0ULL, VT);
+ // fold (or c1, c2) -> c1|c2
+ if (N0C && N1C)
+ return DAG.getNode(ISD::OR, VT, N0, N1);
+ // canonicalize constant to RHS
+ if (N0C && !N1C)
+ return DAG.getNode(ISD::OR, VT, N1, N0);
+ // fold (or x, 0) -> x
+ if (N1C && N1C->isNullValue())
+ return N0;
+ // fold (or x, -1) -> -1
+ if (N1C && N1C->isAllOnesValue())
+ return N1;
+ // fold (or x, c) -> c iff (x & ~c) == 0
+ if (N1C &&
+ DAG.MaskedValueIsZero(N0,~N1C->getValue() & (~0ULL>>(64-OpSizeInBits))))
+ return N1;
+ // reassociate or
+ SDOperand ROR = ReassociateOps(ISD::OR, N0, N1);
+ if (ROR.Val != 0)
+ return ROR;
+ // Canonicalize (or (and X, c1), c2) -> (and (or X, c2), c1|c2)
+ if (N1C && N0.getOpcode() == ISD::AND && N0.Val->hasOneUse() &&
+ isa<ConstantSDNode>(N0.getOperand(1))) {
+ ConstantSDNode *C1 = cast<ConstantSDNode>(N0.getOperand(1));
+ return DAG.getNode(ISD::AND, VT, DAG.getNode(ISD::OR, VT, N0.getOperand(0),
+ N1),
+ DAG.getConstant(N1C->getValue() | C1->getValue(), VT));
+ }
+ // fold (or (setcc x), (setcc y)) -> (setcc (or x, y))
+ if (isSetCCEquivalent(N0, LL, LR, CC0) && isSetCCEquivalent(N1, RL, RR, CC1)){
+ ISD::CondCode Op0 = cast<CondCodeSDNode>(CC0)->get();
+ ISD::CondCode Op1 = cast<CondCodeSDNode>(CC1)->get();
+
+ if (LR == RR && isa<ConstantSDNode>(LR) && Op0 == Op1 &&
+ MVT::isInteger(LL.getValueType())) {
+ // fold (X != 0) | (Y != 0) -> (X|Y != 0)
+ // fold (X < 0) | (Y < 0) -> (X|Y < 0)
+ if (cast<ConstantSDNode>(LR)->getValue() == 0 &&
+ (Op1 == ISD::SETNE || Op1 == ISD::SETLT)) {
+ SDOperand ORNode = DAG.getNode(ISD::OR, LR.getValueType(), LL, RL);
+ AddToWorkList(ORNode.Val);
+ return DAG.getSetCC(VT, ORNode, LR, Op1);
+ }
+ // fold (X != -1) | (Y != -1) -> (X&Y != -1)
+ // fold (X > -1) | (Y > -1) -> (X&Y > -1)
+ if (cast<ConstantSDNode>(LR)->isAllOnesValue() &&
+ (Op1 == ISD::SETNE || Op1 == ISD::SETGT)) {
+ SDOperand ANDNode = DAG.getNode(ISD::AND, LR.getValueType(), LL, RL);
+ AddToWorkList(ANDNode.Val);
+ return DAG.getSetCC(VT, ANDNode, LR, Op1);
+ }
+ }
+ // canonicalize equivalent to ll == rl
+ if (LL == RR && LR == RL) {
+ Op1 = ISD::getSetCCSwappedOperands(Op1);
+ std::swap(RL, RR);
+ }
+ if (LL == RL && LR == RR) {
+ bool isInteger = MVT::isInteger(LL.getValueType());
+ ISD::CondCode Result = ISD::getSetCCOrOperation(Op0, Op1, isInteger);
+ if (Result != ISD::SETCC_INVALID)
+ return DAG.getSetCC(N0.getValueType(), LL, LR, Result);
+ }
+ }
+
+ // Simplify: or (op x...), (op y...) -> (op (or x, y))
+ if (N0.getOpcode() == N1.getOpcode()) {
+ SDOperand Tmp = SimplifyBinOpWithSameOpcodeHands(N);
+ if (Tmp.Val) return Tmp;
+ }
+
+ // (X & C1) | (Y & C2) -> (X|Y) & C3 if possible.
+ if (N0.getOpcode() == ISD::AND &&
+ N1.getOpcode() == ISD::AND &&
+ N0.getOperand(1).getOpcode() == ISD::Constant &&
+ N1.getOperand(1).getOpcode() == ISD::Constant &&
+ // Don't increase # computations.
+ (N0.Val->hasOneUse() || N1.Val->hasOneUse())) {
+ // We can only do this xform if we know that bits from X that are set in C2
+ // but not in C1 are already zero. Likewise for Y.
+ uint64_t LHSMask = cast<ConstantSDNode>(N0.getOperand(1))->getValue();
+ uint64_t RHSMask = cast<ConstantSDNode>(N1.getOperand(1))->getValue();
+
+ if (DAG.MaskedValueIsZero(N0.getOperand(0), RHSMask&~LHSMask) &&
+ DAG.MaskedValueIsZero(N1.getOperand(0), LHSMask&~RHSMask)) {
+ SDOperand X =DAG.getNode(ISD::OR, VT, N0.getOperand(0), N1.getOperand(0));
+ return DAG.getNode(ISD::AND, VT, X, DAG.getConstant(LHSMask|RHSMask, VT));
+ }
+ }
+
+
+ // See if this is some rotate idiom.
+ if (SDNode *Rot = MatchRotate(N0, N1))
+ return SDOperand(Rot, 0);
+
+ return SDOperand();
+}
+
+
+/// MatchRotateHalf - Match "(X shl/srl V1) & V2" where V2 may not be present.
+static bool MatchRotateHalf(SDOperand Op, SDOperand &Shift, SDOperand &Mask) {
+ if (Op.getOpcode() == ISD::AND) {
+ if (isa<ConstantSDNode>(Op.getOperand(1))) {
+ Mask = Op.getOperand(1);
+ Op = Op.getOperand(0);
+ } else {
+ return false;
+ }
+ }
+
+ if (Op.getOpcode() == ISD::SRL || Op.getOpcode() == ISD::SHL) {
+ Shift = Op;
+ return true;
+ }
+ return false;
+}
+
+
+// MatchRotate - Handle an 'or' of two operands. If this is one of the many
+// idioms for rotate, and if the target supports rotation instructions, generate
+// a rot[lr].
+SDNode *DAGCombiner::MatchRotate(SDOperand LHS, SDOperand RHS) {
+ // Must be a legal type. Expanded an promoted things won't work with rotates.
+ MVT::ValueType VT = LHS.getValueType();
+ if (!TLI.isTypeLegal(VT)) return 0;
+
+ // The target must have at least one rotate flavor.
+ bool HasROTL = TLI.isOperationLegal(ISD::ROTL, VT);
+ bool HasROTR = TLI.isOperationLegal(ISD::ROTR, VT);
+ if (!HasROTL && !HasROTR) return 0;
+
+ // Match "(X shl/srl V1) & V2" where V2 may not be present.
+ SDOperand LHSShift; // The shift.
+ SDOperand LHSMask; // AND value if any.
+ if (!MatchRotateHalf(LHS, LHSShift, LHSMask))
+ return 0; // Not part of a rotate.
+
+ SDOperand RHSShift; // The shift.
+ SDOperand RHSMask; // AND value if any.
+ if (!MatchRotateHalf(RHS, RHSShift, RHSMask))
+ return 0; // Not part of a rotate.
+
+ if (LHSShift.getOperand(0) != RHSShift.getOperand(0))
+ return 0; // Not shifting the same value.
+
+ if (LHSShift.getOpcode() == RHSShift.getOpcode())
+ return 0; // Shifts must disagree.
+
+ // Canonicalize shl to left side in a shl/srl pair.
+ if (RHSShift.getOpcode() == ISD::SHL) {
+ std::swap(LHS, RHS);
+ std::swap(LHSShift, RHSShift);
+ std::swap(LHSMask , RHSMask );
+ }
+
+ unsigned OpSizeInBits = MVT::getSizeInBits(VT);
+ SDOperand LHSShiftArg = LHSShift.getOperand(0);
+ SDOperand LHSShiftAmt = LHSShift.getOperand(1);
+ SDOperand RHSShiftAmt = RHSShift.getOperand(1);
+
+ // fold (or (shl x, C1), (srl x, C2)) -> (rotl x, C1)
+ // fold (or (shl x, C1), (srl x, C2)) -> (rotr x, C2)
+ if (LHSShiftAmt.getOpcode() == ISD::Constant &&
+ RHSShiftAmt.getOpcode() == ISD::Constant) {
+ uint64_t LShVal = cast<ConstantSDNode>(LHSShiftAmt)->getValue();
+ uint64_t RShVal = cast<ConstantSDNode>(RHSShiftAmt)->getValue();
+ if ((LShVal + RShVal) != OpSizeInBits)
+ return 0;
+
+ SDOperand Rot;
+ if (HasROTL)
+ Rot = DAG.getNode(ISD::ROTL, VT, LHSShiftArg, LHSShiftAmt);
+ else
+ Rot = DAG.getNode(ISD::ROTR, VT, LHSShiftArg, RHSShiftAmt);
+
+ // If there is an AND of either shifted operand, apply it to the result.
+ if (LHSMask.Val || RHSMask.Val) {
+ uint64_t Mask = MVT::getIntVTBitMask(VT);
+
+ if (LHSMask.Val) {
+ uint64_t RHSBits = (1ULL << LShVal)-1;
+ Mask &= cast<ConstantSDNode>(LHSMask)->getValue() | RHSBits;
+ }
+ if (RHSMask.Val) {
+ uint64_t LHSBits = ~((1ULL << (OpSizeInBits-RShVal))-1);
+ Mask &= cast<ConstantSDNode>(RHSMask)->getValue() | LHSBits;
+ }
+
+ Rot = DAG.getNode(ISD::AND, VT, Rot, DAG.getConstant(Mask, VT));
+ }
+
+ return Rot.Val;
+ }
+
+ // If there is a mask here, and we have a variable shift, we can't be sure
+ // that we're masking out the right stuff.
+ if (LHSMask.Val || RHSMask.Val)
+ return 0;
+
+ // fold (or (shl x, y), (srl x, (sub 32, y))) -> (rotl x, y)
+ // fold (or (shl x, y), (srl x, (sub 32, y))) -> (rotr x, (sub 32, y))
+ if (RHSShiftAmt.getOpcode() == ISD::SUB &&
+ LHSShiftAmt == RHSShiftAmt.getOperand(1)) {
+ if (ConstantSDNode *SUBC =
+ dyn_cast<ConstantSDNode>(RHSShiftAmt.getOperand(0))) {
+ if (SUBC->getValue() == OpSizeInBits)
+ if (HasROTL)
+ return DAG.getNode(ISD::ROTL, VT, LHSShiftArg, LHSShiftAmt).Val;
+ else
+ return DAG.getNode(ISD::ROTR, VT, LHSShiftArg, RHSShiftAmt).Val;
+ }
+ }
+
+ // fold (or (shl x, (sub 32, y)), (srl x, r)) -> (rotr x, y)
+ // fold (or (shl x, (sub 32, y)), (srl x, r)) -> (rotl x, (sub 32, y))
+ if (LHSShiftAmt.getOpcode() == ISD::SUB &&
+ RHSShiftAmt == LHSShiftAmt.getOperand(1)) {
+ if (ConstantSDNode *SUBC =
+ dyn_cast<ConstantSDNode>(LHSShiftAmt.getOperand(0))) {
+ if (SUBC->getValue() == OpSizeInBits)
+ if (HasROTL)
+ return DAG.getNode(ISD::ROTL, VT, LHSShiftArg, LHSShiftAmt).Val;
+ else
+ return DAG.getNode(ISD::ROTR, VT, LHSShiftArg, RHSShiftAmt).Val;
+ }
+ }
+
+ // Look for sign/zext/any-extended cases:
+ if ((LHSShiftAmt.getOpcode() == ISD::SIGN_EXTEND
+ || LHSShiftAmt.getOpcode() == ISD::ZERO_EXTEND
+ || LHSShiftAmt.getOpcode() == ISD::ANY_EXTEND) &&
+ (RHSShiftAmt.getOpcode() == ISD::SIGN_EXTEND
+ || RHSShiftAmt.getOpcode() == ISD::ZERO_EXTEND
+ || RHSShiftAmt.getOpcode() == ISD::ANY_EXTEND)) {
+ SDOperand LExtOp0 = LHSShiftAmt.getOperand(0);
+ SDOperand RExtOp0 = RHSShiftAmt.getOperand(0);
+ if (RExtOp0.getOpcode() == ISD::SUB &&
+ RExtOp0.getOperand(1) == LExtOp0) {
+ // fold (or (shl x, (*ext y)), (srl x, (*ext (sub 32, y)))) ->
+ // (rotr x, y)
+ // fold (or (shl x, (*ext y)), (srl x, (*ext (sub 32, y)))) ->
+ // (rotl x, (sub 32, y))
+ if (ConstantSDNode *SUBC = cast<ConstantSDNode>(RExtOp0.getOperand(0))) {
+ if (SUBC->getValue() == OpSizeInBits) {
+ if (HasROTL)
+ return DAG.getNode(ISD::ROTL, VT, LHSShiftArg, LHSShiftAmt).Val;
+ else
+ return DAG.getNode(ISD::ROTR, VT, LHSShiftArg, RHSShiftAmt).Val;
+ }
+ }
+ } else if (LExtOp0.getOpcode() == ISD::SUB &&
+ RExtOp0 == LExtOp0.getOperand(1)) {
+ // fold (or (shl x, (*ext (sub 32, y))), (srl x, (*ext r))) ->
+ // (rotl x, y)
+ // fold (or (shl x, (*ext (sub 32, y))), (srl x, (*ext r))) ->
+ // (rotr x, (sub 32, y))
+ if (ConstantSDNode *SUBC = cast<ConstantSDNode>(LExtOp0.getOperand(0))) {
+ if (SUBC->getValue() == OpSizeInBits) {
+ if (HasROTL)
+ return DAG.getNode(ISD::ROTL, VT, LHSShiftArg, RHSShiftAmt).Val;
+ else
+ return DAG.getNode(ISD::ROTL, VT, LHSShiftArg, LHSShiftAmt).Val;
+ }
+ }
+ }
+ }
+
+ return 0;
+}
+
+
+SDOperand DAGCombiner::visitXOR(SDNode *N) {
+ SDOperand N0 = N->getOperand(0);
+ SDOperand N1 = N->getOperand(1);
+ SDOperand LHS, RHS, CC;
+ ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
+ ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
+ MVT::ValueType VT = N0.getValueType();
+
+ // fold vector ops
+ if (MVT::isVector(VT)) {
+ SDOperand FoldedVOp = SimplifyVBinOp(N);
+ if (FoldedVOp.Val) return FoldedVOp;
+ }
+
+ // fold (xor x, undef) -> undef
+ if (N0.getOpcode() == ISD::UNDEF)
+ return N0;
+ if (N1.getOpcode() == ISD::UNDEF)
+ return N1;
+ // fold (xor c1, c2) -> c1^c2
+ if (N0C && N1C)
+ return DAG.getNode(ISD::XOR, VT, N0, N1);
+ // canonicalize constant to RHS
+ if (N0C && !N1C)
+ return DAG.getNode(ISD::XOR, VT, N1, N0);
+ // fold (xor x, 0) -> x
+ if (N1C && N1C->isNullValue())
+ return N0;
+ // reassociate xor
+ SDOperand RXOR = ReassociateOps(ISD::XOR, N0, N1);
+ if (RXOR.Val != 0)
+ return RXOR;
+ // fold !(x cc y) -> (x !cc y)
+ if (N1C && N1C->getValue() == 1 && isSetCCEquivalent(N0, LHS, RHS, CC)) {
+ bool isInt = MVT::isInteger(LHS.getValueType());
+ ISD::CondCode NotCC = ISD::getSetCCInverse(cast<CondCodeSDNode>(CC)->get(),
+ isInt);
+ if (N0.getOpcode() == ISD::SETCC)
+ return DAG.getSetCC(VT, LHS, RHS, NotCC);
+ if (N0.getOpcode() == ISD::SELECT_CC)
+ return DAG.getSelectCC(LHS, RHS, N0.getOperand(2),N0.getOperand(3),NotCC);
+ assert(0 && "Unhandled SetCC Equivalent!");
+ abort();
+ }
+ // fold !(x or y) -> (!x and !y) iff x or y are setcc
+ if (N1C && N1C->getValue() == 1 && VT == MVT::i1 &&
+ (N0.getOpcode() == ISD::OR || N0.getOpcode() == ISD::AND)) {
+ SDOperand LHS = N0.getOperand(0), RHS = N0.getOperand(1);
+ if (isOneUseSetCC(RHS) || isOneUseSetCC(LHS)) {
+ unsigned NewOpcode = N0.getOpcode() == ISD::AND ? ISD::OR : ISD::AND;
+ LHS = DAG.getNode(ISD::XOR, VT, LHS, N1); // RHS = ~LHS
+ RHS = DAG.getNode(ISD::XOR, VT, RHS, N1); // RHS = ~RHS
+ AddToWorkList(LHS.Val); AddToWorkList(RHS.Val);
+ return DAG.getNode(NewOpcode, VT, LHS, RHS);
+ }
+ }
+ // fold !(x or y) -> (!x and !y) iff x or y are constants
+ if (N1C && N1C->isAllOnesValue() &&
+ (N0.getOpcode() == ISD::OR || N0.getOpcode() == ISD::AND)) {
+ SDOperand LHS = N0.getOperand(0), RHS = N0.getOperand(1);
+ if (isa<ConstantSDNode>(RHS) || isa<ConstantSDNode>(LHS)) {
+ unsigned NewOpcode = N0.getOpcode() == ISD::AND ? ISD::OR : ISD::AND;
+ LHS = DAG.getNode(ISD::XOR, VT, LHS, N1); // RHS = ~LHS
+ RHS = DAG.getNode(ISD::XOR, VT, RHS, N1); // RHS = ~RHS
+ AddToWorkList(LHS.Val); AddToWorkList(RHS.Val);
+ return DAG.getNode(NewOpcode, VT, LHS, RHS);
+ }
+ }
+ // fold (xor (xor x, c1), c2) -> (xor x, c1^c2)
+ if (N1C && N0.getOpcode() == ISD::XOR) {
+ ConstantSDNode *N00C = dyn_cast<ConstantSDNode>(N0.getOperand(0));
+ ConstantSDNode *N01C = dyn_cast<ConstantSDNode>(N0.getOperand(1));
+ if (N00C)
+ return DAG.getNode(ISD::XOR, VT, N0.getOperand(1),
+ DAG.getConstant(N1C->getValue()^N00C->getValue(), VT));
+ if (N01C)
+ return DAG.getNode(ISD::XOR, VT, N0.getOperand(0),
+ DAG.getConstant(N1C->getValue()^N01C->getValue(), VT));
+ }
+ // fold (xor x, x) -> 0
+ if (N0 == N1) {
+ if (!MVT::isVector(VT)) {
+ return DAG.getConstant(0, VT);
+ } else if (!AfterLegalize || TLI.isOperationLegal(ISD::BUILD_VECTOR, VT)) {
+ // Produce a vector of zeros.
+ SDOperand El = DAG.getConstant(0, MVT::getVectorElementType(VT));
+ std::vector<SDOperand> Ops(MVT::getVectorNumElements(VT), El);
+ return DAG.getNode(ISD::BUILD_VECTOR, VT, &Ops[0], Ops.size());
+ }
+ }
+
+ // Simplify: xor (op x...), (op y...) -> (op (xor x, y))
+ if (N0.getOpcode() == N1.getOpcode()) {
+ SDOperand Tmp = SimplifyBinOpWithSameOpcodeHands(N);
+ if (Tmp.Val) return Tmp;
+ }
+
+ // Simplify the expression using non-local knowledge.
+ if (!MVT::isVector(VT) &&
+ SimplifyDemandedBits(SDOperand(N, 0)))
+ return SDOperand(N, 0);
+
+ return SDOperand();
+}
+
+SDOperand DAGCombiner::visitSHL(SDNode *N) {
+ SDOperand N0 = N->getOperand(0);
+ SDOperand N1 = N->getOperand(1);
+ ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
+ ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
+ MVT::ValueType VT = N0.getValueType();
+ unsigned OpSizeInBits = MVT::getSizeInBits(VT);
+
+ // fold (shl c1, c2) -> c1<<c2
+ if (N0C && N1C)
+ return DAG.getNode(ISD::SHL, VT, N0, N1);
+ // fold (shl 0, x) -> 0
+ if (N0C && N0C->isNullValue())
+ return N0;
+ // fold (shl x, c >= size(x)) -> undef
+ if (N1C && N1C->getValue() >= OpSizeInBits)
+ return DAG.getNode(ISD::UNDEF, VT);
+ // fold (shl x, 0) -> x
+ if (N1C && N1C->isNullValue())
+ return N0;
+ // if (shl x, c) is known to be zero, return 0
+ if (DAG.MaskedValueIsZero(SDOperand(N, 0), MVT::getIntVTBitMask(VT)))
+ return DAG.getConstant(0, VT);
+ if (N1C && SimplifyDemandedBits(SDOperand(N, 0)))
+ return SDOperand(N, 0);
+ // fold (shl (shl x, c1), c2) -> 0 or (shl x, c1+c2)
+ if (N1C && N0.getOpcode() == ISD::SHL &&
+ N0.getOperand(1).getOpcode() == ISD::Constant) {
+ uint64_t c1 = cast<ConstantSDNode>(N0.getOperand(1))->getValue();
+ uint64_t c2 = N1C->getValue();
+ if (c1 + c2 > OpSizeInBits)
+ return DAG.getConstant(0, VT);
+ return DAG.getNode(ISD::SHL, VT, N0.getOperand(0),
+ DAG.getConstant(c1 + c2, N1.getValueType()));
+ }
+ // fold (shl (srl x, c1), c2) -> (shl (and x, -1 << c1), c2-c1) or
+ // (srl (and x, -1 << c1), c1-c2)
+ if (N1C && N0.getOpcode() == ISD::SRL &&
+ N0.getOperand(1).getOpcode() == ISD::Constant) {
+ uint64_t c1 = cast<ConstantSDNode>(N0.getOperand(1))->getValue();
+ uint64_t c2 = N1C->getValue();
+ SDOperand Mask = DAG.getNode(ISD::AND, VT, N0.getOperand(0),
+ DAG.getConstant(~0ULL << c1, VT));
+ if (c2 > c1)
+ return DAG.getNode(ISD::SHL, VT, Mask,
+ DAG.getConstant(c2-c1, N1.getValueType()));
+ else
+ return DAG.getNode(ISD::SRL, VT, Mask,
+ DAG.getConstant(c1-c2, N1.getValueType()));
+ }
+ // fold (shl (sra x, c1), c1) -> (and x, -1 << c1)
+ if (N1C && N0.getOpcode() == ISD::SRA && N1 == N0.getOperand(1))
+ return DAG.getNode(ISD::AND, VT, N0.getOperand(0),
+ DAG.getConstant(~0ULL << N1C->getValue(), VT));
+ return SDOperand();
+}
+
+SDOperand DAGCombiner::visitSRA(SDNode *N) {
+ SDOperand N0 = N->getOperand(0);
+ SDOperand N1 = N->getOperand(1);
+ ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
+ ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
+ MVT::ValueType VT = N0.getValueType();
+
+ // fold (sra c1, c2) -> c1>>c2
+ if (N0C && N1C)
+ return DAG.getNode(ISD::SRA, VT, N0, N1);
+ // fold (sra 0, x) -> 0
+ if (N0C && N0C->isNullValue())
+ return N0;
+ // fold (sra -1, x) -> -1
+ if (N0C && N0C->isAllOnesValue())
+ return N0;
+ // fold (sra x, c >= size(x)) -> undef
+ if (N1C && N1C->getValue() >= MVT::getSizeInBits(VT))
+ return DAG.getNode(ISD::UNDEF, VT);
+ // fold (sra x, 0) -> x
+ if (N1C && N1C->isNullValue())
+ return N0;
+ // fold (sra (shl x, c1), c1) -> sext_inreg for some c1 and target supports
+ // sext_inreg.
+ if (N1C && N0.getOpcode() == ISD::SHL && N1 == N0.getOperand(1)) {
+ unsigned LowBits = MVT::getSizeInBits(VT) - (unsigned)N1C->getValue();
+ MVT::ValueType EVT;
+ switch (LowBits) {
+ default: EVT = MVT::Other; break;
+ case 1: EVT = MVT::i1; break;
+ case 8: EVT = MVT::i8; break;
+ case 16: EVT = MVT::i16; break;
+ case 32: EVT = MVT::i32; break;
+ }
+ if (EVT > MVT::Other && TLI.isOperationLegal(ISD::SIGN_EXTEND_INREG, EVT))
+ return DAG.getNode(ISD::SIGN_EXTEND_INREG, VT, N0.getOperand(0),
+ DAG.getValueType(EVT));
+ }
+
+ // fold (sra (sra x, c1), c2) -> (sra x, c1+c2)
+ if (N1C && N0.getOpcode() == ISD::SRA) {
+ if (ConstantSDNode *C1 = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
+ unsigned Sum = N1C->getValue() + C1->getValue();
+ if (Sum >= MVT::getSizeInBits(VT)) Sum = MVT::getSizeInBits(VT)-1;
+ return DAG.getNode(ISD::SRA, VT, N0.getOperand(0),
+ DAG.getConstant(Sum, N1C->getValueType(0)));
+ }
+ }
+
+ // Simplify, based on bits shifted out of the LHS.
+ if (N1C && SimplifyDemandedBits(SDOperand(N, 0)))
+ return SDOperand(N, 0);
+
+
+ // If the sign bit is known to be zero, switch this to a SRL.
+ if (DAG.MaskedValueIsZero(N0, MVT::getIntVTSignBit(VT)))
+ return DAG.getNode(ISD::SRL, VT, N0, N1);
+ return SDOperand();
+}
+
+SDOperand DAGCombiner::visitSRL(SDNode *N) {
+ SDOperand N0 = N->getOperand(0);
+ SDOperand N1 = N->getOperand(1);
+ ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
+ ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
+ MVT::ValueType VT = N0.getValueType();
+ unsigned OpSizeInBits = MVT::getSizeInBits(VT);
+
+ // fold (srl c1, c2) -> c1 >>u c2
+ if (N0C && N1C)
+ return DAG.getNode(ISD::SRL, VT, N0, N1);
+ // fold (srl 0, x) -> 0
+ if (N0C && N0C->isNullValue())
+ return N0;
+ // fold (srl x, c >= size(x)) -> undef
+ if (N1C && N1C->getValue() >= OpSizeInBits)
+ return DAG.getNode(ISD::UNDEF, VT);
+ // fold (srl x, 0) -> x
+ if (N1C && N1C->isNullValue())
+ return N0;
+ // if (srl x, c) is known to be zero, return 0
+ if (N1C && DAG.MaskedValueIsZero(SDOperand(N, 0), ~0ULL >> (64-OpSizeInBits)))
+ return DAG.getConstant(0, VT);
+
+ // fold (srl (srl x, c1), c2) -> 0 or (srl x, c1+c2)
+ if (N1C && N0.getOpcode() == ISD::SRL &&
+ N0.getOperand(1).getOpcode() == ISD::Constant) {
+ uint64_t c1 = cast<ConstantSDNode>(N0.getOperand(1))->getValue();
+ uint64_t c2 = N1C->getValue();
+ if (c1 + c2 > OpSizeInBits)
+ return DAG.getConstant(0, VT);
+ return DAG.getNode(ISD::SRL, VT, N0.getOperand(0),
+ DAG.getConstant(c1 + c2, N1.getValueType()));
+ }
+
+ // fold (srl (anyextend x), c) -> (anyextend (srl x, c))
+ if (N1C && N0.getOpcode() == ISD::ANY_EXTEND) {
+ // Shifting in all undef bits?
+ MVT::ValueType SmallVT = N0.getOperand(0).getValueType();
+ if (N1C->getValue() >= MVT::getSizeInBits(SmallVT))
+ return DAG.getNode(ISD::UNDEF, VT);
+
+ SDOperand SmallShift = DAG.getNode(ISD::SRL, SmallVT, N0.getOperand(0), N1);
+ AddToWorkList(SmallShift.Val);
+ return DAG.getNode(ISD::ANY_EXTEND, VT, SmallShift);
+ }
+
+ // fold (srl (sra X, Y), 31) -> (srl X, 31). This srl only looks at the sign
+ // bit, which is unmodified by sra.
+ if (N1C && N1C->getValue()+1 == MVT::getSizeInBits(VT)) {
+ if (N0.getOpcode() == ISD::SRA)
+ return DAG.getNode(ISD::SRL, VT, N0.getOperand(0), N1);
+ }
+
+ // fold (srl (ctlz x), "5") -> x iff x has one bit set (the low bit).
+ if (N1C && N0.getOpcode() == ISD::CTLZ &&
+ N1C->getValue() == Log2_32(MVT::getSizeInBits(VT))) {
+ uint64_t KnownZero, KnownOne, Mask = MVT::getIntVTBitMask(VT);
+ DAG.ComputeMaskedBits(N0.getOperand(0), Mask, KnownZero, KnownOne);
+
+ // If any of the input bits are KnownOne, then the input couldn't be all
+ // zeros, thus the result of the srl will always be zero.
+ if (KnownOne) return DAG.getConstant(0, VT);
+
+ // If all of the bits input the to ctlz node are known to be zero, then
+ // the result of the ctlz is "32" and the result of the shift is one.
+ uint64_t UnknownBits = ~KnownZero & Mask;
+ if (UnknownBits == 0) return DAG.getConstant(1, VT);
+
+ // Otherwise, check to see if there is exactly one bit input to the ctlz.
+ if ((UnknownBits & (UnknownBits-1)) == 0) {
+ // Okay, we know that only that the single bit specified by UnknownBits
+ // could be set on input to the CTLZ node. If this bit is set, the SRL
+ // will return 0, if it is clear, it returns 1. Change the CTLZ/SRL pair
+ // to an SRL,XOR pair, which is likely to simplify more.
+ unsigned ShAmt = CountTrailingZeros_64(UnknownBits);
+ SDOperand Op = N0.getOperand(0);
+ if (ShAmt) {
+ Op = DAG.getNode(ISD::SRL, VT, Op,
+ DAG.getConstant(ShAmt, TLI.getShiftAmountTy()));
+ AddToWorkList(Op.Val);
+ }
+ return DAG.getNode(ISD::XOR, VT, Op, DAG.getConstant(1, VT));
+ }
+ }
+
+ // fold operands of srl based on knowledge that the low bits are not
+ // demanded.
+ if (N1C && SimplifyDemandedBits(SDOperand(N, 0)))
+ return SDOperand(N, 0);
+
+ return SDOperand();
+}
+
+SDOperand DAGCombiner::visitCTLZ(SDNode *N) {
+ SDOperand N0 = N->getOperand(0);
+ MVT::ValueType VT = N->getValueType(0);
+
+ // fold (ctlz c1) -> c2
+ if (isa<ConstantSDNode>(N0))
+ return DAG.getNode(ISD::CTLZ, VT, N0);
+ return SDOperand();
+}
+
+SDOperand DAGCombiner::visitCTTZ(SDNode *N) {
+ SDOperand N0 = N->getOperand(0);
+ MVT::ValueType VT = N->getValueType(0);
+
+ // fold (cttz c1) -> c2
+ if (isa<ConstantSDNode>(N0))
+ return DAG.getNode(ISD::CTTZ, VT, N0);
+ return SDOperand();
+}
+
+SDOperand DAGCombiner::visitCTPOP(SDNode *N) {
+ SDOperand N0 = N->getOperand(0);
+ MVT::ValueType VT = N->getValueType(0);
+
+ // fold (ctpop c1) -> c2
+ if (isa<ConstantSDNode>(N0))
+ return DAG.getNode(ISD::CTPOP, VT, N0);
+ return SDOperand();
+}
+
+SDOperand DAGCombiner::visitSELECT(SDNode *N) {
+ SDOperand N0 = N->getOperand(0);
+ SDOperand N1 = N->getOperand(1);
+ SDOperand N2 = N->getOperand(2);
+ ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
+ ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
+ ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2);
+ MVT::ValueType VT = N->getValueType(0);
+
+ // fold select C, X, X -> X
+ if (N1 == N2)
+ return N1;
+ // fold select true, X, Y -> X
+ if (N0C && !N0C->isNullValue())
+ return N1;
+ // fold select false, X, Y -> Y
+ if (N0C && N0C->isNullValue())
+ return N2;
+ // fold select C, 1, X -> C | X
+ if (MVT::i1 == VT && N1C && N1C->getValue() == 1)
+ return DAG.getNode(ISD::OR, VT, N0, N2);
+ // fold select C, 0, X -> ~C & X
+ // FIXME: this should check for C type == X type, not i1?
+ if (MVT::i1 == VT && N1C && N1C->isNullValue()) {
+ SDOperand XORNode = DAG.getNode(ISD::XOR, VT, N0, DAG.getConstant(1, VT));
+ AddToWorkList(XORNode.Val);
+ return DAG.getNode(ISD::AND, VT, XORNode, N2);
+ }
+ // fold select C, X, 1 -> ~C | X
+ if (MVT::i1 == VT && N2C && N2C->getValue() == 1) {
+ SDOperand XORNode = DAG.getNode(ISD::XOR, VT, N0, DAG.getConstant(1, VT));
+ AddToWorkList(XORNode.Val);
+ return DAG.getNode(ISD::OR, VT, XORNode, N1);
+ }
+ // fold select C, X, 0 -> C & X
+ // FIXME: this should check for C type == X type, not i1?
+ if (MVT::i1 == VT && N2C && N2C->isNullValue())
+ return DAG.getNode(ISD::AND, VT, N0, N1);
+ // fold X ? X : Y --> X ? 1 : Y --> X | Y
+ if (MVT::i1 == VT && N0 == N1)
+ return DAG.getNode(ISD::OR, VT, N0, N2);
+ // fold X ? Y : X --> X ? Y : 0 --> X & Y
+ if (MVT::i1 == VT && N0 == N2)
+ return DAG.getNode(ISD::AND, VT, N0, N1);
+
+ // If we can fold this based on the true/false value, do so.
+ if (SimplifySelectOps(N, N1, N2))
+ return SDOperand(N, 0); // Don't revisit N.
+
+ // fold selects based on a setcc into other things, such as min/max/abs
+ if (N0.getOpcode() == ISD::SETCC)
+ // FIXME:
+ // Check against MVT::Other for SELECT_CC, which is a workaround for targets
+ // having to say they don't support SELECT_CC on every type the DAG knows
+ // about, since there is no way to mark an opcode illegal at all value types
+ if (TLI.isOperationLegal(ISD::SELECT_CC, MVT::Other))
+ return DAG.getNode(ISD::SELECT_CC, VT, N0.getOperand(0), N0.getOperand(1),
+ N1, N2, N0.getOperand(2));
+ else
+ return SimplifySelect(N0, N1, N2);
+ return SDOperand();
+}
+
+SDOperand DAGCombiner::visitSELECT_CC(SDNode *N) {
+ SDOperand N0 = N->getOperand(0);
+ SDOperand N1 = N->getOperand(1);
+ SDOperand N2 = N->getOperand(2);
+ SDOperand N3 = N->getOperand(3);
+ SDOperand N4 = N->getOperand(4);
+ ISD::CondCode CC = cast<CondCodeSDNode>(N4)->get();
+
+ // fold select_cc lhs, rhs, x, x, cc -> x
+ if (N2 == N3)
+ return N2;
+
+ // Determine if the condition we're dealing with is constant
+ SDOperand SCC = SimplifySetCC(TLI.getSetCCResultTy(), N0, N1, CC, false);
+ if (SCC.Val) AddToWorkList(SCC.Val);
+
+ if (ConstantSDNode *SCCC = dyn_cast_or_null<ConstantSDNode>(SCC.Val)) {
+ if (SCCC->getValue())
+ return N2; // cond always true -> true val
+ else
+ return N3; // cond always false -> false val
+ }
+
+ // Fold to a simpler select_cc
+ if (SCC.Val && SCC.getOpcode() == ISD::SETCC)
+ return DAG.getNode(ISD::SELECT_CC, N2.getValueType(),
+ SCC.getOperand(0), SCC.getOperand(1), N2, N3,
+ SCC.getOperand(2));
+
+ // If we can fold this based on the true/false value, do so.
+ if (SimplifySelectOps(N, N2, N3))
+ return SDOperand(N, 0); // Don't revisit N.
+
+ // fold select_cc into other things, such as min/max/abs
+ return SimplifySelectCC(N0, N1, N2, N3, CC);
+}
+
+SDOperand DAGCombiner::visitSETCC(SDNode *N) {
+ return SimplifySetCC(N->getValueType(0), N->getOperand(0), N->getOperand(1),
+ cast<CondCodeSDNode>(N->getOperand(2))->get());
+}
+
+SDOperand DAGCombiner::visitSIGN_EXTEND(SDNode *N) {
+ SDOperand N0 = N->getOperand(0);
+ MVT::ValueType VT = N->getValueType(0);
+
+ // fold (sext c1) -> c1
+ if (isa<ConstantSDNode>(N0))
+ return DAG.getNode(ISD::SIGN_EXTEND, VT, N0);
+
+ // fold (sext (sext x)) -> (sext x)
+ // fold (sext (aext x)) -> (sext x)
+ if (N0.getOpcode() == ISD::SIGN_EXTEND || N0.getOpcode() == ISD::ANY_EXTEND)
+ return DAG.getNode(ISD::SIGN_EXTEND, VT, N0.getOperand(0));
+
+ // fold (sext (truncate (load x))) -> (sext (smaller load x))
+ // fold (sext (truncate (srl (load x), c))) -> (sext (smaller load (x+c/n)))
+ if (N0.getOpcode() == ISD::TRUNCATE) {
+ SDOperand NarrowLoad = ReduceLoadWidth(N0.Val);
+ if (NarrowLoad.Val) {
+ if (NarrowLoad.Val != N0.Val)
+ CombineTo(N0.Val, NarrowLoad);
+ return DAG.getNode(ISD::SIGN_EXTEND, VT, NarrowLoad);
+ }
+ }
+
+ // See if the value being truncated is already sign extended. If so, just
+ // eliminate the trunc/sext pair.
+ if (N0.getOpcode() == ISD::TRUNCATE) {
+ SDOperand Op = N0.getOperand(0);
+ unsigned OpBits = MVT::getSizeInBits(Op.getValueType());
+ unsigned MidBits = MVT::getSizeInBits(N0.getValueType());
+ unsigned DestBits = MVT::getSizeInBits(VT);
+ unsigned NumSignBits = DAG.ComputeNumSignBits(Op);
+
+ if (OpBits == DestBits) {
+ // Op is i32, Mid is i8, and Dest is i32. If Op has more than 24 sign
+ // bits, it is already ready.
+ if (NumSignBits > DestBits-MidBits)
+ return Op;
+ } else if (OpBits < DestBits) {
+ // Op is i32, Mid is i8, and Dest is i64. If Op has more than 24 sign
+ // bits, just sext from i32.
+ if (NumSignBits > OpBits-MidBits)
+ return DAG.getNode(ISD::SIGN_EXTEND, VT, Op);
+ } else {
+ // Op is i64, Mid is i8, and Dest is i32. If Op has more than 56 sign
+ // bits, just truncate to i32.
+ if (NumSignBits > OpBits-MidBits)
+ return DAG.getNode(ISD::TRUNCATE, VT, Op);
+ }
+
+ // fold (sext (truncate x)) -> (sextinreg x).
+ if (!AfterLegalize || TLI.isOperationLegal(ISD::SIGN_EXTEND_INREG,
+ N0.getValueType())) {
+ if (Op.getValueType() < VT)
+ Op = DAG.getNode(ISD::ANY_EXTEND, VT, Op);
+ else if (Op.getValueType() > VT)
+ Op = DAG.getNode(ISD::TRUNCATE, VT, Op);
+ return DAG.getNode(ISD::SIGN_EXTEND_INREG, VT, Op,
+ DAG.getValueType(N0.getValueType()));
+ }
+ }
+
+ // fold (sext (load x)) -> (sext (truncate (sextload x)))
+ if (ISD::isNON_EXTLoad(N0.Val) && N0.hasOneUse() &&
+ (!AfterLegalize||TLI.isLoadXLegal(ISD::SEXTLOAD, N0.getValueType()))){
+ LoadSDNode *LN0 = cast<LoadSDNode>(N0);
+ SDOperand ExtLoad = DAG.getExtLoad(ISD::SEXTLOAD, VT, LN0->getChain(),
+ LN0->getBasePtr(), LN0->getSrcValue(),
+ LN0->getSrcValueOffset(),
+ N0.getValueType(),
+ LN0->isVolatile(),
+ LN0->getAlignment());
+ CombineTo(N, ExtLoad);
+ CombineTo(N0.Val, DAG.getNode(ISD::TRUNCATE, N0.getValueType(), ExtLoad),
+ ExtLoad.getValue(1));
+ return SDOperand(N, 0); // Return N so it doesn't get rechecked!
+ }
+
+ // fold (sext (sextload x)) -> (sext (truncate (sextload x)))
+ // fold (sext ( extload x)) -> (sext (truncate (sextload x)))
+ if ((ISD::isSEXTLoad(N0.Val) || ISD::isEXTLoad(N0.Val)) &&
+ ISD::isUNINDEXEDLoad(N0.Val) && N0.hasOneUse()) {
+ LoadSDNode *LN0 = cast<LoadSDNode>(N0);
+ MVT::ValueType EVT = LN0->getLoadedVT();
+ if (!AfterLegalize || TLI.isLoadXLegal(ISD::SEXTLOAD, EVT)) {
+ SDOperand ExtLoad = DAG.getExtLoad(ISD::SEXTLOAD, VT, LN0->getChain(),
+ LN0->getBasePtr(), LN0->getSrcValue(),
+ LN0->getSrcValueOffset(), EVT,
+ LN0->isVolatile(),
+ LN0->getAlignment());
+ CombineTo(N, ExtLoad);
+ CombineTo(N0.Val, DAG.getNode(ISD::TRUNCATE, N0.getValueType(), ExtLoad),
+ ExtLoad.getValue(1));
+ return SDOperand(N, 0); // Return N so it doesn't get rechecked!
+ }
+ }
+
+ // sext(setcc x,y,cc) -> select_cc x, y, -1, 0, cc
+ if (N0.getOpcode() == ISD::SETCC) {
+ SDOperand SCC =
+ SimplifySelectCC(N0.getOperand(0), N0.getOperand(1),
+ DAG.getConstant(~0ULL, VT), DAG.getConstant(0, VT),
+ cast<CondCodeSDNode>(N0.getOperand(2))->get(), true);
+ if (SCC.Val) return SCC;
+ }
+
+ return SDOperand();
+}
+
+SDOperand DAGCombiner::visitZERO_EXTEND(SDNode *N) {
+ SDOperand N0 = N->getOperand(0);
+ MVT::ValueType VT = N->getValueType(0);
+
+ // fold (zext c1) -> c1
+ if (isa<ConstantSDNode>(N0))
+ return DAG.getNode(ISD::ZERO_EXTEND, VT, N0);
+ // fold (zext (zext x)) -> (zext x)
+ // fold (zext (aext x)) -> (zext x)
+ if (N0.getOpcode() == ISD::ZERO_EXTEND || N0.getOpcode() == ISD::ANY_EXTEND)
+ return DAG.getNode(ISD::ZERO_EXTEND, VT, N0.getOperand(0));
+
+ // fold (zext (truncate (load x))) -> (zext (smaller load x))
+ // fold (zext (truncate (srl (load x), c))) -> (zext (small load (x+c/n)))
+ if (N0.getOpcode() == ISD::TRUNCATE) {
+ SDOperand NarrowLoad = ReduceLoadWidth(N0.Val);
+ if (NarrowLoad.Val) {
+ if (NarrowLoad.Val != N0.Val)
+ CombineTo(N0.Val, NarrowLoad);
+ return DAG.getNode(ISD::ZERO_EXTEND, VT, NarrowLoad);
+ }
+ }
+
+ // fold (zext (truncate x)) -> (and x, mask)
+ if (N0.getOpcode() == ISD::TRUNCATE &&
+ (!AfterLegalize || TLI.isOperationLegal(ISD::AND, VT))) {
+ SDOperand Op = N0.getOperand(0);
+ if (Op.getValueType() < VT) {
+ Op = DAG.getNode(ISD::ANY_EXTEND, VT, Op);
+ } else if (Op.getValueType() > VT) {
+ Op = DAG.getNode(ISD::TRUNCATE, VT, Op);
+ }
+ return DAG.getZeroExtendInReg(Op, N0.getValueType());
+ }
+
+ // fold (zext (and (trunc x), cst)) -> (and x, cst).
+ if (N0.getOpcode() == ISD::AND &&
+ N0.getOperand(0).getOpcode() == ISD::TRUNCATE &&
+ N0.getOperand(1).getOpcode() == ISD::Constant) {
+ SDOperand X = N0.getOperand(0).getOperand(0);
+ if (X.getValueType() < VT) {
+ X = DAG.getNode(ISD::ANY_EXTEND, VT, X);
+ } else if (X.getValueType() > VT) {
+ X = DAG.getNode(ISD::TRUNCATE, VT, X);
+ }
+ uint64_t Mask = cast<ConstantSDNode>(N0.getOperand(1))->getValue();
+ return DAG.getNode(ISD::AND, VT, X, DAG.getConstant(Mask, VT));
+ }
+
+ // fold (zext (load x)) -> (zext (truncate (zextload x)))
+ if (ISD::isNON_EXTLoad(N0.Val) && N0.hasOneUse() &&
+ (!AfterLegalize||TLI.isLoadXLegal(ISD::ZEXTLOAD, N0.getValueType()))) {
+ LoadSDNode *LN0 = cast<LoadSDNode>(N0);
+ SDOperand ExtLoad = DAG.getExtLoad(ISD::ZEXTLOAD, VT, LN0->getChain(),
+ LN0->getBasePtr(), LN0->getSrcValue(),
+ LN0->getSrcValueOffset(),
+ N0.getValueType(),
+ LN0->isVolatile(),
+ LN0->getAlignment());
+ CombineTo(N, ExtLoad);
+ CombineTo(N0.Val, DAG.getNode(ISD::TRUNCATE, N0.getValueType(), ExtLoad),
+ ExtLoad.getValue(1));
+ return SDOperand(N, 0); // Return N so it doesn't get rechecked!
+ }
+
+ // fold (zext (zextload x)) -> (zext (truncate (zextload x)))
+ // fold (zext ( extload x)) -> (zext (truncate (zextload x)))
+ if ((ISD::isZEXTLoad(N0.Val) || ISD::isEXTLoad(N0.Val)) &&
+ ISD::isUNINDEXEDLoad(N0.Val) && N0.hasOneUse()) {
+ LoadSDNode *LN0 = cast<LoadSDNode>(N0);
+ MVT::ValueType EVT = LN0->getLoadedVT();
+ SDOperand ExtLoad = DAG.getExtLoad(ISD::ZEXTLOAD, VT, LN0->getChain(),
+ LN0->getBasePtr(), LN0->getSrcValue(),
+ LN0->getSrcValueOffset(), EVT,
+ LN0->isVolatile(),
+ LN0->getAlignment());
+ CombineTo(N, ExtLoad);
+ CombineTo(N0.Val, DAG.getNode(ISD::TRUNCATE, N0.getValueType(), ExtLoad),
+ ExtLoad.getValue(1));
+ return SDOperand(N, 0); // Return N so it doesn't get rechecked!
+ }
+
+ // zext(setcc x,y,cc) -> select_cc x, y, 1, 0, cc
+ if (N0.getOpcode() == ISD::SETCC) {
+ SDOperand SCC =
+ SimplifySelectCC(N0.getOperand(0), N0.getOperand(1),
+ DAG.getConstant(1, VT), DAG.getConstant(0, VT),
+ cast<CondCodeSDNode>(N0.getOperand(2))->get(), true);
+ if (SCC.Val) return SCC;
+ }
+
+ return SDOperand();
+}
+
+SDOperand DAGCombiner::visitANY_EXTEND(SDNode *N) {
+ SDOperand N0 = N->getOperand(0);
+ MVT::ValueType VT = N->getValueType(0);
+
+ // fold (aext c1) -> c1
+ if (isa<ConstantSDNode>(N0))
+ return DAG.getNode(ISD::ANY_EXTEND, VT, N0);
+ // fold (aext (aext x)) -> (aext x)
+ // fold (aext (zext x)) -> (zext x)
+ // fold (aext (sext x)) -> (sext x)
+ if (N0.getOpcode() == ISD::ANY_EXTEND ||
+ N0.getOpcode() == ISD::ZERO_EXTEND ||
+ N0.getOpcode() == ISD::SIGN_EXTEND)
+ return DAG.getNode(N0.getOpcode(), VT, N0.getOperand(0));
+
+ // fold (aext (truncate (load x))) -> (aext (smaller load x))
+ // fold (aext (truncate (srl (load x), c))) -> (aext (small load (x+c/n)))
+ if (N0.getOpcode() == ISD::TRUNCATE) {
+ SDOperand NarrowLoad = ReduceLoadWidth(N0.Val);
+ if (NarrowLoad.Val) {
+ if (NarrowLoad.Val != N0.Val)
+ CombineTo(N0.Val, NarrowLoad);
+ return DAG.getNode(ISD::ANY_EXTEND, VT, NarrowLoad);
+ }
+ }
+
+ // fold (aext (truncate x))
+ if (N0.getOpcode() == ISD::TRUNCATE) {
+ SDOperand TruncOp = N0.getOperand(0);
+ if (TruncOp.getValueType() == VT)
+ return TruncOp; // x iff x size == zext size.
+ if (TruncOp.getValueType() > VT)
+ return DAG.getNode(ISD::TRUNCATE, VT, TruncOp);
+ return DAG.getNode(ISD::ANY_EXTEND, VT, TruncOp);
+ }
+
+ // fold (aext (and (trunc x), cst)) -> (and x, cst).
+ if (N0.getOpcode() == ISD::AND &&
+ N0.getOperand(0).getOpcode() == ISD::TRUNCATE &&
+ N0.getOperand(1).getOpcode() == ISD::Constant) {
+ SDOperand X = N0.getOperand(0).getOperand(0);
+ if (X.getValueType() < VT) {
+ X = DAG.getNode(ISD::ANY_EXTEND, VT, X);
+ } else if (X.getValueType() > VT) {
+ X = DAG.getNode(ISD::TRUNCATE, VT, X);
+ }
+ uint64_t Mask = cast<ConstantSDNode>(N0.getOperand(1))->getValue();
+ return DAG.getNode(ISD::AND, VT, X, DAG.getConstant(Mask, VT));
+ }
+
+ // fold (aext (load x)) -> (aext (truncate (extload x)))
+ if (ISD::isNON_EXTLoad(N0.Val) && N0.hasOneUse() &&
+ (!AfterLegalize||TLI.isLoadXLegal(ISD::EXTLOAD, N0.getValueType()))) {
+ LoadSDNode *LN0 = cast<LoadSDNode>(N0);
+ SDOperand ExtLoad = DAG.getExtLoad(ISD::EXTLOAD, VT, LN0->getChain(),
+ LN0->getBasePtr(), LN0->getSrcValue(),
+ LN0->getSrcValueOffset(),
+ N0.getValueType(),
+ LN0->isVolatile(),
+ LN0->getAlignment());
+ CombineTo(N, ExtLoad);
+ CombineTo(N0.Val, DAG.getNode(ISD::TRUNCATE, N0.getValueType(), ExtLoad),
+ ExtLoad.getValue(1));
+ return SDOperand(N, 0); // Return N so it doesn't get rechecked!
+ }
+
+ // fold (aext (zextload x)) -> (aext (truncate (zextload x)))
+ // fold (aext (sextload x)) -> (aext (truncate (sextload x)))
+ // fold (aext ( extload x)) -> (aext (truncate (extload x)))
+ if (N0.getOpcode() == ISD::LOAD &&
+ !ISD::isNON_EXTLoad(N0.Val) && ISD::isUNINDEXEDLoad(N0.Val) &&
+ N0.hasOneUse()) {
+ LoadSDNode *LN0 = cast<LoadSDNode>(N0);
+ MVT::ValueType EVT = LN0->getLoadedVT();
+ SDOperand ExtLoad = DAG.getExtLoad(LN0->getExtensionType(), VT,
+ LN0->getChain(), LN0->getBasePtr(),
+ LN0->getSrcValue(),
+ LN0->getSrcValueOffset(), EVT,
+ LN0->isVolatile(),
+ LN0->getAlignment());
+ CombineTo(N, ExtLoad);
+ CombineTo(N0.Val, DAG.getNode(ISD::TRUNCATE, N0.getValueType(), ExtLoad),
+ ExtLoad.getValue(1));
+ return SDOperand(N, 0); // Return N so it doesn't get rechecked!
+ }
+
+ // aext(setcc x,y,cc) -> select_cc x, y, 1, 0, cc
+ if (N0.getOpcode() == ISD::SETCC) {
+ SDOperand SCC =
+ SimplifySelectCC(N0.getOperand(0), N0.getOperand(1),
+ DAG.getConstant(1, VT), DAG.getConstant(0, VT),
+ cast<CondCodeSDNode>(N0.getOperand(2))->get(), true);
+ if (SCC.Val)
+ return SCC;
+ }
+
+ return SDOperand();
+}
+
+/// ReduceLoadWidth - If the result of a wider load is shifted to right of N
+/// bits and then truncated to a narrower type and where N is a multiple
+/// of number of bits of the narrower type, transform it to a narrower load
+/// from address + N / num of bits of new type. If the result is to be
+/// extended, also fold the extension to form a extending load.
+SDOperand DAGCombiner::ReduceLoadWidth(SDNode *N) {
+ unsigned Opc = N->getOpcode();
+ ISD::LoadExtType ExtType = ISD::NON_EXTLOAD;
+ SDOperand N0 = N->getOperand(0);
+ MVT::ValueType VT = N->getValueType(0);
+ MVT::ValueType EVT = N->getValueType(0);
+
+ // Special case: SIGN_EXTEND_INREG is basically truncating to EVT then
+ // extended to VT.
+ if (Opc == ISD::SIGN_EXTEND_INREG) {
+ ExtType = ISD::SEXTLOAD;
+ EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
+ if (AfterLegalize && !TLI.isLoadXLegal(ISD::SEXTLOAD, EVT))
+ return SDOperand();
+ }
+
+ unsigned EVTBits = MVT::getSizeInBits(EVT);
+ unsigned ShAmt = 0;
+ bool CombineSRL = false;
+ if (N0.getOpcode() == ISD::SRL && N0.hasOneUse()) {
+ if (ConstantSDNode *N01 = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
+ ShAmt = N01->getValue();
+ // Is the shift amount a multiple of size of VT?
+ if ((ShAmt & (EVTBits-1)) == 0) {
+ N0 = N0.getOperand(0);
+ if (MVT::getSizeInBits(N0.getValueType()) <= EVTBits)
+ return SDOperand();
+ CombineSRL = true;
+ }
+ }
+ }
+
+ if (ISD::isNON_EXTLoad(N0.Val) && N0.hasOneUse() &&
+ // Do not allow folding to i1 here. i1 is implicitly stored in memory in
+ // zero extended form: by shrinking the load, we lose track of the fact
+ // that it is already zero extended.
+ // FIXME: This should be reevaluated.
+ VT != MVT::i1) {
+ assert(MVT::getSizeInBits(N0.getValueType()) > EVTBits &&
+ "Cannot truncate to larger type!");
+ LoadSDNode *LN0 = cast<LoadSDNode>(N0);
+ MVT::ValueType PtrType = N0.getOperand(1).getValueType();
+ // For big endian targets, we need to adjust the offset to the pointer to
+ // load the correct bytes.
+ if (!TLI.isLittleEndian())
+ ShAmt = MVT::getSizeInBits(N0.getValueType()) - ShAmt - EVTBits;
+ uint64_t PtrOff = ShAmt / 8;
+ SDOperand NewPtr = DAG.getNode(ISD::ADD, PtrType, LN0->getBasePtr(),
+ DAG.getConstant(PtrOff, PtrType));
+ AddToWorkList(NewPtr.Val);
+ SDOperand Load = (ExtType == ISD::NON_EXTLOAD)
+ ? DAG.getLoad(VT, LN0->getChain(), NewPtr,
+ LN0->getSrcValue(), LN0->getSrcValueOffset(),
+ LN0->isVolatile(), LN0->getAlignment())
+ : DAG.getExtLoad(ExtType, VT, LN0->getChain(), NewPtr,
+ LN0->getSrcValue(), LN0->getSrcValueOffset(), EVT,
+ LN0->isVolatile(), LN0->getAlignment());
+ AddToWorkList(N);
+ if (CombineSRL) {
+ std::vector<SDNode*> NowDead;
+ DAG.ReplaceAllUsesOfValueWith(N0.getValue(1), Load.getValue(1), NowDead);
+ CombineTo(N->getOperand(0).Val, Load);
+ } else
+ CombineTo(N0.Val, Load, Load.getValue(1));
+ if (ShAmt) {
+ if (Opc == ISD::SIGN_EXTEND_INREG)
+ return DAG.getNode(Opc, VT, Load, N->getOperand(1));
+ else
+ return DAG.getNode(Opc, VT, Load);
+ }
+ return SDOperand(N, 0); // Return N so it doesn't get rechecked!
+ }
+
+ return SDOperand();
+}
+
+
+SDOperand DAGCombiner::visitSIGN_EXTEND_INREG(SDNode *N) {
+ SDOperand N0 = N->getOperand(0);
+ SDOperand N1 = N->getOperand(1);
+ MVT::ValueType VT = N->getValueType(0);
+ MVT::ValueType EVT = cast<VTSDNode>(N1)->getVT();
+ unsigned EVTBits = MVT::getSizeInBits(EVT);
+
+ // fold (sext_in_reg c1) -> c1
+ if (isa<ConstantSDNode>(N0) || N0.getOpcode() == ISD::UNDEF)
+ return DAG.getNode(ISD::SIGN_EXTEND_INREG, VT, N0, N1);
+
+ // If the input is already sign extended, just drop the extension.
+ if (DAG.ComputeNumSignBits(N0) >= MVT::getSizeInBits(VT)-EVTBits+1)
+ return N0;
+
+ // fold (sext_in_reg (sext_in_reg x, VT2), VT1) -> (sext_in_reg x, minVT) pt2
+ if (N0.getOpcode() == ISD::SIGN_EXTEND_INREG &&
+ EVT < cast<VTSDNode>(N0.getOperand(1))->getVT()) {
+ return DAG.getNode(ISD::SIGN_EXTEND_INREG, VT, N0.getOperand(0), N1);
+ }
+
+ // fold (sext_in_reg x) -> (zext_in_reg x) if the sign bit is known zero.
+ if (DAG.MaskedValueIsZero(N0, 1ULL << (EVTBits-1)))
+ return DAG.getZeroExtendInReg(N0, EVT);
+
+ // fold operands of sext_in_reg based on knowledge that the top bits are not
+ // demanded.
+ if (SimplifyDemandedBits(SDOperand(N, 0)))
+ return SDOperand(N, 0);
+
+ // fold (sext_in_reg (load x)) -> (smaller sextload x)
+ // fold (sext_in_reg (srl (load x), c)) -> (smaller sextload (x+c/evtbits))
+ SDOperand NarrowLoad = ReduceLoadWidth(N);
+ if (NarrowLoad.Val)
+ return NarrowLoad;
+
+ // fold (sext_in_reg (srl X, 24), i8) -> sra X, 24
+ // fold (sext_in_reg (srl X, 23), i8) -> sra X, 23 iff possible.
+ // We already fold "(sext_in_reg (srl X, 25), i8) -> srl X, 25" above.
+ if (N0.getOpcode() == ISD::SRL) {
+ if (ConstantSDNode *ShAmt = dyn_cast<ConstantSDNode>(N0.getOperand(1)))
+ if (ShAmt->getValue()+EVTBits <= MVT::getSizeInBits(VT)) {
+ // We can turn this into an SRA iff the input to the SRL is already sign
+ // extended enough.
+ unsigned InSignBits = DAG.ComputeNumSignBits(N0.getOperand(0));
+ if (MVT::getSizeInBits(VT)-(ShAmt->getValue()+EVTBits) < InSignBits)
+ return DAG.getNode(ISD::SRA, VT, N0.getOperand(0), N0.getOperand(1));
+ }
+ }
+
+ // fold (sext_inreg (extload x)) -> (sextload x)
+ if (ISD::isEXTLoad(N0.Val) &&
+ ISD::isUNINDEXEDLoad(N0.Val) &&
+ EVT == cast<LoadSDNode>(N0)->getLoadedVT() &&
+ (!AfterLegalize || TLI.isLoadXLegal(ISD::SEXTLOAD, EVT))) {
+ LoadSDNode *LN0 = cast<LoadSDNode>(N0);
+ SDOperand ExtLoad = DAG.getExtLoad(ISD::SEXTLOAD, VT, LN0->getChain(),
+ LN0->getBasePtr(), LN0->getSrcValue(),
+ LN0->getSrcValueOffset(), EVT,
+ LN0->isVolatile(),
+ LN0->getAlignment());
+ CombineTo(N, ExtLoad);
+ CombineTo(N0.Val, ExtLoad, ExtLoad.getValue(1));
+ return SDOperand(N, 0); // Return N so it doesn't get rechecked!
+ }
+ // fold (sext_inreg (zextload x)) -> (sextload x) iff load has one use
+ if (ISD::isZEXTLoad(N0.Val) && ISD::isUNINDEXEDLoad(N0.Val) &&
+ N0.hasOneUse() &&
+ EVT == cast<LoadSDNode>(N0)->getLoadedVT() &&
+ (!AfterLegalize || TLI.isLoadXLegal(ISD::SEXTLOAD, EVT))) {
+ LoadSDNode *LN0 = cast<LoadSDNode>(N0);
+ SDOperand ExtLoad = DAG.getExtLoad(ISD::SEXTLOAD, VT, LN0->getChain(),
+ LN0->getBasePtr(), LN0->getSrcValue(),
+ LN0->getSrcValueOffset(), EVT,
+ LN0->isVolatile(),
+ LN0->getAlignment());
+ CombineTo(N, ExtLoad);
+ CombineTo(N0.Val, ExtLoad, ExtLoad.getValue(1));
+ return SDOperand(N, 0); // Return N so it doesn't get rechecked!
+ }
+ return SDOperand();
+}
+
+SDOperand DAGCombiner::visitTRUNCATE(SDNode *N) {
+ SDOperand N0 = N->getOperand(0);
+ MVT::ValueType VT = N->getValueType(0);
+
+ // noop truncate
+ if (N0.getValueType() == N->getValueType(0))
+ return N0;
+ // fold (truncate c1) -> c1
+ if (isa<ConstantSDNode>(N0))
+ return DAG.getNode(ISD::TRUNCATE, VT, N0);
+ // fold (truncate (truncate x)) -> (truncate x)
+ if (N0.getOpcode() == ISD::TRUNCATE)
+ return DAG.getNode(ISD::TRUNCATE, VT, N0.getOperand(0));
+ // fold (truncate (ext x)) -> (ext x) or (truncate x) or x
+ if (N0.getOpcode() == ISD::ZERO_EXTEND || N0.getOpcode() == ISD::SIGN_EXTEND||
+ N0.getOpcode() == ISD::ANY_EXTEND) {
+ if (N0.getOperand(0).getValueType() < VT)
+ // if the source is smaller than the dest, we still need an extend
+ return DAG.getNode(N0.getOpcode(), VT, N0.getOperand(0));
+ else if (N0.getOperand(0).getValueType() > VT)
+ // if the source is larger than the dest, than we just need the truncate
+ return DAG.getNode(ISD::TRUNCATE, VT, N0.getOperand(0));
+ else
+ // if the source and dest are the same type, we can drop both the extend
+ // and the truncate
+ return N0.getOperand(0);
+ }
+
+ // fold (truncate (load x)) -> (smaller load x)
+ // fold (truncate (srl (load x), c)) -> (smaller load (x+c/evtbits))
+ return ReduceLoadWidth(N);
+}
+
+SDOperand DAGCombiner::visitBIT_CONVERT(SDNode *N) {
+ SDOperand N0 = N->getOperand(0);
+ MVT::ValueType VT = N->getValueType(0);
+
+ // If the input is a BUILD_VECTOR with all constant elements, fold this now.
+ // Only do this before legalize, since afterward the target may be depending
+ // on the bitconvert.
+ // First check to see if this is all constant.
+ if (!AfterLegalize &&
+ N0.getOpcode() == ISD::BUILD_VECTOR && N0.Val->hasOneUse() &&
+ MVT::isVector(VT)) {
+ bool isSimple = true;
+ for (unsigned i = 0, e = N0.getNumOperands(); i != e; ++i)
+ if (N0.getOperand(i).getOpcode() != ISD::UNDEF &&
+ N0.getOperand(i).getOpcode() != ISD::Constant &&
+ N0.getOperand(i).getOpcode() != ISD::ConstantFP) {
+ isSimple = false;
+ break;
+ }
+
+ MVT::ValueType DestEltVT = MVT::getVectorElementType(N->getValueType(0));
+ assert(!MVT::isVector(DestEltVT) &&
+ "Element type of vector ValueType must not be vector!");
+ if (isSimple) {
+ return ConstantFoldBIT_CONVERTofBUILD_VECTOR(N0.Val, DestEltVT);
+ }
+ }
+
+ // If the input is a constant, let getNode() fold it.
+ if (isa<ConstantSDNode>(N0) || isa<ConstantFPSDNode>(N0)) {
+ SDOperand Res = DAG.getNode(ISD::BIT_CONVERT, VT, N0);
+ if (Res.Val != N) return Res;
+ }
+
+ if (N0.getOpcode() == ISD::BIT_CONVERT) // conv(conv(x,t1),t2) -> conv(x,t2)
+ return DAG.getNode(ISD::BIT_CONVERT, VT, N0.getOperand(0));
+
+ // fold (conv (load x)) -> (load (conv*)x)
+ // If the resultant load doesn't need a higher alignment than the original!
+ if (ISD::isNON_EXTLoad(N0.Val) && N0.hasOneUse() &&
+ ISD::isUNINDEXEDLoad(N0.Val) &&
+ TLI.isOperationLegal(ISD::LOAD, VT)) {
+ LoadSDNode *LN0 = cast<LoadSDNode>(N0);
+ unsigned Align = TLI.getTargetMachine().getTargetData()->
+ getABITypeAlignment(MVT::getTypeForValueType(VT));
+ unsigned OrigAlign = LN0->getAlignment();
+ if (Align <= OrigAlign) {
+ SDOperand Load = DAG.getLoad(VT, LN0->getChain(), LN0->getBasePtr(),
+ LN0->getSrcValue(), LN0->getSrcValueOffset(),
+ LN0->isVolatile(), Align);
+ AddToWorkList(N);
+ CombineTo(N0.Val, DAG.getNode(ISD::BIT_CONVERT, N0.getValueType(), Load),
+ Load.getValue(1));
+ return Load;
+ }
+ }
+
+ return SDOperand();
+}
+
+/// ConstantFoldBIT_CONVERTofBUILD_VECTOR - We know that BV is a build_vector
+/// node with Constant, ConstantFP or Undef operands. DstEltVT indicates the
+/// destination element value type.
+SDOperand DAGCombiner::
+ConstantFoldBIT_CONVERTofBUILD_VECTOR(SDNode *BV, MVT::ValueType DstEltVT) {
+ MVT::ValueType SrcEltVT = BV->getOperand(0).getValueType();
+
+ // If this is already the right type, we're done.
+ if (SrcEltVT == DstEltVT) return SDOperand(BV, 0);
+
+ unsigned SrcBitSize = MVT::getSizeInBits(SrcEltVT);
+ unsigned DstBitSize = MVT::getSizeInBits(DstEltVT);
+
+ // If this is a conversion of N elements of one type to N elements of another
+ // type, convert each element. This handles FP<->INT cases.
+ if (SrcBitSize == DstBitSize) {
+ SmallVector<SDOperand, 8> Ops;
+ for (unsigned i = 0, e = BV->getNumOperands(); i != e; ++i) {
+ Ops.push_back(DAG.getNode(ISD::BIT_CONVERT, DstEltVT, BV->getOperand(i)));
+ AddToWorkList(Ops.back().Val);
+ }
+ MVT::ValueType VT =
+ MVT::getVectorType(DstEltVT,
+ MVT::getVectorNumElements(BV->getValueType(0)));
+ return DAG.getNode(ISD::BUILD_VECTOR, VT, &Ops[0], Ops.size());
+ }
+
+ // Otherwise, we're growing or shrinking the elements. To avoid having to
+ // handle annoying details of growing/shrinking FP values, we convert them to
+ // int first.
+ if (MVT::isFloatingPoint(SrcEltVT)) {
+ // Convert the input float vector to a int vector where the elements are the
+ // same sizes.
+ assert((SrcEltVT == MVT::f32 || SrcEltVT == MVT::f64) && "Unknown FP VT!");
+ MVT::ValueType IntVT = SrcEltVT == MVT::f32 ? MVT::i32 : MVT::i64;
+ BV = ConstantFoldBIT_CONVERTofBUILD_VECTOR(BV, IntVT).Val;
+ SrcEltVT = IntVT;
+ }
+
+ // Now we know the input is an integer vector. If the output is a FP type,
+ // convert to integer first, then to FP of the right size.
+ if (MVT::isFloatingPoint(DstEltVT)) {
+ assert((DstEltVT == MVT::f32 || DstEltVT == MVT::f64) && "Unknown FP VT!");
+ MVT::ValueType TmpVT = DstEltVT == MVT::f32 ? MVT::i32 : MVT::i64;
+ SDNode *Tmp = ConstantFoldBIT_CONVERTofBUILD_VECTOR(BV, TmpVT).Val;
+
+ // Next, convert to FP elements of the same size.
+ return ConstantFoldBIT_CONVERTofBUILD_VECTOR(Tmp, DstEltVT);
+ }
+
+ // Okay, we know the src/dst types are both integers of differing types.
+ // Handling growing first.
+ assert(MVT::isInteger(SrcEltVT) && MVT::isInteger(DstEltVT));
+ if (SrcBitSize < DstBitSize) {
+ unsigned NumInputsPerOutput = DstBitSize/SrcBitSize;
+
+ SmallVector<SDOperand, 8> Ops;
+ for (unsigned i = 0, e = BV->getNumOperands(); i != e;
+ i += NumInputsPerOutput) {
+ bool isLE = TLI.isLittleEndian();
+ uint64_t NewBits = 0;
+ bool EltIsUndef = true;
+ for (unsigned j = 0; j != NumInputsPerOutput; ++j) {
+ // Shift the previously computed bits over.
+ NewBits <<= SrcBitSize;
+ SDOperand Op = BV->getOperand(i+ (isLE ? (NumInputsPerOutput-j-1) : j));
+ if (Op.getOpcode() == ISD::UNDEF) continue;
+ EltIsUndef = false;
+
+ NewBits |= cast<ConstantSDNode>(Op)->getValue();
+ }
+
+ if (EltIsUndef)
+ Ops.push_back(DAG.getNode(ISD::UNDEF, DstEltVT));
+ else
+ Ops.push_back(DAG.getConstant(NewBits, DstEltVT));
+ }
+
+ MVT::ValueType VT = MVT::getVectorType(DstEltVT,
+ Ops.size());
+ return DAG.getNode(ISD::BUILD_VECTOR, VT, &Ops[0], Ops.size());
+ }
+
+ // Finally, this must be the case where we are shrinking elements: each input
+ // turns into multiple outputs.
+ unsigned NumOutputsPerInput = SrcBitSize/DstBitSize;
+ SmallVector<SDOperand, 8> Ops;
+ for (unsigned i = 0, e = BV->getNumOperands(); i != e; ++i) {
+ if (BV->getOperand(i).getOpcode() == ISD::UNDEF) {
+ for (unsigned j = 0; j != NumOutputsPerInput; ++j)
+ Ops.push_back(DAG.getNode(ISD::UNDEF, DstEltVT));
+ continue;
+ }
+ uint64_t OpVal = cast<ConstantSDNode>(BV->getOperand(i))->getValue();
+
+ for (unsigned j = 0; j != NumOutputsPerInput; ++j) {
+ unsigned ThisVal = OpVal & ((1ULL << DstBitSize)-1);
+ OpVal >>= DstBitSize;
+ Ops.push_back(DAG.getConstant(ThisVal, DstEltVT));
+ }
+
+ // For big endian targets, swap the order of the pieces of each element.
+ if (!TLI.isLittleEndian())
+ std::reverse(Ops.end()-NumOutputsPerInput, Ops.end());
+ }
+ MVT::ValueType VT = MVT::getVectorType(DstEltVT, Ops.size());
+ return DAG.getNode(ISD::BUILD_VECTOR, VT, &Ops[0], Ops.size());
+}
+
+
+
+SDOperand DAGCombiner::visitFADD(SDNode *N) {
+ SDOperand N0 = N->getOperand(0);
+ SDOperand N1 = N->getOperand(1);
+ ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(N0);
+ ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1);
+ MVT::ValueType VT = N->getValueType(0);
+
+ // fold vector ops
+ if (MVT::isVector(VT)) {
+ SDOperand FoldedVOp = SimplifyVBinOp(N);
+ if (FoldedVOp.Val) return FoldedVOp;
+ }
+
+ // fold (fadd c1, c2) -> c1+c2
+ if (N0CFP && N1CFP)
+ return DAG.getNode(ISD::FADD, VT, N0, N1);
+ // canonicalize constant to RHS
+ if (N0CFP && !N1CFP)
+ return DAG.getNode(ISD::FADD, VT, N1, N0);
+ // fold (A + (-B)) -> A-B
+ if (isNegatibleForFree(N1) == 2)
+ return DAG.getNode(ISD::FSUB, VT, N0, GetNegatedExpression(N1, DAG));
+ // fold ((-A) + B) -> B-A
+ if (isNegatibleForFree(N0) == 2)
+ return DAG.getNode(ISD::FSUB, VT, N1, GetNegatedExpression(N0, DAG));
+
+ // If allowed, fold (fadd (fadd x, c1), c2) -> (fadd x, (fadd c1, c2))
+ if (UnsafeFPMath && N1CFP && N0.getOpcode() == ISD::FADD &&
+ N0.Val->hasOneUse() && isa<ConstantFPSDNode>(N0.getOperand(1)))
+ return DAG.getNode(ISD::FADD, VT, N0.getOperand(0),
+ DAG.getNode(ISD::FADD, VT, N0.getOperand(1), N1));
+
+ return SDOperand();
+}
+
+SDOperand DAGCombiner::visitFSUB(SDNode *N) {
+ SDOperand N0 = N->getOperand(0);
+ SDOperand N1 = N->getOperand(1);
+ ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(N0);
+ ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1);
+ MVT::ValueType VT = N->getValueType(0);
+
+ // fold vector ops
+ if (MVT::isVector(VT)) {
+ SDOperand FoldedVOp = SimplifyVBinOp(N);
+ if (FoldedVOp.Val) return FoldedVOp;
+ }
+
+ // fold (fsub c1, c2) -> c1-c2
+ if (N0CFP && N1CFP)
+ return DAG.getNode(ISD::FSUB, VT, N0, N1);
+ // fold (0-B) -> -B
+ if (UnsafeFPMath && N0CFP && N0CFP->getValue() == 0.0) {
+ if (isNegatibleForFree(N1))
+ return GetNegatedExpression(N1, DAG);
+ return DAG.getNode(ISD::FNEG, VT, N1);
+ }
+ // fold (A-(-B)) -> A+B
+ if (isNegatibleForFree(N1))
+ return DAG.getNode(ISD::FADD, VT, N0, GetNegatedExpression(N1, DAG));
+
+ return SDOperand();
+}
+
+SDOperand DAGCombiner::visitFMUL(SDNode *N) {
+ SDOperand N0 = N->getOperand(0);
+ SDOperand N1 = N->getOperand(1);
+ ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(N0);
+ ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1);
+ MVT::ValueType VT = N->getValueType(0);
+
+ // fold vector ops
+ if (MVT::isVector(VT)) {
+ SDOperand FoldedVOp = SimplifyVBinOp(N);
+ if (FoldedVOp.Val) return FoldedVOp;
+ }
+
+ // fold (fmul c1, c2) -> c1*c2
+ if (N0CFP && N1CFP)
+ return DAG.getNode(ISD::FMUL, VT, N0, N1);
+ // canonicalize constant to RHS
+ if (N0CFP && !N1CFP)
+ return DAG.getNode(ISD::FMUL, VT, N1, N0);
+ // fold (fmul X, 2.0) -> (fadd X, X)
+ if (N1CFP && N1CFP->isExactlyValue(+2.0))
+ return DAG.getNode(ISD::FADD, VT, N0, N0);
+ // fold (fmul X, -1.0) -> (fneg X)
+ if (N1CFP && N1CFP->isExactlyValue(-1.0))
+ return DAG.getNode(ISD::FNEG, VT, N0);
+
+ // -X * -Y -> X*Y
+ if (char LHSNeg = isNegatibleForFree(N0)) {
+ if (char RHSNeg = isNegatibleForFree(N1)) {
+ // Both can be negated for free, check to see if at least one is cheaper
+ // negated.
+ if (LHSNeg == 2 || RHSNeg == 2)
+ return DAG.getNode(ISD::FMUL, VT, GetNegatedExpression(N0, DAG),
+ GetNegatedExpression(N1, DAG));
+ }
+ }
+
+ // If allowed, fold (fmul (fmul x, c1), c2) -> (fmul x, (fmul c1, c2))
+ if (UnsafeFPMath && N1CFP && N0.getOpcode() == ISD::FMUL &&
+ N0.Val->hasOneUse() && isa<ConstantFPSDNode>(N0.getOperand(1)))
+ return DAG.getNode(ISD::FMUL, VT, N0.getOperand(0),
+ DAG.getNode(ISD::FMUL, VT, N0.getOperand(1), N1));
+
+ return SDOperand();
+}
+
+SDOperand DAGCombiner::visitFDIV(SDNode *N) {
+ SDOperand N0 = N->getOperand(0);
+ SDOperand N1 = N->getOperand(1);
+ ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(N0);
+ ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1);
+ MVT::ValueType VT = N->getValueType(0);
+
+ // fold vector ops
+ if (MVT::isVector(VT)) {
+ SDOperand FoldedVOp = SimplifyVBinOp(N);
+ if (FoldedVOp.Val) return FoldedVOp;
+ }
+
+ // fold (fdiv c1, c2) -> c1/c2
+ if (N0CFP && N1CFP)
+ return DAG.getNode(ISD::FDIV, VT, N0, N1);
+
+
+ // -X / -Y -> X*Y
+ if (char LHSNeg = isNegatibleForFree(N0)) {
+ if (char RHSNeg = isNegatibleForFree(N1)) {
+ // Both can be negated for free, check to see if at least one is cheaper
+ // negated.
+ if (LHSNeg == 2 || RHSNeg == 2)
+ return DAG.getNode(ISD::FDIV, VT, GetNegatedExpression(N0, DAG),
+ GetNegatedExpression(N1, DAG));
+ }
+ }
+
+ return SDOperand();
+}
+
+SDOperand DAGCombiner::visitFREM(SDNode *N) {
+ SDOperand N0 = N->getOperand(0);
+ SDOperand N1 = N->getOperand(1);
+ ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(N0);
+ ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1);
+ MVT::ValueType VT = N->getValueType(0);
+
+ // fold (frem c1, c2) -> fmod(c1,c2)
+ if (N0CFP && N1CFP)
+ return DAG.getNode(ISD::FREM, VT, N0, N1);
+
+ return SDOperand();
+}
+
+SDOperand DAGCombiner::visitFCOPYSIGN(SDNode *N) {
+ SDOperand N0 = N->getOperand(0);
+ SDOperand N1 = N->getOperand(1);
+ ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(N0);
+ ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1);
+ MVT::ValueType VT = N->getValueType(0);
+
+ if (N0CFP && N1CFP) // Constant fold
+ return DAG.getNode(ISD::FCOPYSIGN, VT, N0, N1);
+
+ if (N1CFP) {
+ // copysign(x, c1) -> fabs(x) iff ispos(c1)
+ // copysign(x, c1) -> fneg(fabs(x)) iff isneg(c1)
+ union {
+ double d;
+ int64_t i;
+ } u;
+ u.d = N1CFP->getValue();
+ if (u.i >= 0)
+ return DAG.getNode(ISD::FABS, VT, N0);
+ else
+ return DAG.getNode(ISD::FNEG, VT, DAG.getNode(ISD::FABS, VT, N0));
+ }
+
+ // copysign(fabs(x), y) -> copysign(x, y)
+ // copysign(fneg(x), y) -> copysign(x, y)
+ // copysign(copysign(x,z), y) -> copysign(x, y)
+ if (N0.getOpcode() == ISD::FABS || N0.getOpcode() == ISD::FNEG ||
+ N0.getOpcode() == ISD::FCOPYSIGN)
+ return DAG.getNode(ISD::FCOPYSIGN, VT, N0.getOperand(0), N1);
+
+ // copysign(x, abs(y)) -> abs(x)
+ if (N1.getOpcode() == ISD::FABS)
+ return DAG.getNode(ISD::FABS, VT, N0);
+
+ // copysign(x, copysign(y,z)) -> copysign(x, z)
+ if (N1.getOpcode() == ISD::FCOPYSIGN)
+ return DAG.getNode(ISD::FCOPYSIGN, VT, N0, N1.getOperand(1));
+
+ // copysign(x, fp_extend(y)) -> copysign(x, y)
+ // copysign(x, fp_round(y)) -> copysign(x, y)
+ if (N1.getOpcode() == ISD::FP_EXTEND || N1.getOpcode() == ISD::FP_ROUND)
+ return DAG.getNode(ISD::FCOPYSIGN, VT, N0, N1.getOperand(0));
+
+ return SDOperand();
+}
+
+
+
+SDOperand DAGCombiner::visitSINT_TO_FP(SDNode *N) {
+ SDOperand N0 = N->getOperand(0);
+ ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
+ MVT::ValueType VT = N->getValueType(0);
+
+ // fold (sint_to_fp c1) -> c1fp
+ if (N0C)
+ return DAG.getNode(ISD::SINT_TO_FP, VT, N0);
+ return SDOperand();
+}
+
+SDOperand DAGCombiner::visitUINT_TO_FP(SDNode *N) {
+ SDOperand N0 = N->getOperand(0);
+ ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
+ MVT::ValueType VT = N->getValueType(0);
+
+ // fold (uint_to_fp c1) -> c1fp
+ if (N0C)
+ return DAG.getNode(ISD::UINT_TO_FP, VT, N0);
+ return SDOperand();
+}
+
+SDOperand DAGCombiner::visitFP_TO_SINT(SDNode *N) {
+ SDOperand N0 = N->getOperand(0);
+ ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(N0);
+ MVT::ValueType VT = N->getValueType(0);
+
+ // fold (fp_to_sint c1fp) -> c1
+ if (N0CFP)
+ return DAG.getNode(ISD::FP_TO_SINT, VT, N0);
+ return SDOperand();
+}
+
+SDOperand DAGCombiner::visitFP_TO_UINT(SDNode *N) {
+ SDOperand N0 = N->getOperand(0);
+ ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(N0);
+ MVT::ValueType VT = N->getValueType(0);
+
+ // fold (fp_to_uint c1fp) -> c1
+ if (N0CFP)
+ return DAG.getNode(ISD::FP_TO_UINT, VT, N0);
+ return SDOperand();
+}
+
+SDOperand DAGCombiner::visitFP_ROUND(SDNode *N) {
+ SDOperand N0 = N->getOperand(0);
+ ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(N0);
+ MVT::ValueType VT = N->getValueType(0);
+
+ // fold (fp_round c1fp) -> c1fp
+ if (N0CFP)
+ return DAG.getNode(ISD::FP_ROUND, VT, N0);
+
+ // fold (fp_round (fp_extend x)) -> x
+ if (N0.getOpcode() == ISD::FP_EXTEND && VT == N0.getOperand(0).getValueType())
+ return N0.getOperand(0);
+
+ // fold (fp_round (copysign X, Y)) -> (copysign (fp_round X), Y)
+ if (N0.getOpcode() == ISD::FCOPYSIGN && N0.Val->hasOneUse()) {
+ SDOperand Tmp = DAG.getNode(ISD::FP_ROUND, VT, N0.getOperand(0));
+ AddToWorkList(Tmp.Val);
+ return DAG.getNode(ISD::FCOPYSIGN, VT, Tmp, N0.getOperand(1));
+ }
+
+ return SDOperand();
+}
+
+SDOperand DAGCombiner::visitFP_ROUND_INREG(SDNode *N) {
+ SDOperand N0 = N->getOperand(0);
+ MVT::ValueType VT = N->getValueType(0);
+ MVT::ValueType EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
+ ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(N0);
+
+ // fold (fp_round_inreg c1fp) -> c1fp
+ if (N0CFP) {
+ SDOperand Round = DAG.getConstantFP(N0CFP->getValue(), EVT);
+ return DAG.getNode(ISD::FP_EXTEND, VT, Round);
+ }
+ return SDOperand();
+}
+
+SDOperand DAGCombiner::visitFP_EXTEND(SDNode *N) {
+ SDOperand N0 = N->getOperand(0);
+ ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(N0);
+ MVT::ValueType VT = N->getValueType(0);
+
+ // fold (fp_extend c1fp) -> c1fp
+ if (N0CFP)
+ return DAG.getNode(ISD::FP_EXTEND, VT, N0);
+
+ // fold (fpext (load x)) -> (fpext (fpround (extload x)))
+ if (ISD::isNON_EXTLoad(N0.Val) && N0.hasOneUse() &&
+ (!AfterLegalize||TLI.isLoadXLegal(ISD::EXTLOAD, N0.getValueType()))) {
+ LoadSDNode *LN0 = cast<LoadSDNode>(N0);
+ SDOperand ExtLoad = DAG.getExtLoad(ISD::EXTLOAD, VT, LN0->getChain(),
+ LN0->getBasePtr(), LN0->getSrcValue(),
+ LN0->getSrcValueOffset(),
+ N0.getValueType(),
+ LN0->isVolatile(),
+ LN0->getAlignment());
+ CombineTo(N, ExtLoad);
+ CombineTo(N0.Val, DAG.getNode(ISD::FP_ROUND, N0.getValueType(), ExtLoad),
+ ExtLoad.getValue(1));
+ return SDOperand(N, 0); // Return N so it doesn't get rechecked!
+ }
+
+
+ return SDOperand();
+}
+
+SDOperand DAGCombiner::visitFNEG(SDNode *N) {
+ SDOperand N0 = N->getOperand(0);
+
+ if (isNegatibleForFree(N0))
+ return GetNegatedExpression(N0, DAG);
+
+ return SDOperand();
+}
+
+SDOperand DAGCombiner::visitFABS(SDNode *N) {
+ SDOperand N0 = N->getOperand(0);
+ ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(N0);
+ MVT::ValueType VT = N->getValueType(0);
+
+ // fold (fabs c1) -> fabs(c1)
+ if (N0CFP)
+ return DAG.getNode(ISD::FABS, VT, N0);
+ // fold (fabs (fabs x)) -> (fabs x)
+ if (N0.getOpcode() == ISD::FABS)
+ return N->getOperand(0);
+ // fold (fabs (fneg x)) -> (fabs x)
+ // fold (fabs (fcopysign x, y)) -> (fabs x)
+ if (N0.getOpcode() == ISD::FNEG || N0.getOpcode() == ISD::FCOPYSIGN)
+ return DAG.getNode(ISD::FABS, VT, N0.getOperand(0));
+
+ return SDOperand();
+}
+
+SDOperand DAGCombiner::visitBRCOND(SDNode *N) {
+ SDOperand Chain = N->getOperand(0);
+ SDOperand N1 = N->getOperand(1);
+ SDOperand N2 = N->getOperand(2);
+ ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
+
+ // never taken branch, fold to chain
+ if (N1C && N1C->isNullValue())
+ return Chain;
+ // unconditional branch
+ if (N1C && N1C->getValue() == 1)
+ return DAG.getNode(ISD::BR, MVT::Other, Chain, N2);
+ // fold a brcond with a setcc condition into a BR_CC node if BR_CC is legal
+ // on the target.
+ if (N1.getOpcode() == ISD::SETCC &&
+ TLI.isOperationLegal(ISD::BR_CC, MVT::Other)) {
+ return DAG.getNode(ISD::BR_CC, MVT::Other, Chain, N1.getOperand(2),
+ N1.getOperand(0), N1.getOperand(1), N2);
+ }
+ return SDOperand();
+}
+
+// Operand List for BR_CC: Chain, CondCC, CondLHS, CondRHS, DestBB.
+//
+SDOperand DAGCombiner::visitBR_CC(SDNode *N) {
+ CondCodeSDNode *CC = cast<CondCodeSDNode>(N->getOperand(1));
+ SDOperand CondLHS = N->getOperand(2), CondRHS = N->getOperand(3);
+
+ // Use SimplifySetCC to simplify SETCC's.
+ SDOperand Simp = SimplifySetCC(MVT::i1, CondLHS, CondRHS, CC->get(), false);
+ if (Simp.Val) AddToWorkList(Simp.Val);
+
+ ConstantSDNode *SCCC = dyn_cast_or_null<ConstantSDNode>(Simp.Val);
+
+ // fold br_cc true, dest -> br dest (unconditional branch)
+ if (SCCC && SCCC->getValue())
+ return DAG.getNode(ISD::BR, MVT::Other, N->getOperand(0),
+ N->getOperand(4));
+ // fold br_cc false, dest -> unconditional fall through
+ if (SCCC && SCCC->isNullValue())
+ return N->getOperand(0);
+
+ // fold to a simpler setcc
+ if (Simp.Val && Simp.getOpcode() == ISD::SETCC)
+ return DAG.getNode(ISD::BR_CC, MVT::Other, N->getOperand(0),
+ Simp.getOperand(2), Simp.getOperand(0),
+ Simp.getOperand(1), N->getOperand(4));
+ return SDOperand();
+}
+
+
+/// CombineToPreIndexedLoadStore - Try turning a load / store and a
+/// pre-indexed load / store when the base pointer is a add or subtract
+/// and it has other uses besides the load / store. After the
+/// transformation, the new indexed load / store has effectively folded
+/// the add / subtract in and all of its other uses are redirected to the
+/// new load / store.
+bool DAGCombiner::CombineToPreIndexedLoadStore(SDNode *N) {
+ if (!AfterLegalize)
+ return false;
+
+ bool isLoad = true;
+ SDOperand Ptr;
+ MVT::ValueType VT;
+ if (LoadSDNode *LD = dyn_cast<LoadSDNode>(N)) {
+ if (LD->getAddressingMode() != ISD::UNINDEXED)
+ return false;
+ VT = LD->getLoadedVT();
+ if (!TLI.isIndexedLoadLegal(ISD::PRE_INC, VT) &&
+ !TLI.isIndexedLoadLegal(ISD::PRE_DEC, VT))
+ return false;
+ Ptr = LD->getBasePtr();
+ } else if (StoreSDNode *ST = dyn_cast<StoreSDNode>(N)) {
+ if (ST->getAddressingMode() != ISD::UNINDEXED)
+ return false;
+ VT = ST->getStoredVT();
+ if (!TLI.isIndexedStoreLegal(ISD::PRE_INC, VT) &&
+ !TLI.isIndexedStoreLegal(ISD::PRE_DEC, VT))
+ return false;
+ Ptr = ST->getBasePtr();
+ isLoad = false;
+ } else
+ return false;
+
+ // If the pointer is not an add/sub, or if it doesn't have multiple uses, bail
+ // out. There is no reason to make this a preinc/predec.
+ if ((Ptr.getOpcode() != ISD::ADD && Ptr.getOpcode() != ISD::SUB) ||
+ Ptr.Val->hasOneUse())
+ return false;
+
+ // Ask the target to do addressing mode selection.
+ SDOperand BasePtr;
+ SDOperand Offset;
+ ISD::MemIndexedMode AM = ISD::UNINDEXED;
+ if (!TLI.getPreIndexedAddressParts(N, BasePtr, Offset, AM, DAG))
+ return false;
+ // Don't create a indexed load / store with zero offset.
+ if (isa<ConstantSDNode>(Offset) &&
+ cast<ConstantSDNode>(Offset)->getValue() == 0)
+ return false;
+
+ // Try turning it into a pre-indexed load / store except when:
+ // 1) The new base ptr is a frame index.
+ // 2) If N is a store and the new base ptr is either the same as or is a
+ // predecessor of the value being stored.
+ // 3) Another use of old base ptr is a predecessor of N. If ptr is folded
+ // that would create a cycle.
+ // 4) All uses are load / store ops that use it as old base ptr.
+
+ // Check #1. Preinc'ing a frame index would require copying the stack pointer
+ // (plus the implicit offset) to a register to preinc anyway.
+ if (isa<FrameIndexSDNode>(BasePtr))
+ return false;
+
+ // Check #2.
+ if (!isLoad) {
+ SDOperand Val = cast<StoreSDNode>(N)->getValue();
+ if (Val == BasePtr || BasePtr.Val->isPredecessor(Val.Val))
+ return false;
+ }
+
+ // Now check for #3 and #4.
+ bool RealUse = false;
+ for (SDNode::use_iterator I = Ptr.Val->use_begin(),
+ E = Ptr.Val->use_end(); I != E; ++I) {
+ SDNode *Use = *I;
+ if (Use == N)
+ continue;
+ if (Use->isPredecessor(N))
+ return false;
+
+ if (!((Use->getOpcode() == ISD::LOAD &&
+ cast<LoadSDNode>(Use)->getBasePtr() == Ptr) ||
+ (Use->getOpcode() == ISD::STORE) &&
+ cast<StoreSDNode>(Use)->getBasePtr() == Ptr))
+ RealUse = true;
+ }
+ if (!RealUse)
+ return false;
+
+ SDOperand Result;
+ if (isLoad)
+ Result = DAG.getIndexedLoad(SDOperand(N,0), BasePtr, Offset, AM);
+ else
+ Result = DAG.getIndexedStore(SDOperand(N,0), BasePtr, Offset, AM);
+ ++PreIndexedNodes;
+ ++NodesCombined;
+ DOUT << "\nReplacing.4 "; DEBUG(N->dump(&DAG));
+ DOUT << "\nWith: "; DEBUG(Result.Val->dump(&DAG));
+ DOUT << '\n';
+ std::vector<SDNode*> NowDead;
+ if (isLoad) {
+ DAG.ReplaceAllUsesOfValueWith(SDOperand(N, 0), Result.getValue(0),
+ NowDead);
+ DAG.ReplaceAllUsesOfValueWith(SDOperand(N, 1), Result.getValue(2),
+ NowDead);
+ } else {
+ DAG.ReplaceAllUsesOfValueWith(SDOperand(N, 0), Result.getValue(1),
+ NowDead);
+ }
+
+ // Nodes can end up on the worklist more than once. Make sure we do
+ // not process a node that has been replaced.
+ for (unsigned i = 0, e = NowDead.size(); i != e; ++i)
+ removeFromWorkList(NowDead[i]);
+ // Finally, since the node is now dead, remove it from the graph.
+ DAG.DeleteNode(N);
+
+ // Replace the uses of Ptr with uses of the updated base value.
+ DAG.ReplaceAllUsesOfValueWith(Ptr, Result.getValue(isLoad ? 1 : 0),
+ NowDead);
+ removeFromWorkList(Ptr.Val);
+ for (unsigned i = 0, e = NowDead.size(); i != e; ++i)
+ removeFromWorkList(NowDead[i]);
+ DAG.DeleteNode(Ptr.Val);
+
+ return true;
+}
+
+/// CombineToPostIndexedLoadStore - Try combine a load / store with a
+/// add / sub of the base pointer node into a post-indexed load / store.
+/// The transformation folded the add / subtract into the new indexed
+/// load / store effectively and all of its uses are redirected to the
+/// new load / store.
+bool DAGCombiner::CombineToPostIndexedLoadStore(SDNode *N) {
+ if (!AfterLegalize)
+ return false;
+
+ bool isLoad = true;
+ SDOperand Ptr;
+ MVT::ValueType VT;
+ if (LoadSDNode *LD = dyn_cast<LoadSDNode>(N)) {
+ if (LD->getAddressingMode() != ISD::UNINDEXED)
+ return false;
+ VT = LD->getLoadedVT();
+ if (!TLI.isIndexedLoadLegal(ISD::POST_INC, VT) &&
+ !TLI.isIndexedLoadLegal(ISD::POST_DEC, VT))
+ return false;
+ Ptr = LD->getBasePtr();
+ } else if (StoreSDNode *ST = dyn_cast<StoreSDNode>(N)) {
+ if (ST->getAddressingMode() != ISD::UNINDEXED)
+ return false;
+ VT = ST->getStoredVT();
+ if (!TLI.isIndexedStoreLegal(ISD::POST_INC, VT) &&
+ !TLI.isIndexedStoreLegal(ISD::POST_DEC, VT))
+ return false;
+ Ptr = ST->getBasePtr();
+ isLoad = false;
+ } else
+ return false;
+
+ if (Ptr.Val->hasOneUse())
+ return false;
+
+ for (SDNode::use_iterator I = Ptr.Val->use_begin(),
+ E = Ptr.Val->use_end(); I != E; ++I) {
+ SDNode *Op = *I;
+ if (Op == N ||
+ (Op->getOpcode() != ISD::ADD && Op->getOpcode() != ISD::SUB))
+ continue;
+
+ SDOperand BasePtr;
+ SDOperand Offset;
+ ISD::MemIndexedMode AM = ISD::UNINDEXED;
+ if (TLI.getPostIndexedAddressParts(N, Op, BasePtr, Offset, AM, DAG)) {
+ if (Ptr == Offset)
+ std::swap(BasePtr, Offset);
+ if (Ptr != BasePtr)
+ continue;
+ // Don't create a indexed load / store with zero offset.
+ if (isa<ConstantSDNode>(Offset) &&
+ cast<ConstantSDNode>(Offset)->getValue() == 0)
+ continue;
+
+ // Try turning it into a post-indexed load / store except when
+ // 1) All uses are load / store ops that use it as base ptr.
+ // 2) Op must be independent of N, i.e. Op is neither a predecessor
+ // nor a successor of N. Otherwise, if Op is folded that would
+ // create a cycle.
+
+ // Check for #1.
+ bool TryNext = false;
+ for (SDNode::use_iterator II = BasePtr.Val->use_begin(),
+ EE = BasePtr.Val->use_end(); II != EE; ++II) {
+ SDNode *Use = *II;
+ if (Use == Ptr.Val)
+ continue;
+
+ // If all the uses are load / store addresses, then don't do the
+ // transformation.
+ if (Use->getOpcode() == ISD::ADD || Use->getOpcode() == ISD::SUB){
+ bool RealUse = false;
+ for (SDNode::use_iterator III = Use->use_begin(),
+ EEE = Use->use_end(); III != EEE; ++III) {
+ SDNode *UseUse = *III;
+ if (!((UseUse->getOpcode() == ISD::LOAD &&
+ cast<LoadSDNode>(UseUse)->getBasePtr().Val == Use) ||
+ (UseUse->getOpcode() == ISD::STORE) &&
+ cast<StoreSDNode>(UseUse)->getBasePtr().Val == Use))
+ RealUse = true;
+ }
+
+ if (!RealUse) {
+ TryNext = true;
+ break;
+ }
+ }
+ }
+ if (TryNext)
+ continue;
+
+ // Check for #2
+ if (!Op->isPredecessor(N) && !N->isPredecessor(Op)) {
+ SDOperand Result = isLoad
+ ? DAG.getIndexedLoad(SDOperand(N,0), BasePtr, Offset, AM)
+ : DAG.getIndexedStore(SDOperand(N,0), BasePtr, Offset, AM);
+ ++PostIndexedNodes;
+ ++NodesCombined;
+ DOUT << "\nReplacing.5 "; DEBUG(N->dump(&DAG));
+ DOUT << "\nWith: "; DEBUG(Result.Val->dump(&DAG));
+ DOUT << '\n';
+ std::vector<SDNode*> NowDead;
+ if (isLoad) {
+ DAG.ReplaceAllUsesOfValueWith(SDOperand(N, 0), Result.getValue(0),
+ NowDead);
+ DAG.ReplaceAllUsesOfValueWith(SDOperand(N, 1), Result.getValue(2),
+ NowDead);
+ } else {
+ DAG.ReplaceAllUsesOfValueWith(SDOperand(N, 0), Result.getValue(1),
+ NowDead);
+ }
+
+ // Nodes can end up on the worklist more than once. Make sure we do
+ // not process a node that has been replaced.
+ for (unsigned i = 0, e = NowDead.size(); i != e; ++i)
+ removeFromWorkList(NowDead[i]);
+ // Finally, since the node is now dead, remove it from the graph.
+ DAG.DeleteNode(N);
+
+ // Replace the uses of Use with uses of the updated base value.
+ DAG.ReplaceAllUsesOfValueWith(SDOperand(Op, 0),
+ Result.getValue(isLoad ? 1 : 0),
+ NowDead);
+ removeFromWorkList(Op);
+ for (unsigned i = 0, e = NowDead.size(); i != e; ++i)
+ removeFromWorkList(NowDead[i]);
+ DAG.DeleteNode(Op);
+
+ return true;
+ }
+ }
+ }
+ return false;
+}
+
+
+SDOperand DAGCombiner::visitLOAD(SDNode *N) {
+ LoadSDNode *LD = cast<LoadSDNode>(N);
+ SDOperand Chain = LD->getChain();
+ SDOperand Ptr = LD->getBasePtr();
+
+ // If load is not volatile and there are no uses of the loaded value (and
+ // the updated indexed value in case of indexed loads), change uses of the
+ // chain value into uses of the chain input (i.e. delete the dead load).
+ if (!LD->isVolatile()) {
+ if (N->getValueType(1) == MVT::Other) {
+ // Unindexed loads.
+ if (N->hasNUsesOfValue(0, 0))
+ return CombineTo(N, DAG.getNode(ISD::UNDEF, N->getValueType(0)), Chain);
+ } else {
+ // Indexed loads.
+ assert(N->getValueType(2) == MVT::Other && "Malformed indexed loads?");
+ if (N->hasNUsesOfValue(0, 0) && N->hasNUsesOfValue(0, 1)) {
+ SDOperand Undef0 = DAG.getNode(ISD::UNDEF, N->getValueType(0));
+ SDOperand Undef1 = DAG.getNode(ISD::UNDEF, N->getValueType(1));
+ SDOperand To[] = { Undef0, Undef1, Chain };
+ return CombineTo(N, To, 3);
+ }
+ }
+ }
+
+ // If this load is directly stored, replace the load value with the stored
+ // value.
+ // TODO: Handle store large -> read small portion.
+ // TODO: Handle TRUNCSTORE/LOADEXT
+ if (LD->getExtensionType() == ISD::NON_EXTLOAD) {
+ if (ISD::isNON_TRUNCStore(Chain.Val)) {
+ StoreSDNode *PrevST = cast<StoreSDNode>(Chain);
+ if (PrevST->getBasePtr() == Ptr &&
+ PrevST->getValue().getValueType() == N->getValueType(0))
+ return CombineTo(N, Chain.getOperand(1), Chain);
+ }
+ }
+
+ if (CombinerAA) {
+ // Walk up chain skipping non-aliasing memory nodes.
+ SDOperand BetterChain = FindBetterChain(N, Chain);
+
+ // If there is a better chain.
+ if (Chain != BetterChain) {
+ SDOperand ReplLoad;
+
+ // Replace the chain to void dependency.
+ if (LD->getExtensionType() == ISD::NON_EXTLOAD) {
+ ReplLoad = DAG.getLoad(N->getValueType(0), BetterChain, Ptr,
+ LD->getSrcValue(), LD->getSrcValueOffset(),
+ LD->isVolatile(), LD->getAlignment());
+ } else {
+ ReplLoad = DAG.getExtLoad(LD->getExtensionType(),
+ LD->getValueType(0),
+ BetterChain, Ptr, LD->getSrcValue(),
+ LD->getSrcValueOffset(),
+ LD->getLoadedVT(),
+ LD->isVolatile(),
+ LD->getAlignment());
+ }
+
+ // Create token factor to keep old chain connected.
+ SDOperand Token = DAG.getNode(ISD::TokenFactor, MVT::Other,
+ Chain, ReplLoad.getValue(1));
+
+ // Replace uses with load result and token factor. Don't add users
+ // to work list.
+ return CombineTo(N, ReplLoad.getValue(0), Token, false);
+ }
+ }
+
+ // Try transforming N to an indexed load.
+ if (CombineToPreIndexedLoadStore(N) || CombineToPostIndexedLoadStore(N))
+ return SDOperand(N, 0);
+
+ return SDOperand();
+}
+
+SDOperand DAGCombiner::visitSTORE(SDNode *N) {
+ StoreSDNode *ST = cast<StoreSDNode>(N);
+ SDOperand Chain = ST->getChain();
+ SDOperand Value = ST->getValue();
+ SDOperand Ptr = ST->getBasePtr();
+
+ // If this is a store of a bit convert, store the input value if the
+ // resultant store does not need a higher alignment than the original.
+ if (Value.getOpcode() == ISD::BIT_CONVERT && !ST->isTruncatingStore() &&
+ ST->getAddressingMode() == ISD::UNINDEXED) {
+ unsigned Align = ST->getAlignment();
+ MVT::ValueType SVT = Value.getOperand(0).getValueType();
+ unsigned OrigAlign = TLI.getTargetMachine().getTargetData()->
+ getABITypeAlignment(MVT::getTypeForValueType(SVT));
+ if (Align <= OrigAlign && TLI.isOperationLegal(ISD::STORE, SVT))
+ return DAG.getStore(Chain, Value.getOperand(0), Ptr, ST->getSrcValue(),
+ ST->getSrcValueOffset(), ST->isVolatile(), Align);
+ }
+
+ // Turn 'store float 1.0, Ptr' -> 'store int 0x12345678, Ptr'
+ if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(Value)) {
+ if (Value.getOpcode() != ISD::TargetConstantFP) {
+ SDOperand Tmp;
+ switch (CFP->getValueType(0)) {
+ default: assert(0 && "Unknown FP type");
+ case MVT::f32:
+ if (!AfterLegalize || TLI.isTypeLegal(MVT::i32)) {
+ Tmp = DAG.getConstant(FloatToBits(CFP->getValue()), MVT::i32);
+ return DAG.getStore(Chain, Tmp, Ptr, ST->getSrcValue(),
+ ST->getSrcValueOffset(), ST->isVolatile(),
+ ST->getAlignment());
+ }
+ break;
+ case MVT::f64:
+ if (!AfterLegalize || TLI.isTypeLegal(MVT::i64)) {
+ Tmp = DAG.getConstant(DoubleToBits(CFP->getValue()), MVT::i64);
+ return DAG.getStore(Chain, Tmp, Ptr, ST->getSrcValue(),
+ ST->getSrcValueOffset(), ST->isVolatile(),
+ ST->getAlignment());
+ } else if (TLI.isTypeLegal(MVT::i32)) {
+ // Many FP stores are not make apparent until after legalize, e.g. for
+ // argument passing. Since this is so common, custom legalize the
+ // 64-bit integer store into two 32-bit stores.
+ uint64_t Val = DoubleToBits(CFP->getValue());
+ SDOperand Lo = DAG.getConstant(Val & 0xFFFFFFFF, MVT::i32);
+ SDOperand Hi = DAG.getConstant(Val >> 32, MVT::i32);
+ if (!TLI.isLittleEndian()) std::swap(Lo, Hi);
+
+ int SVOffset = ST->getSrcValueOffset();
+ unsigned Alignment = ST->getAlignment();
+ bool isVolatile = ST->isVolatile();
+
+ SDOperand St0 = DAG.getStore(Chain, Lo, Ptr, ST->getSrcValue(),
+ ST->getSrcValueOffset(),
+ isVolatile, ST->getAlignment());
+ Ptr = DAG.getNode(ISD::ADD, Ptr.getValueType(), Ptr,
+ DAG.getConstant(4, Ptr.getValueType()));
+ SVOffset += 4;
+ if (Alignment > 4)
+ Alignment = 4;
+ SDOperand St1 = DAG.getStore(Chain, Hi, Ptr, ST->getSrcValue(),
+ SVOffset, isVolatile, Alignment);
+ return DAG.getNode(ISD::TokenFactor, MVT::Other, St0, St1);
+ }
+ break;
+ }
+ }
+ }
+
+ if (CombinerAA) {
+ // Walk up chain skipping non-aliasing memory nodes.
+ SDOperand BetterChain = FindBetterChain(N, Chain);
+
+ // If there is a better chain.
+ if (Chain != BetterChain) {
+ // Replace the chain to avoid dependency.
+ SDOperand ReplStore;
+ if (ST->isTruncatingStore()) {
+ ReplStore = DAG.getTruncStore(BetterChain, Value, Ptr,
+ ST->getSrcValue(), ST->getSrcValueOffset(), ST->getStoredVT(),
+ ST->isVolatile(), ST->getAlignment());
+ } else {
+ ReplStore = DAG.getStore(BetterChain, Value, Ptr,
+ ST->getSrcValue(), ST->getSrcValueOffset(),
+ ST->isVolatile(), ST->getAlignment());
+ }
+
+ // Create token to keep both nodes around.
+ SDOperand Token =
+ DAG.getNode(ISD::TokenFactor, MVT::Other, Chain, ReplStore);
+
+ // Don't add users to work list.
+ return CombineTo(N, Token, false);
+ }
+ }
+
+ // Try transforming N to an indexed store.
+ if (CombineToPreIndexedLoadStore(N) || CombineToPostIndexedLoadStore(N))
+ return SDOperand(N, 0);
+
+ return SDOperand();
+}
+
+SDOperand DAGCombiner::visitINSERT_VECTOR_ELT(SDNode *N) {
+ SDOperand InVec = N->getOperand(0);
+ SDOperand InVal = N->getOperand(1);
+ SDOperand EltNo = N->getOperand(2);
+
+ // If the invec is a BUILD_VECTOR and if EltNo is a constant, build a new
+ // vector with the inserted element.
+ if (InVec.getOpcode() == ISD::BUILD_VECTOR && isa<ConstantSDNode>(EltNo)) {
+ unsigned Elt = cast<ConstantSDNode>(EltNo)->getValue();
+ SmallVector<SDOperand, 8> Ops(InVec.Val->op_begin(), InVec.Val->op_end());
+ if (Elt < Ops.size())
+ Ops[Elt] = InVal;
+ return DAG.getNode(ISD::BUILD_VECTOR, InVec.getValueType(),
+ &Ops[0], Ops.size());
+ }
+
+ return SDOperand();
+}
+
+SDOperand DAGCombiner::visitBUILD_VECTOR(SDNode *N) {
+ unsigned NumInScalars = N->getNumOperands();
+ MVT::ValueType VT = N->getValueType(0);
+ unsigned NumElts = MVT::getVectorNumElements(VT);
+ MVT::ValueType EltType = MVT::getVectorElementType(VT);
+
+ // Check to see if this is a BUILD_VECTOR of a bunch of EXTRACT_VECTOR_ELT
+ // operations. If so, and if the EXTRACT_VECTOR_ELT vector inputs come from
+ // at most two distinct vectors, turn this into a shuffle node.
+ SDOperand VecIn1, VecIn2;
+ for (unsigned i = 0; i != NumInScalars; ++i) {
+ // Ignore undef inputs.
+ if (N->getOperand(i).getOpcode() == ISD::UNDEF) continue;
+
+ // If this input is something other than a EXTRACT_VECTOR_ELT with a
+ // constant index, bail out.
+ if (N->getOperand(i).getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
+ !isa<ConstantSDNode>(N->getOperand(i).getOperand(1))) {
+ VecIn1 = VecIn2 = SDOperand(0, 0);
+ break;
+ }
+
+ // If the input vector type disagrees with the result of the build_vector,
+ // we can't make a shuffle.
+ SDOperand ExtractedFromVec = N->getOperand(i).getOperand(0);
+ if (ExtractedFromVec.getValueType() != VT) {
+ VecIn1 = VecIn2 = SDOperand(0, 0);
+ break;
+ }
+
+ // Otherwise, remember this. We allow up to two distinct input vectors.
+ if (ExtractedFromVec == VecIn1 || ExtractedFromVec == VecIn2)
+ continue;
+
+ if (VecIn1.Val == 0) {
+ VecIn1 = ExtractedFromVec;
+ } else if (VecIn2.Val == 0) {
+ VecIn2 = ExtractedFromVec;
+ } else {
+ // Too many inputs.
+ VecIn1 = VecIn2 = SDOperand(0, 0);
+ break;
+ }
+ }
+
+ // If everything is good, we can make a shuffle operation.
+ if (VecIn1.Val) {
+ SmallVector<SDOperand, 8> BuildVecIndices;
+ for (unsigned i = 0; i != NumInScalars; ++i) {
+ if (N->getOperand(i).getOpcode() == ISD::UNDEF) {
+ BuildVecIndices.push_back(DAG.getNode(ISD::UNDEF, TLI.getPointerTy()));
+ continue;
+ }
+
+ SDOperand Extract = N->getOperand(i);
+
+ // If extracting from the first vector, just use the index directly.
+ if (Extract.getOperand(0) == VecIn1) {
+ BuildVecIndices.push_back(Extract.getOperand(1));
+ continue;
+ }
+
+ // Otherwise, use InIdx + VecSize
+ unsigned Idx = cast<ConstantSDNode>(Extract.getOperand(1))->getValue();
+ BuildVecIndices.push_back(DAG.getConstant(Idx+NumInScalars,
+ TLI.getPointerTy()));
+ }
+
+ // Add count and size info.
+ MVT::ValueType BuildVecVT =
+ MVT::getVectorType(TLI.getPointerTy(), NumElts);
+
+ // Return the new VECTOR_SHUFFLE node.
+ SDOperand Ops[5];
+ Ops[0] = VecIn1;
+ if (VecIn2.Val) {
+ Ops[1] = VecIn2;
+ } else {
+ // Use an undef build_vector as input for the second operand.
+ std::vector<SDOperand> UnOps(NumInScalars,
+ DAG.getNode(ISD::UNDEF,
+ EltType));
+ Ops[1] = DAG.getNode(ISD::BUILD_VECTOR, VT,
+ &UnOps[0], UnOps.size());
+ AddToWorkList(Ops[1].Val);
+ }
+ Ops[2] = DAG.getNode(ISD::BUILD_VECTOR, BuildVecVT,
+ &BuildVecIndices[0], BuildVecIndices.size());
+ return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, Ops, 3);
+ }
+
+ return SDOperand();
+}
+
+SDOperand DAGCombiner::visitCONCAT_VECTORS(SDNode *N) {
+ // TODO: Check to see if this is a CONCAT_VECTORS of a bunch of
+ // EXTRACT_SUBVECTOR operations. If so, and if the EXTRACT_SUBVECTOR vector
+ // inputs come from at most two distinct vectors, turn this into a shuffle
+ // node.
+
+ // If we only have one input vector, we don't need to do any concatenation.
+ if (N->getNumOperands() == 1) {
+ return N->getOperand(0);
+ }
+
+ return SDOperand();
+}
+
+SDOperand DAGCombiner::visitVECTOR_SHUFFLE(SDNode *N) {
+ SDOperand ShufMask = N->getOperand(2);
+ unsigned NumElts = ShufMask.getNumOperands();
+
+ // If the shuffle mask is an identity operation on the LHS, return the LHS.
+ bool isIdentity = true;
+ for (unsigned i = 0; i != NumElts; ++i) {
+ if (ShufMask.getOperand(i).getOpcode() != ISD::UNDEF &&
+ cast<ConstantSDNode>(ShufMask.getOperand(i))->getValue() != i) {
+ isIdentity = false;
+ break;
+ }
+ }
+ if (isIdentity) return N->getOperand(0);
+
+ // If the shuffle mask is an identity operation on the RHS, return the RHS.
+ isIdentity = true;
+ for (unsigned i = 0; i != NumElts; ++i) {
+ if (ShufMask.getOperand(i).getOpcode() != ISD::UNDEF &&
+ cast<ConstantSDNode>(ShufMask.getOperand(i))->getValue() != i+NumElts) {
+ isIdentity = false;
+ break;
+ }
+ }
+ if (isIdentity) return N->getOperand(1);
+
+ // Check if the shuffle is a unary shuffle, i.e. one of the vectors is not
+ // needed at all.
+ bool isUnary = true;
+ bool isSplat = true;
+ int VecNum = -1;
+ unsigned BaseIdx = 0;
+ for (unsigned i = 0; i != NumElts; ++i)
+ if (ShufMask.getOperand(i).getOpcode() != ISD::UNDEF) {
+ unsigned Idx = cast<ConstantSDNode>(ShufMask.getOperand(i))->getValue();
+ int V = (Idx < NumElts) ? 0 : 1;
+ if (VecNum == -1) {
+ VecNum = V;
+ BaseIdx = Idx;
+ } else {
+ if (BaseIdx != Idx)
+ isSplat = false;
+ if (VecNum != V) {
+ isUnary = false;
+ break;
+ }
+ }
+ }
+
+ SDOperand N0 = N->getOperand(0);
+ SDOperand N1 = N->getOperand(1);
+ // Normalize unary shuffle so the RHS is undef.
+ if (isUnary && VecNum == 1)
+ std::swap(N0, N1);
+
+ // If it is a splat, check if the argument vector is a build_vector with
+ // all scalar elements the same.
+ if (isSplat) {
+ SDNode *V = N0.Val;
+
+ // If this is a bit convert that changes the element type of the vector but
+ // not the number of vector elements, look through it. Be careful not to
+ // look though conversions that change things like v4f32 to v2f64.
+ if (V->getOpcode() == ISD::BIT_CONVERT) {
+ SDOperand ConvInput = V->getOperand(0);
+ if (MVT::getVectorNumElements(ConvInput.getValueType()) == NumElts)
+ V = ConvInput.Val;
+ }
+
+ if (V->getOpcode() == ISD::BUILD_VECTOR) {
+ unsigned NumElems = V->getNumOperands();
+ if (NumElems > BaseIdx) {
+ SDOperand Base;
+ bool AllSame = true;
+ for (unsigned i = 0; i != NumElems; ++i) {
+ if (V->getOperand(i).getOpcode() != ISD::UNDEF) {
+ Base = V->getOperand(i);
+ break;
+ }
+ }
+ // Splat of <u, u, u, u>, return <u, u, u, u>
+ if (!Base.Val)
+ return N0;
+ for (unsigned i = 0; i != NumElems; ++i) {
+ if (V->getOperand(i).getOpcode() != ISD::UNDEF &&
+ V->getOperand(i) != Base) {
+ AllSame = false;
+ break;
+ }
+ }
+ // Splat of <x, x, x, x>, return <x, x, x, x>
+ if (AllSame)
+ return N0;
+ }
+ }
+ }
+
+ // If it is a unary or the LHS and the RHS are the same node, turn the RHS
+ // into an undef.
+ if (isUnary || N0 == N1) {
+ // Check the SHUFFLE mask, mapping any inputs from the 2nd operand into the
+ // first operand.
+ SmallVector<SDOperand, 8> MappedOps;
+ for (unsigned i = 0; i != NumElts; ++i) {
+ if (ShufMask.getOperand(i).getOpcode() == ISD::UNDEF ||
+ cast<ConstantSDNode>(ShufMask.getOperand(i))->getValue() < NumElts) {
+ MappedOps.push_back(ShufMask.getOperand(i));
+ } else {
+ unsigned NewIdx =
+ cast<ConstantSDNode>(ShufMask.getOperand(i))->getValue() - NumElts;
+ MappedOps.push_back(DAG.getConstant(NewIdx, MVT::i32));
+ }
+ }
+ ShufMask = DAG.getNode(ISD::BUILD_VECTOR, ShufMask.getValueType(),
+ &MappedOps[0], MappedOps.size());
+ AddToWorkList(ShufMask.Val);
+ return DAG.getNode(ISD::VECTOR_SHUFFLE, N->getValueType(0),
+ N0,
+ DAG.getNode(ISD::UNDEF, N->getValueType(0)),
+ ShufMask);
+ }
+
+ return SDOperand();
+}
+
+/// XformToShuffleWithZero - Returns a vector_shuffle if it able to transform
+/// an AND to a vector_shuffle with the destination vector and a zero vector.
+/// e.g. AND V, <0xffffffff, 0, 0xffffffff, 0>. ==>
+/// vector_shuffle V, Zero, <0, 4, 2, 4>
+SDOperand DAGCombiner::XformToShuffleWithZero(SDNode *N) {
+ SDOperand LHS = N->getOperand(0);
+ SDOperand RHS = N->getOperand(1);
+ if (N->getOpcode() == ISD::AND) {
+ if (RHS.getOpcode() == ISD::BIT_CONVERT)
+ RHS = RHS.getOperand(0);
+ if (RHS.getOpcode() == ISD::BUILD_VECTOR) {
+ std::vector<SDOperand> IdxOps;
+ unsigned NumOps = RHS.getNumOperands();
+ unsigned NumElts = NumOps;
+ MVT::ValueType EVT = MVT::getVectorElementType(RHS.getValueType());
+ for (unsigned i = 0; i != NumElts; ++i) {
+ SDOperand Elt = RHS.getOperand(i);
+ if (!isa<ConstantSDNode>(Elt))
+ return SDOperand();
+ else if (cast<ConstantSDNode>(Elt)->isAllOnesValue())
+ IdxOps.push_back(DAG.getConstant(i, EVT));
+ else if (cast<ConstantSDNode>(Elt)->isNullValue())
+ IdxOps.push_back(DAG.getConstant(NumElts, EVT));
+ else
+ return SDOperand();
+ }
+
+ // Let's see if the target supports this vector_shuffle.
+ if (!TLI.isVectorClearMaskLegal(IdxOps, EVT, DAG))
+ return SDOperand();
+
+ // Return the new VECTOR_SHUFFLE node.
+ MVT::ValueType VT = MVT::getVectorType(EVT, NumElts);
+ std::vector<SDOperand> Ops;
+ LHS = DAG.getNode(ISD::BIT_CONVERT, VT, LHS);
+ Ops.push_back(LHS);
+ AddToWorkList(LHS.Val);
+ std::vector<SDOperand> ZeroOps(NumElts, DAG.getConstant(0, EVT));
+ Ops.push_back(DAG.getNode(ISD::BUILD_VECTOR, VT,
+ &ZeroOps[0], ZeroOps.size()));
+ Ops.push_back(DAG.getNode(ISD::BUILD_VECTOR, VT,
+ &IdxOps[0], IdxOps.size()));
+ SDOperand Result = DAG.getNode(ISD::VECTOR_SHUFFLE, VT,
+ &Ops[0], Ops.size());
+ if (VT != LHS.getValueType()) {
+ Result = DAG.getNode(ISD::BIT_CONVERT, LHS.getValueType(), Result);
+ }
+ return Result;
+ }
+ }
+ return SDOperand();
+}
+
+/// SimplifyVBinOp - Visit a binary vector operation, like ADD.
+SDOperand DAGCombiner::SimplifyVBinOp(SDNode *N) {
+ // After legalize, the target may be depending on adds and other
+ // binary ops to provide legal ways to construct constants or other
+ // things. Simplifying them may result in a loss of legality.
+ if (AfterLegalize) return SDOperand();
+
+ MVT::ValueType VT = N->getValueType(0);
+ assert(MVT::isVector(VT) && "SimplifyVBinOp only works on vectors!");
+
+ MVT::ValueType EltType = MVT::getVectorElementType(VT);
+ SDOperand LHS = N->getOperand(0);
+ SDOperand RHS = N->getOperand(1);
+ SDOperand Shuffle = XformToShuffleWithZero(N);
+ if (Shuffle.Val) return Shuffle;
+
+ // If the LHS and RHS are BUILD_VECTOR nodes, see if we can constant fold
+ // this operation.
+ if (LHS.getOpcode() == ISD::BUILD_VECTOR &&
+ RHS.getOpcode() == ISD::BUILD_VECTOR) {
+ SmallVector<SDOperand, 8> Ops;
+ for (unsigned i = 0, e = LHS.getNumOperands(); i != e; ++i) {
+ SDOperand LHSOp = LHS.getOperand(i);
+ SDOperand RHSOp = RHS.getOperand(i);
+ // If these two elements can't be folded, bail out.
+ if ((LHSOp.getOpcode() != ISD::UNDEF &&
+ LHSOp.getOpcode() != ISD::Constant &&
+ LHSOp.getOpcode() != ISD::ConstantFP) ||
+ (RHSOp.getOpcode() != ISD::UNDEF &&
+ RHSOp.getOpcode() != ISD::Constant &&
+ RHSOp.getOpcode() != ISD::ConstantFP))
+ break;
+ // Can't fold divide by zero.
+ if (N->getOpcode() == ISD::SDIV || N->getOpcode() == ISD::UDIV ||
+ N->getOpcode() == ISD::FDIV) {
+ if ((RHSOp.getOpcode() == ISD::Constant &&
+ cast<ConstantSDNode>(RHSOp.Val)->isNullValue()) ||
+ (RHSOp.getOpcode() == ISD::ConstantFP &&
+ !cast<ConstantFPSDNode>(RHSOp.Val)->getValue()))
+ break;
+ }
+ Ops.push_back(DAG.getNode(N->getOpcode(), EltType, LHSOp, RHSOp));
+ AddToWorkList(Ops.back().Val);
+ assert((Ops.back().getOpcode() == ISD::UNDEF ||
+ Ops.back().getOpcode() == ISD::Constant ||
+ Ops.back().getOpcode() == ISD::ConstantFP) &&
+ "Scalar binop didn't fold!");
+ }
+
+ if (Ops.size() == LHS.getNumOperands()) {
+ MVT::ValueType VT = LHS.getValueType();
+ return DAG.getNode(ISD::BUILD_VECTOR, VT, &Ops[0], Ops.size());
+ }
+ }
+
+ return SDOperand();
+}
+
+SDOperand DAGCombiner::SimplifySelect(SDOperand N0, SDOperand N1, SDOperand N2){
+ assert(N0.getOpcode() ==ISD::SETCC && "First argument must be a SetCC node!");
+
+ SDOperand SCC = SimplifySelectCC(N0.getOperand(0), N0.getOperand(1), N1, N2,
+ cast<CondCodeSDNode>(N0.getOperand(2))->get());
+ // If we got a simplified select_cc node back from SimplifySelectCC, then
+ // break it down into a new SETCC node, and a new SELECT node, and then return
+ // the SELECT node, since we were called with a SELECT node.
+ if (SCC.Val) {
+ // Check to see if we got a select_cc back (to turn into setcc/select).
+ // Otherwise, just return whatever node we got back, like fabs.
+ if (SCC.getOpcode() == ISD::SELECT_CC) {
+ SDOperand SETCC = DAG.getNode(ISD::SETCC, N0.getValueType(),
+ SCC.getOperand(0), SCC.getOperand(1),
+ SCC.getOperand(4));
+ AddToWorkList(SETCC.Val);
+ return DAG.getNode(ISD::SELECT, SCC.getValueType(), SCC.getOperand(2),
+ SCC.getOperand(3), SETCC);
+ }
+ return SCC;
+ }
+ return SDOperand();
+}
+
+/// SimplifySelectOps - Given a SELECT or a SELECT_CC node, where LHS and RHS
+/// are the two values being selected between, see if we can simplify the
+/// select. Callers of this should assume that TheSelect is deleted if this
+/// returns true. As such, they should return the appropriate thing (e.g. the
+/// node) back to the top-level of the DAG combiner loop to avoid it being
+/// looked at.
+///
+bool DAGCombiner::SimplifySelectOps(SDNode *TheSelect, SDOperand LHS,
+ SDOperand RHS) {
+
+ // If this is a select from two identical things, try to pull the operation
+ // through the select.
+ if (LHS.getOpcode() == RHS.getOpcode() && LHS.hasOneUse() && RHS.hasOneUse()){
+ // If this is a load and the token chain is identical, replace the select
+ // of two loads with a load through a select of the address to load from.
+ // This triggers in things like "select bool X, 10.0, 123.0" after the FP
+ // constants have been dropped into the constant pool.
+ if (LHS.getOpcode() == ISD::LOAD &&
+ // Token chains must be identical.
+ LHS.getOperand(0) == RHS.getOperand(0)) {
+ LoadSDNode *LLD = cast<LoadSDNode>(LHS);
+ LoadSDNode *RLD = cast<LoadSDNode>(RHS);
+
+ // If this is an EXTLOAD, the VT's must match.
+ if (LLD->getLoadedVT() == RLD->getLoadedVT()) {
+ // FIXME: this conflates two src values, discarding one. This is not
+ // the right thing to do, but nothing uses srcvalues now. When they do,
+ // turn SrcValue into a list of locations.
+ SDOperand Addr;
+ if (TheSelect->getOpcode() == ISD::SELECT) {
+ // Check that the condition doesn't reach either load. If so, folding
+ // this will induce a cycle into the DAG.
+ if (!LLD->isPredecessor(TheSelect->getOperand(0).Val) &&
+ !RLD->isPredecessor(TheSelect->getOperand(0).Val)) {
+ Addr = DAG.getNode(ISD::SELECT, LLD->getBasePtr().getValueType(),
+ TheSelect->getOperand(0), LLD->getBasePtr(),
+ RLD->getBasePtr());
+ }
+ } else {
+ // Check that the condition doesn't reach either load. If so, folding
+ // this will induce a cycle into the DAG.
+ if (!LLD->isPredecessor(TheSelect->getOperand(0).Val) &&
+ !RLD->isPredecessor(TheSelect->getOperand(0).Val) &&
+ !LLD->isPredecessor(TheSelect->getOperand(1).Val) &&
+ !RLD->isPredecessor(TheSelect->getOperand(1).Val)) {
+ Addr = DAG.getNode(ISD::SELECT_CC, LLD->getBasePtr().getValueType(),
+ TheSelect->getOperand(0),
+ TheSelect->getOperand(1),
+ LLD->getBasePtr(), RLD->getBasePtr(),
+ TheSelect->getOperand(4));
+ }
+ }
+
+ if (Addr.Val) {
+ SDOperand Load;
+ if (LLD->getExtensionType() == ISD::NON_EXTLOAD)
+ Load = DAG.getLoad(TheSelect->getValueType(0), LLD->getChain(),
+ Addr,LLD->getSrcValue(),
+ LLD->getSrcValueOffset(),
+ LLD->isVolatile(),
+ LLD->getAlignment());
+ else {
+ Load = DAG.getExtLoad(LLD->getExtensionType(),
+ TheSelect->getValueType(0),
+ LLD->getChain(), Addr, LLD->getSrcValue(),
+ LLD->getSrcValueOffset(),
+ LLD->getLoadedVT(),
+ LLD->isVolatile(),
+ LLD->getAlignment());
+ }
+ // Users of the select now use the result of the load.
+ CombineTo(TheSelect, Load);
+
+ // Users of the old loads now use the new load's chain. We know the
+ // old-load value is dead now.
+ CombineTo(LHS.Val, Load.getValue(0), Load.getValue(1));
+ CombineTo(RHS.Val, Load.getValue(0), Load.getValue(1));
+ return true;
+ }
+ }
+ }
+ }
+
+ return false;
+}
+
+SDOperand DAGCombiner::SimplifySelectCC(SDOperand N0, SDOperand N1,
+ SDOperand N2, SDOperand N3,
+ ISD::CondCode CC, bool NotExtCompare) {
+
+ MVT::ValueType VT = N2.getValueType();
+ ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.Val);
+ ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2.Val);
+ ConstantSDNode *N3C = dyn_cast<ConstantSDNode>(N3.Val);
+
+ // Determine if the condition we're dealing with is constant
+ SDOperand SCC = SimplifySetCC(TLI.getSetCCResultTy(), N0, N1, CC, false);
+ if (SCC.Val) AddToWorkList(SCC.Val);
+ ConstantSDNode *SCCC = dyn_cast_or_null<ConstantSDNode>(SCC.Val);
+
+ // fold select_cc true, x, y -> x
+ if (SCCC && SCCC->getValue())
+ return N2;
+ // fold select_cc false, x, y -> y
+ if (SCCC && SCCC->getValue() == 0)
+ return N3;
+
+ // Check to see if we can simplify the select into an fabs node
+ if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(N1)) {
+ // Allow either -0.0 or 0.0
+ if (CFP->getValue() == 0.0) {
+ // select (setg[te] X, +/-0.0), X, fneg(X) -> fabs
+ if ((CC == ISD::SETGE || CC == ISD::SETGT) &&
+ N0 == N2 && N3.getOpcode() == ISD::FNEG &&
+ N2 == N3.getOperand(0))
+ return DAG.getNode(ISD::FABS, VT, N0);
+
+ // select (setl[te] X, +/-0.0), fneg(X), X -> fabs
+ if ((CC == ISD::SETLT || CC == ISD::SETLE) &&
+ N0 == N3 && N2.getOpcode() == ISD::FNEG &&
+ N2.getOperand(0) == N3)
+ return DAG.getNode(ISD::FABS, VT, N3);
+ }
+ }
+
+ // Check to see if we can perform the "gzip trick", transforming
+ // select_cc setlt X, 0, A, 0 -> and (sra X, size(X)-1), A
+ if (N1C && N3C && N3C->isNullValue() && CC == ISD::SETLT &&
+ MVT::isInteger(N0.getValueType()) &&
+ MVT::isInteger(N2.getValueType()) &&
+ (N1C->isNullValue() || // (a < 0) ? b : 0
+ (N1C->getValue() == 1 && N0 == N2))) { // (a < 1) ? a : 0
+ MVT::ValueType XType = N0.getValueType();
+ MVT::ValueType AType = N2.getValueType();
+ if (XType >= AType) {
+ // and (sra X, size(X)-1, A) -> "and (srl X, C2), A" iff A is a
+ // single-bit constant.
+ if (N2C && ((N2C->getValue() & (N2C->getValue()-1)) == 0)) {
+ unsigned ShCtV = Log2_64(N2C->getValue());
+ ShCtV = MVT::getSizeInBits(XType)-ShCtV-1;
+ SDOperand ShCt = DAG.getConstant(ShCtV, TLI.getShiftAmountTy());
+ SDOperand Shift = DAG.getNode(ISD::SRL, XType, N0, ShCt);
+ AddToWorkList(Shift.Val);
+ if (XType > AType) {
+ Shift = DAG.getNode(ISD::TRUNCATE, AType, Shift);
+ AddToWorkList(Shift.Val);
+ }
+ return DAG.getNode(ISD::AND, AType, Shift, N2);
+ }
+ SDOperand Shift = DAG.getNode(ISD::SRA, XType, N0,
+ DAG.getConstant(MVT::getSizeInBits(XType)-1,
+ TLI.getShiftAmountTy()));
+ AddToWorkList(Shift.Val);
+ if (XType > AType) {
+ Shift = DAG.getNode(ISD::TRUNCATE, AType, Shift);
+ AddToWorkList(Shift.Val);
+ }
+ return DAG.getNode(ISD::AND, AType, Shift, N2);
+ }
+ }
+
+ // fold select C, 16, 0 -> shl C, 4
+ if (N2C && N3C && N3C->isNullValue() && isPowerOf2_64(N2C->getValue()) &&
+ TLI.getSetCCResultContents() == TargetLowering::ZeroOrOneSetCCResult) {
+
+ // If the caller doesn't want us to simplify this into a zext of a compare,
+ // don't do it.
+ if (NotExtCompare && N2C->getValue() == 1)
+ return SDOperand();
+
+ // Get a SetCC of the condition
+ // FIXME: Should probably make sure that setcc is legal if we ever have a
+ // target where it isn't.
+ SDOperand Temp, SCC;
+ // cast from setcc result type to select result type
+ if (AfterLegalize) {
+ SCC = DAG.getSetCC(TLI.getSetCCResultTy(), N0, N1, CC);
+ if (N2.getValueType() < SCC.getValueType())
+ Temp = DAG.getZeroExtendInReg(SCC, N2.getValueType());
+ else
+ Temp = DAG.getNode(ISD::ZERO_EXTEND, N2.getValueType(), SCC);
+ } else {
+ SCC = DAG.getSetCC(MVT::i1, N0, N1, CC);
+ Temp = DAG.getNode(ISD::ZERO_EXTEND, N2.getValueType(), SCC);
+ }
+ AddToWorkList(SCC.Val);
+ AddToWorkList(Temp.Val);
+
+ if (N2C->getValue() == 1)
+ return Temp;
+ // shl setcc result by log2 n2c
+ return DAG.getNode(ISD::SHL, N2.getValueType(), Temp,
+ DAG.getConstant(Log2_64(N2C->getValue()),
+ TLI.getShiftAmountTy()));
+ }
+
+ // Check to see if this is the equivalent of setcc
+ // FIXME: Turn all of these into setcc if setcc if setcc is legal
+ // otherwise, go ahead with the folds.
+ if (0 && N3C && N3C->isNullValue() && N2C && (N2C->getValue() == 1ULL)) {
+ MVT::ValueType XType = N0.getValueType();
+ if (TLI.isOperationLegal(ISD::SETCC, TLI.getSetCCResultTy())) {
+ SDOperand Res = DAG.getSetCC(TLI.getSetCCResultTy(), N0, N1, CC);
+ if (Res.getValueType() != VT)
+ Res = DAG.getNode(ISD::ZERO_EXTEND, VT, Res);
+ return Res;
+ }
+
+ // seteq X, 0 -> srl (ctlz X, log2(size(X)))
+ if (N1C && N1C->isNullValue() && CC == ISD::SETEQ &&
+ TLI.isOperationLegal(ISD::CTLZ, XType)) {
+ SDOperand Ctlz = DAG.getNode(ISD::CTLZ, XType, N0);
+ return DAG.getNode(ISD::SRL, XType, Ctlz,
+ DAG.getConstant(Log2_32(MVT::getSizeInBits(XType)),
+ TLI.getShiftAmountTy()));
+ }
+ // setgt X, 0 -> srl (and (-X, ~X), size(X)-1)
+ if (N1C && N1C->isNullValue() && CC == ISD::SETGT) {
+ SDOperand NegN0 = DAG.getNode(ISD::SUB, XType, DAG.getConstant(0, XType),
+ N0);
+ SDOperand NotN0 = DAG.getNode(ISD::XOR, XType, N0,
+ DAG.getConstant(~0ULL, XType));
+ return DAG.getNode(ISD::SRL, XType,
+ DAG.getNode(ISD::AND, XType, NegN0, NotN0),
+ DAG.getConstant(MVT::getSizeInBits(XType)-1,
+ TLI.getShiftAmountTy()));
+ }
+ // setgt X, -1 -> xor (srl (X, size(X)-1), 1)
+ if (N1C && N1C->isAllOnesValue() && CC == ISD::SETGT) {
+ SDOperand Sign = DAG.getNode(ISD::SRL, XType, N0,
+ DAG.getConstant(MVT::getSizeInBits(XType)-1,
+ TLI.getShiftAmountTy()));
+ return DAG.getNode(ISD::XOR, XType, Sign, DAG.getConstant(1, XType));
+ }
+ }
+
+ // Check to see if this is an integer abs. select_cc setl[te] X, 0, -X, X ->
+ // Y = sra (X, size(X)-1); xor (add (X, Y), Y)
+ if (N1C && N1C->isNullValue() && (CC == ISD::SETLT || CC == ISD::SETLE) &&
+ N0 == N3 && N2.getOpcode() == ISD::SUB && N0 == N2.getOperand(1) &&
+ N2.getOperand(0) == N1 && MVT::isInteger(N0.getValueType())) {
+ MVT::ValueType XType = N0.getValueType();
+ SDOperand Shift = DAG.getNode(ISD::SRA, XType, N0,
+ DAG.getConstant(MVT::getSizeInBits(XType)-1,
+ TLI.getShiftAmountTy()));
+ SDOperand Add = DAG.getNode(ISD::ADD, XType, N0, Shift);
+ AddToWorkList(Shift.Val);
+ AddToWorkList(Add.Val);
+ return DAG.getNode(ISD::XOR, XType, Add, Shift);
+ }
+ // Check to see if this is an integer abs. select_cc setgt X, -1, X, -X ->
+ // Y = sra (X, size(X)-1); xor (add (X, Y), Y)
+ if (N1C && N1C->isAllOnesValue() && CC == ISD::SETGT &&
+ N0 == N2 && N3.getOpcode() == ISD::SUB && N0 == N3.getOperand(1)) {
+ if (ConstantSDNode *SubC = dyn_cast<ConstantSDNode>(N3.getOperand(0))) {
+ MVT::ValueType XType = N0.getValueType();
+ if (SubC->isNullValue() && MVT::isInteger(XType)) {
+ SDOperand Shift = DAG.getNode(ISD::SRA, XType, N0,
+ DAG.getConstant(MVT::getSizeInBits(XType)-1,
+ TLI.getShiftAmountTy()));
+ SDOperand Add = DAG.getNode(ISD::ADD, XType, N0, Shift);
+ AddToWorkList(Shift.Val);
+ AddToWorkList(Add.Val);
+ return DAG.getNode(ISD::XOR, XType, Add, Shift);
+ }
+ }
+ }
+
+ return SDOperand();
+}
+
+/// SimplifySetCC - This is a stub for TargetLowering::SimplifySetCC.
+SDOperand DAGCombiner::SimplifySetCC(MVT::ValueType VT, SDOperand N0,
+ SDOperand N1, ISD::CondCode Cond,
+ bool foldBooleans) {
+ TargetLowering::DAGCombinerInfo
+ DagCombineInfo(DAG, !AfterLegalize, false, this);
+ return TLI.SimplifySetCC(VT, N0, N1, Cond, foldBooleans, DagCombineInfo);
+}
+
+/// BuildSDIVSequence - Given an ISD::SDIV node expressing a divide by constant,
+/// return a DAG expression to select that will generate the same value by
+/// multiplying by a magic number. See:
+/// <http://the.wall.riscom.net/books/proc/ppc/cwg/code2.html>
+SDOperand DAGCombiner::BuildSDIV(SDNode *N) {
+ std::vector<SDNode*> Built;
+ SDOperand S = TLI.BuildSDIV(N, DAG, &Built);
+
+ for (std::vector<SDNode*>::iterator ii = Built.begin(), ee = Built.end();
+ ii != ee; ++ii)
+ AddToWorkList(*ii);
+ return S;
+}
+
+/// BuildUDIVSequence - Given an ISD::UDIV node expressing a divide by constant,
+/// return a DAG expression to select that will generate the same value by
+/// multiplying by a magic number. See:
+/// <http://the.wall.riscom.net/books/proc/ppc/cwg/code2.html>
+SDOperand DAGCombiner::BuildUDIV(SDNode *N) {
+ std::vector<SDNode*> Built;
+ SDOperand S = TLI.BuildUDIV(N, DAG, &Built);
+
+ for (std::vector<SDNode*>::iterator ii = Built.begin(), ee = Built.end();
+ ii != ee; ++ii)
+ AddToWorkList(*ii);
+ return S;
+}
+
+/// FindBaseOffset - Return true if base is known not to alias with anything
+/// but itself. Provides base object and offset as results.
+static bool FindBaseOffset(SDOperand Ptr, SDOperand &Base, int64_t &Offset) {
+ // Assume it is a primitive operation.
+ Base = Ptr; Offset = 0;
+
+ // If it's an adding a simple constant then integrate the offset.
+ if (Base.getOpcode() == ISD::ADD) {
+ if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Base.getOperand(1))) {
+ Base = Base.getOperand(0);
+ Offset += C->getValue();
+ }
+ }
+
+ // If it's any of the following then it can't alias with anything but itself.
+ return isa<FrameIndexSDNode>(Base) ||
+ isa<ConstantPoolSDNode>(Base) ||
+ isa<GlobalAddressSDNode>(Base);
+}
+
+/// isAlias - Return true if there is any possibility that the two addresses
+/// overlap.
+bool DAGCombiner::isAlias(SDOperand Ptr1, int64_t Size1,
+ const Value *SrcValue1, int SrcValueOffset1,
+ SDOperand Ptr2, int64_t Size2,
+ const Value *SrcValue2, int SrcValueOffset2)
+{
+ // If they are the same then they must be aliases.
+ if (Ptr1 == Ptr2) return true;
+
+ // Gather base node and offset information.
+ SDOperand Base1, Base2;
+ int64_t Offset1, Offset2;
+ bool KnownBase1 = FindBaseOffset(Ptr1, Base1, Offset1);
+ bool KnownBase2 = FindBaseOffset(Ptr2, Base2, Offset2);
+
+ // If they have a same base address then...
+ if (Base1 == Base2) {
+ // Check to see if the addresses overlap.
+ return!((Offset1 + Size1) <= Offset2 || (Offset2 + Size2) <= Offset1);
+ }
+
+ // If we know both bases then they can't alias.
+ if (KnownBase1 && KnownBase2) return false;
+
+ if (CombinerGlobalAA) {
+ // Use alias analysis information.
+ int Overlap1 = Size1 + SrcValueOffset1 + Offset1;
+ int Overlap2 = Size2 + SrcValueOffset2 + Offset2;
+ AliasAnalysis::AliasResult AAResult =
+ AA.alias(SrcValue1, Overlap1, SrcValue2, Overlap2);
+ if (AAResult == AliasAnalysis::NoAlias)
+ return false;
+ }
+
+ // Otherwise we have to assume they alias.
+ return true;
+}
+
+/// FindAliasInfo - Extracts the relevant alias information from the memory
+/// node. Returns true if the operand was a load.
+bool DAGCombiner::FindAliasInfo(SDNode *N,
+ SDOperand &Ptr, int64_t &Size,
+ const Value *&SrcValue, int &SrcValueOffset) {
+ if (LoadSDNode *LD = dyn_cast<LoadSDNode>(N)) {
+ Ptr = LD->getBasePtr();
+ Size = MVT::getSizeInBits(LD->getLoadedVT()) >> 3;
+ SrcValue = LD->getSrcValue();
+ SrcValueOffset = LD->getSrcValueOffset();
+ return true;
+ } else if (StoreSDNode *ST = dyn_cast<StoreSDNode>(N)) {
+ Ptr = ST->getBasePtr();
+ Size = MVT::getSizeInBits(ST->getStoredVT()) >> 3;
+ SrcValue = ST->getSrcValue();
+ SrcValueOffset = ST->getSrcValueOffset();
+ } else {
+ assert(0 && "FindAliasInfo expected a memory operand");
+ }
+
+ return false;
+}
+
+/// GatherAllAliases - Walk up chain skipping non-aliasing memory nodes,
+/// looking for aliasing nodes and adding them to the Aliases vector.
+void DAGCombiner::GatherAllAliases(SDNode *N, SDOperand OriginalChain,
+ SmallVector<SDOperand, 8> &Aliases) {
+ SmallVector<SDOperand, 8> Chains; // List of chains to visit.
+ std::set<SDNode *> Visited; // Visited node set.
+
+ // Get alias information for node.
+ SDOperand Ptr;
+ int64_t Size;
+ const Value *SrcValue;
+ int SrcValueOffset;
+ bool IsLoad = FindAliasInfo(N, Ptr, Size, SrcValue, SrcValueOffset);
+
+ // Starting off.
+ Chains.push_back(OriginalChain);
+
+ // Look at each chain and determine if it is an alias. If so, add it to the
+ // aliases list. If not, then continue up the chain looking for the next
+ // candidate.
+ while (!Chains.empty()) {
+ SDOperand Chain = Chains.back();
+ Chains.pop_back();
+
+ // Don't bother if we've been before.
+ if (Visited.find(Chain.Val) != Visited.end()) continue;
+ Visited.insert(Chain.Val);
+
+ switch (Chain.getOpcode()) {
+ case ISD::EntryToken:
+ // Entry token is ideal chain operand, but handled in FindBetterChain.
+ break;
+
+ case ISD::LOAD:
+ case ISD::STORE: {
+ // Get alias information for Chain.
+ SDOperand OpPtr;
+ int64_t OpSize;
+ const Value *OpSrcValue;
+ int OpSrcValueOffset;
+ bool IsOpLoad = FindAliasInfo(Chain.Val, OpPtr, OpSize,
+ OpSrcValue, OpSrcValueOffset);
+
+ // If chain is alias then stop here.
+ if (!(IsLoad && IsOpLoad) &&
+ isAlias(Ptr, Size, SrcValue, SrcValueOffset,
+ OpPtr, OpSize, OpSrcValue, OpSrcValueOffset)) {
+ Aliases.push_back(Chain);
+ } else {
+ // Look further up the chain.
+ Chains.push_back(Chain.getOperand(0));
+ // Clean up old chain.
+ AddToWorkList(Chain.Val);
+ }
+ break;
+ }
+
+ case ISD::TokenFactor:
+ // We have to check each of the operands of the token factor, so we queue
+ // then up. Adding the operands to the queue (stack) in reverse order
+ // maintains the original order and increases the likelihood that getNode
+ // will find a matching token factor (CSE.)
+ for (unsigned n = Chain.getNumOperands(); n;)
+ Chains.push_back(Chain.getOperand(--n));
+ // Eliminate the token factor if we can.
+ AddToWorkList(Chain.Val);
+ break;
+
+ default:
+ // For all other instructions we will just have to take what we can get.
+ Aliases.push_back(Chain);
+ break;
+ }
+ }
+}
+
+/// FindBetterChain - Walk up chain skipping non-aliasing memory nodes, looking
+/// for a better chain (aliasing node.)
+SDOperand DAGCombiner::FindBetterChain(SDNode *N, SDOperand OldChain) {
+ SmallVector<SDOperand, 8> Aliases; // Ops for replacing token factor.
+
+ // Accumulate all the aliases to this node.
+ GatherAllAliases(N, OldChain, Aliases);
+
+ if (Aliases.size() == 0) {
+ // If no operands then chain to entry token.
+ return DAG.getEntryNode();
+ } else if (Aliases.size() == 1) {
+ // If a single operand then chain to it. We don't need to revisit it.
+ return Aliases[0];
+ }
+
+ // Construct a custom tailored token factor.
+ SDOperand NewChain = DAG.getNode(ISD::TokenFactor, MVT::Other,
+ &Aliases[0], Aliases.size());
+
+ // Make sure the old chain gets cleaned up.
+ if (NewChain != OldChain) AddToWorkList(OldChain.Val);
+
+ return NewChain;
+}
+
+// SelectionDAG::Combine - This is the entry point for the file.
+//
+void SelectionDAG::Combine(bool RunningAfterLegalize, AliasAnalysis &AA) {
+ if (!RunningAfterLegalize && ViewDAGCombine1)
+ viewGraph();
+ if (RunningAfterLegalize && ViewDAGCombine2)
+ viewGraph();
+ /// run - This is the main entry point to this class.
+ ///
+ DAGCombiner(*this, AA).Run(RunningAfterLegalize);
+}
diff --git a/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
new file mode 100644
index 0000000..5bcee35
--- /dev/null
+++ b/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -0,0 +1,5755 @@
+//===-- LegalizeDAG.cpp - Implement SelectionDAG::Legalize ----------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file was developed by the LLVM research group and is distributed under
+// the University of Illinois Open Source License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements the SelectionDAG::Legalize method.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/CodeGen/SelectionDAG.h"
+#include "llvm/CodeGen/MachineFunction.h"
+#include "llvm/CodeGen/MachineFrameInfo.h"
+#include "llvm/CodeGen/MachineJumpTableInfo.h"
+#include "llvm/Target/TargetLowering.h"
+#include "llvm/Target/TargetData.h"
+#include "llvm/Target/TargetMachine.h"
+#include "llvm/Target/TargetOptions.h"
+#include "llvm/CallingConv.h"
+#include "llvm/Constants.h"
+#include "llvm/DerivedTypes.h"
+#include "llvm/Support/MathExtras.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Compiler.h"
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/SmallPtrSet.h"
+#include <map>
+using namespace llvm;
+
+#ifndef NDEBUG
+static cl::opt<bool>
+ViewLegalizeDAGs("view-legalize-dags", cl::Hidden,
+ cl::desc("Pop up a window to show dags before legalize"));
+#else
+static const bool ViewLegalizeDAGs = 0;
+#endif
+
+//===----------------------------------------------------------------------===//
+/// SelectionDAGLegalize - This takes an arbitrary SelectionDAG as input and
+/// hacks on it until the target machine can handle it. This involves
+/// eliminating value sizes the machine cannot handle (promoting small sizes to
+/// large sizes or splitting up large values into small values) as well as
+/// eliminating operations the machine cannot handle.
+///
+/// This code also does a small amount of optimization and recognition of idioms
+/// as part of its processing. For example, if a target does not support a
+/// 'setcc' instruction efficiently, but does support 'brcc' instruction, this
+/// will attempt merge setcc and brc instructions into brcc's.
+///
+namespace {
+class VISIBILITY_HIDDEN SelectionDAGLegalize {
+ TargetLowering &TLI;
+ SelectionDAG &DAG;
+
+ // Libcall insertion helpers.
+
+ /// LastCALLSEQ_END - This keeps track of the CALLSEQ_END node that has been
+ /// legalized. We use this to ensure that calls are properly serialized
+ /// against each other, including inserted libcalls.
+ SDOperand LastCALLSEQ_END;
+
+ /// IsLegalizingCall - This member is used *only* for purposes of providing
+ /// helpful assertions that a libcall isn't created while another call is
+ /// being legalized (which could lead to non-serialized call sequences).
+ bool IsLegalizingCall;
+
+ enum LegalizeAction {
+ Legal, // The target natively supports this operation.
+ Promote, // This operation should be executed in a larger type.
+ Expand // Try to expand this to other ops, otherwise use a libcall.
+ };
+
+ /// ValueTypeActions - This is a bitvector that contains two bits for each
+ /// value type, where the two bits correspond to the LegalizeAction enum.
+ /// This can be queried with "getTypeAction(VT)".
+ TargetLowering::ValueTypeActionImpl ValueTypeActions;
+
+ /// LegalizedNodes - For nodes that are of legal width, and that have more
+ /// than one use, this map indicates what regularized operand to use. This
+ /// allows us to avoid legalizing the same thing more than once.
+ DenseMap<SDOperand, SDOperand> LegalizedNodes;
+
+ /// PromotedNodes - For nodes that are below legal width, and that have more
+ /// than one use, this map indicates what promoted value to use. This allows
+ /// us to avoid promoting the same thing more than once.
+ DenseMap<SDOperand, SDOperand> PromotedNodes;
+
+ /// ExpandedNodes - For nodes that need to be expanded this map indicates
+ /// which which operands are the expanded version of the input. This allows
+ /// us to avoid expanding the same node more than once.
+ DenseMap<SDOperand, std::pair<SDOperand, SDOperand> > ExpandedNodes;
+
+ /// SplitNodes - For vector nodes that need to be split, this map indicates
+ /// which which operands are the split version of the input. This allows us
+ /// to avoid splitting the same node more than once.
+ std::map<SDOperand, std::pair<SDOperand, SDOperand> > SplitNodes;
+
+ /// ScalarizedNodes - For nodes that need to be converted from vector types to
+ /// scalar types, this contains the mapping of ones we have already
+ /// processed to the result.
+ std::map<SDOperand, SDOperand> ScalarizedNodes;
+
+ void AddLegalizedOperand(SDOperand From, SDOperand To) {
+ LegalizedNodes.insert(std::make_pair(From, To));
+ // If someone requests legalization of the new node, return itself.
+ if (From != To)
+ LegalizedNodes.insert(std::make_pair(To, To));
+ }
+ void AddPromotedOperand(SDOperand From, SDOperand To) {
+ bool isNew = PromotedNodes.insert(std::make_pair(From, To));
+ assert(isNew && "Got into the map somehow?");
+ // If someone requests legalization of the new node, return itself.
+ LegalizedNodes.insert(std::make_pair(To, To));
+ }
+
+public:
+
+ SelectionDAGLegalize(SelectionDAG &DAG);
+
+ /// getTypeAction - Return how we should legalize values of this type, either
+ /// it is already legal or we need to expand it into multiple registers of
+ /// smaller integer type, or we need to promote it to a larger type.
+ LegalizeAction getTypeAction(MVT::ValueType VT) const {
+ return (LegalizeAction)ValueTypeActions.getTypeAction(VT);
+ }
+
+ /// isTypeLegal - Return true if this type is legal on this target.
+ ///
+ bool isTypeLegal(MVT::ValueType VT) const {
+ return getTypeAction(VT) == Legal;
+ }
+
+ void LegalizeDAG();
+
+private:
+ /// HandleOp - Legalize, Promote, or Expand the specified operand as
+ /// appropriate for its type.
+ void HandleOp(SDOperand Op);
+
+ /// LegalizeOp - We know that the specified value has a legal type.
+ /// Recursively ensure that the operands have legal types, then return the
+ /// result.
+ SDOperand LegalizeOp(SDOperand O);
+
+ /// PromoteOp - Given an operation that produces a value in an invalid type,
+ /// promote it to compute the value into a larger type. The produced value
+ /// will have the correct bits for the low portion of the register, but no
+ /// guarantee is made about the top bits: it may be zero, sign-extended, or
+ /// garbage.
+ SDOperand PromoteOp(SDOperand O);
+
+ /// ExpandOp - Expand the specified SDOperand into its two component pieces
+ /// Lo&Hi. Note that the Op MUST be an expanded type. As a result of this,
+ /// the LegalizeNodes map is filled in for any results that are not expanded,
+ /// the ExpandedNodes map is filled in for any results that are expanded, and
+ /// the Lo/Hi values are returned. This applies to integer types and Vector
+ /// types.
+ void ExpandOp(SDOperand O, SDOperand &Lo, SDOperand &Hi);
+
+ /// SplitVectorOp - Given an operand of vector type, break it down into
+ /// two smaller values.
+ void SplitVectorOp(SDOperand O, SDOperand &Lo, SDOperand &Hi);
+
+ /// ScalarizeVectorOp - Given an operand of single-element vector type
+ /// (e.g. v1f32), convert it into the equivalent operation that returns a
+ /// scalar (e.g. f32) value.
+ SDOperand ScalarizeVectorOp(SDOperand O);
+
+ /// isShuffleLegal - Return true if a vector shuffle is legal with the
+ /// specified mask and type. Targets can specify exactly which masks they
+ /// support and the code generator is tasked with not creating illegal masks.
+ ///
+ /// Note that this will also return true for shuffles that are promoted to a
+ /// different type.
+ ///
+ /// If this is a legal shuffle, this method returns the (possibly promoted)
+ /// build_vector Mask. If it's not a legal shuffle, it returns null.
+ SDNode *isShuffleLegal(MVT::ValueType VT, SDOperand Mask) const;
+
+ bool LegalizeAllNodesNotLeadingTo(SDNode *N, SDNode *Dest,
+ SmallPtrSet<SDNode*, 32> &NodesLeadingTo);
+
+ void LegalizeSetCCOperands(SDOperand &LHS, SDOperand &RHS, SDOperand &CC);
+
+ SDOperand CreateStackTemporary(MVT::ValueType VT);
+
+ SDOperand ExpandLibCall(const char *Name, SDNode *Node, bool isSigned,
+ SDOperand &Hi);
+ SDOperand ExpandIntToFP(bool isSigned, MVT::ValueType DestTy,
+ SDOperand Source);
+
+ SDOperand ExpandBIT_CONVERT(MVT::ValueType DestVT, SDOperand SrcOp);
+ SDOperand ExpandBUILD_VECTOR(SDNode *Node);
+ SDOperand ExpandSCALAR_TO_VECTOR(SDNode *Node);
+ SDOperand ExpandLegalINT_TO_FP(bool isSigned,
+ SDOperand LegalOp,
+ MVT::ValueType DestVT);
+ SDOperand PromoteLegalINT_TO_FP(SDOperand LegalOp, MVT::ValueType DestVT,
+ bool isSigned);
+ SDOperand PromoteLegalFP_TO_INT(SDOperand LegalOp, MVT::ValueType DestVT,
+ bool isSigned);
+
+ SDOperand ExpandBSWAP(SDOperand Op);
+ SDOperand ExpandBitCount(unsigned Opc, SDOperand Op);
+ bool ExpandShift(unsigned Opc, SDOperand Op, SDOperand Amt,
+ SDOperand &Lo, SDOperand &Hi);
+ void ExpandShiftParts(unsigned NodeOp, SDOperand Op, SDOperand Amt,
+ SDOperand &Lo, SDOperand &Hi);
+
+ SDOperand ExpandEXTRACT_SUBVECTOR(SDOperand Op);
+ SDOperand ExpandEXTRACT_VECTOR_ELT(SDOperand Op);
+
+ SDOperand getIntPtrConstant(uint64_t Val) {
+ return DAG.getConstant(Val, TLI.getPointerTy());
+ }
+};
+}
+
+/// isVectorShuffleLegal - Return true if a vector shuffle is legal with the
+/// specified mask and type. Targets can specify exactly which masks they
+/// support and the code generator is tasked with not creating illegal masks.
+///
+/// Note that this will also return true for shuffles that are promoted to a
+/// different type.
+SDNode *SelectionDAGLegalize::isShuffleLegal(MVT::ValueType VT,
+ SDOperand Mask) const {
+ switch (TLI.getOperationAction(ISD::VECTOR_SHUFFLE, VT)) {
+ default: return 0;
+ case TargetLowering::Legal:
+ case TargetLowering::Custom:
+ break;
+ case TargetLowering::Promote: {
+ // If this is promoted to a different type, convert the shuffle mask and
+ // ask if it is legal in the promoted type!
+ MVT::ValueType NVT = TLI.getTypeToPromoteTo(ISD::VECTOR_SHUFFLE, VT);
+
+ // If we changed # elements, change the shuffle mask.
+ unsigned NumEltsGrowth =
+ MVT::getVectorNumElements(NVT) / MVT::getVectorNumElements(VT);
+ assert(NumEltsGrowth && "Cannot promote to vector type with fewer elts!");
+ if (NumEltsGrowth > 1) {
+ // Renumber the elements.
+ SmallVector<SDOperand, 8> Ops;
+ for (unsigned i = 0, e = Mask.getNumOperands(); i != e; ++i) {
+ SDOperand InOp = Mask.getOperand(i);
+ for (unsigned j = 0; j != NumEltsGrowth; ++j) {
+ if (InOp.getOpcode() == ISD::UNDEF)
+ Ops.push_back(DAG.getNode(ISD::UNDEF, MVT::i32));
+ else {
+ unsigned InEltNo = cast<ConstantSDNode>(InOp)->getValue();
+ Ops.push_back(DAG.getConstant(InEltNo*NumEltsGrowth+j, MVT::i32));
+ }
+ }
+ }
+ Mask = DAG.getNode(ISD::BUILD_VECTOR, NVT, &Ops[0], Ops.size());
+ }
+ VT = NVT;
+ break;
+ }
+ }
+ return TLI.isShuffleMaskLegal(Mask, VT) ? Mask.Val : 0;
+}
+
+SelectionDAGLegalize::SelectionDAGLegalize(SelectionDAG &dag)
+ : TLI(dag.getTargetLoweringInfo()), DAG(dag),
+ ValueTypeActions(TLI.getValueTypeActions()) {
+ assert(MVT::LAST_VALUETYPE <= 32 &&
+ "Too many value types for ValueTypeActions to hold!");
+}
+
+/// ComputeTopDownOrdering - Compute a top-down ordering of the dag, where Order
+/// contains all of a nodes operands before it contains the node.
+static void ComputeTopDownOrdering(SelectionDAG &DAG,
+ SmallVector<SDNode*, 64> &Order) {
+
+ DenseMap<SDNode*, unsigned> Visited;
+ std::vector<SDNode*> Worklist;
+ Worklist.reserve(128);
+
+ // Compute ordering from all of the leaves in the graphs, those (like the
+ // entry node) that have no operands.
+ for (SelectionDAG::allnodes_iterator I = DAG.allnodes_begin(),
+ E = DAG.allnodes_end(); I != E; ++I) {
+ if (I->getNumOperands() == 0) {
+ Visited[I] = 0 - 1U;
+ Worklist.push_back(I);
+ }
+ }
+
+ while (!Worklist.empty()) {
+ SDNode *N = Worklist.back();
+ Worklist.pop_back();
+
+ if (++Visited[N] != N->getNumOperands())
+ continue; // Haven't visited all operands yet
+
+ Order.push_back(N);
+
+ // Now that we have N in, add anything that uses it if all of their operands
+ // are now done.
+ for (SDNode::use_iterator UI = N->use_begin(), E = N->use_end();
+ UI != E; ++UI)
+ Worklist.push_back(*UI);
+ }
+
+ assert(Order.size() == Visited.size() &&
+ Order.size() ==
+ (unsigned)std::distance(DAG.allnodes_begin(), DAG.allnodes_end()) &&
+ "Error: DAG is cyclic!");
+}
+
+
+void SelectionDAGLegalize::LegalizeDAG() {
+ LastCALLSEQ_END = DAG.getEntryNode();
+ IsLegalizingCall = false;
+
+ // The legalize process is inherently a bottom-up recursive process (users
+ // legalize their uses before themselves). Given infinite stack space, we
+ // could just start legalizing on the root and traverse the whole graph. In
+ // practice however, this causes us to run out of stack space on large basic
+ // blocks. To avoid this problem, compute an ordering of the nodes where each
+ // node is only legalized after all of its operands are legalized.
+ SmallVector<SDNode*, 64> Order;
+ ComputeTopDownOrdering(DAG, Order);
+
+ for (unsigned i = 0, e = Order.size(); i != e; ++i)
+ HandleOp(SDOperand(Order[i], 0));
+
+ // Finally, it's possible the root changed. Get the new root.
+ SDOperand OldRoot = DAG.getRoot();
+ assert(LegalizedNodes.count(OldRoot) && "Root didn't get legalized?");
+ DAG.setRoot(LegalizedNodes[OldRoot]);
+
+ ExpandedNodes.clear();
+ LegalizedNodes.clear();
+ PromotedNodes.clear();
+ SplitNodes.clear();
+ ScalarizedNodes.clear();
+
+ // Remove dead nodes now.
+ DAG.RemoveDeadNodes();
+}
+
+
+/// FindCallEndFromCallStart - Given a chained node that is part of a call
+/// sequence, find the CALLSEQ_END node that terminates the call sequence.
+static SDNode *FindCallEndFromCallStart(SDNode *Node) {
+ if (Node->getOpcode() == ISD::CALLSEQ_END)
+ return Node;
+ if (Node->use_empty())
+ return 0; // No CallSeqEnd
+
+ // The chain is usually at the end.
+ SDOperand TheChain(Node, Node->getNumValues()-1);
+ if (TheChain.getValueType() != MVT::Other) {
+ // Sometimes it's at the beginning.
+ TheChain = SDOperand(Node, 0);
+ if (TheChain.getValueType() != MVT::Other) {
+ // Otherwise, hunt for it.
+ for (unsigned i = 1, e = Node->getNumValues(); i != e; ++i)
+ if (Node->getValueType(i) == MVT::Other) {
+ TheChain = SDOperand(Node, i);
+ break;
+ }
+
+ // Otherwise, we walked into a node without a chain.
+ if (TheChain.getValueType() != MVT::Other)
+ return 0;
+ }
+ }
+
+ for (SDNode::use_iterator UI = Node->use_begin(),
+ E = Node->use_end(); UI != E; ++UI) {
+
+ // Make sure to only follow users of our token chain.
+ SDNode *User = *UI;
+ for (unsigned i = 0, e = User->getNumOperands(); i != e; ++i)
+ if (User->getOperand(i) == TheChain)
+ if (SDNode *Result = FindCallEndFromCallStart(User))
+ return Result;
+ }
+ return 0;
+}
+
+/// FindCallStartFromCallEnd - Given a chained node that is part of a call
+/// sequence, find the CALLSEQ_START node that initiates the call sequence.
+static SDNode *FindCallStartFromCallEnd(SDNode *Node) {
+ assert(Node && "Didn't find callseq_start for a call??");
+ if (Node->getOpcode() == ISD::CALLSEQ_START) return Node;
+
+ assert(Node->getOperand(0).getValueType() == MVT::Other &&
+ "Node doesn't have a token chain argument!");
+ return FindCallStartFromCallEnd(Node->getOperand(0).Val);
+}
+
+/// LegalizeAllNodesNotLeadingTo - Recursively walk the uses of N, looking to
+/// see if any uses can reach Dest. If no dest operands can get to dest,
+/// legalize them, legalize ourself, and return false, otherwise, return true.
+///
+/// Keep track of the nodes we fine that actually do lead to Dest in
+/// NodesLeadingTo. This avoids retraversing them exponential number of times.
+///
+bool SelectionDAGLegalize::LegalizeAllNodesNotLeadingTo(SDNode *N, SDNode *Dest,
+ SmallPtrSet<SDNode*, 32> &NodesLeadingTo) {
+ if (N == Dest) return true; // N certainly leads to Dest :)
+
+ // If we've already processed this node and it does lead to Dest, there is no
+ // need to reprocess it.
+ if (NodesLeadingTo.count(N)) return true;
+
+ // If the first result of this node has been already legalized, then it cannot
+ // reach N.
+ switch (getTypeAction(N->getValueType(0))) {
+ case Legal:
+ if (LegalizedNodes.count(SDOperand(N, 0))) return false;
+ break;
+ case Promote:
+ if (PromotedNodes.count(SDOperand(N, 0))) return false;
+ break;
+ case Expand:
+ if (ExpandedNodes.count(SDOperand(N, 0))) return false;
+ break;
+ }
+
+ // Okay, this node has not already been legalized. Check and legalize all
+ // operands. If none lead to Dest, then we can legalize this node.
+ bool OperandsLeadToDest = false;
+ for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i)
+ OperandsLeadToDest |= // If an operand leads to Dest, so do we.
+ LegalizeAllNodesNotLeadingTo(N->getOperand(i).Val, Dest, NodesLeadingTo);
+
+ if (OperandsLeadToDest) {
+ NodesLeadingTo.insert(N);
+ return true;
+ }
+
+ // Okay, this node looks safe, legalize it and return false.
+ HandleOp(SDOperand(N, 0));
+ return false;
+}
+
+/// HandleOp - Legalize, Promote, or Expand the specified operand as
+/// appropriate for its type.
+void SelectionDAGLegalize::HandleOp(SDOperand Op) {
+ MVT::ValueType VT = Op.getValueType();
+ switch (getTypeAction(VT)) {
+ default: assert(0 && "Bad type action!");
+ case Legal: (void)LegalizeOp(Op); break;
+ case Promote: (void)PromoteOp(Op); break;
+ case Expand:
+ if (!MVT::isVector(VT)) {
+ // If this is an illegal scalar, expand it into its two component
+ // pieces.
+ SDOperand X, Y;
+ ExpandOp(Op, X, Y);
+ } else if (MVT::getVectorNumElements(VT) == 1) {
+ // If this is an illegal single element vector, convert it to a
+ // scalar operation.
+ (void)ScalarizeVectorOp(Op);
+ } else {
+ // Otherwise, this is an illegal multiple element vector.
+ // Split it in half and legalize both parts.
+ SDOperand X, Y;
+ SplitVectorOp(Op, X, Y);
+ }
+ break;
+ }
+}
+
+/// ExpandConstantFP - Expands the ConstantFP node to an integer constant or
+/// a load from the constant pool.
+static SDOperand ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP,
+ SelectionDAG &DAG, TargetLowering &TLI) {
+ bool Extend = false;
+
+ // If a FP immediate is precise when represented as a float and if the
+ // target can do an extending load from float to double, we put it into
+ // the constant pool as a float, even if it's is statically typed as a
+ // double.
+ MVT::ValueType VT = CFP->getValueType(0);
+ bool isDouble = VT == MVT::f64;
+ ConstantFP *LLVMC = ConstantFP::get(isDouble ? Type::DoubleTy :
+ Type::FloatTy, CFP->getValue());
+ if (!UseCP) {
+ double Val = LLVMC->getValue();
+ return isDouble
+ ? DAG.getConstant(DoubleToBits(Val), MVT::i64)
+ : DAG.getConstant(FloatToBits(Val), MVT::i32);
+ }
+
+ if (isDouble && CFP->isExactlyValue((float)CFP->getValue()) &&
+ // Only do this if the target has a native EXTLOAD instruction from f32.
+ TLI.isLoadXLegal(ISD::EXTLOAD, MVT::f32)) {
+ LLVMC = cast<ConstantFP>(ConstantExpr::getFPTrunc(LLVMC,Type::FloatTy));
+ VT = MVT::f32;
+ Extend = true;
+ }
+
+ SDOperand CPIdx = DAG.getConstantPool(LLVMC, TLI.getPointerTy());
+ if (Extend) {
+ return DAG.getExtLoad(ISD::EXTLOAD, MVT::f64, DAG.getEntryNode(),
+ CPIdx, NULL, 0, MVT::f32);
+ } else {
+ return DAG.getLoad(VT, DAG.getEntryNode(), CPIdx, NULL, 0);
+ }
+}
+
+
+/// ExpandFCOPYSIGNToBitwiseOps - Expands fcopysign to a series of bitwise
+/// operations.
+static
+SDOperand ExpandFCOPYSIGNToBitwiseOps(SDNode *Node, MVT::ValueType NVT,
+ SelectionDAG &DAG, TargetLowering &TLI) {
+ MVT::ValueType VT = Node->getValueType(0);
+ MVT::ValueType SrcVT = Node->getOperand(1).getValueType();
+ assert((SrcVT == MVT::f32 || SrcVT == MVT::f64) &&
+ "fcopysign expansion only supported for f32 and f64");
+ MVT::ValueType SrcNVT = (SrcVT == MVT::f64) ? MVT::i64 : MVT::i32;
+
+ // First get the sign bit of second operand.
+ SDOperand Mask1 = (SrcVT == MVT::f64)
+ ? DAG.getConstantFP(BitsToDouble(1ULL << 63), SrcVT)
+ : DAG.getConstantFP(BitsToFloat(1U << 31), SrcVT);
+ Mask1 = DAG.getNode(ISD::BIT_CONVERT, SrcNVT, Mask1);
+ SDOperand SignBit= DAG.getNode(ISD::BIT_CONVERT, SrcNVT, Node->getOperand(1));
+ SignBit = DAG.getNode(ISD::AND, SrcNVT, SignBit, Mask1);
+ // Shift right or sign-extend it if the two operands have different types.
+ int SizeDiff = MVT::getSizeInBits(SrcNVT) - MVT::getSizeInBits(NVT);
+ if (SizeDiff > 0) {
+ SignBit = DAG.getNode(ISD::SRL, SrcNVT, SignBit,
+ DAG.getConstant(SizeDiff, TLI.getShiftAmountTy()));
+ SignBit = DAG.getNode(ISD::TRUNCATE, NVT, SignBit);
+ } else if (SizeDiff < 0)
+ SignBit = DAG.getNode(ISD::SIGN_EXTEND, NVT, SignBit);
+
+ // Clear the sign bit of first operand.
+ SDOperand Mask2 = (VT == MVT::f64)
+ ? DAG.getConstantFP(BitsToDouble(~(1ULL << 63)), VT)
+ : DAG.getConstantFP(BitsToFloat(~(1U << 31)), VT);
+ Mask2 = DAG.getNode(ISD::BIT_CONVERT, NVT, Mask2);
+ SDOperand Result = DAG.getNode(ISD::BIT_CONVERT, NVT, Node->getOperand(0));
+ Result = DAG.getNode(ISD::AND, NVT, Result, Mask2);
+
+ // Or the value with the sign bit.
+ Result = DAG.getNode(ISD::OR, NVT, Result, SignBit);
+ return Result;
+}
+
+
+/// LegalizeOp - We know that the specified value has a legal type, and
+/// that its operands are legal. Now ensure that the operation itself
+/// is legal, recursively ensuring that the operands' operations remain
+/// legal.
+SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) {
+ assert(isTypeLegal(Op.getValueType()) &&
+ "Caller should expand or promote operands that are not legal!");
+ SDNode *Node = Op.Val;
+
+ // If this operation defines any values that cannot be represented in a
+ // register on this target, make sure to expand or promote them.
+ if (Node->getNumValues() > 1) {
+ for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i)
+ if (getTypeAction(Node->getValueType(i)) != Legal) {
+ HandleOp(Op.getValue(i));
+ assert(LegalizedNodes.count(Op) &&
+ "Handling didn't add legal operands!");
+ return LegalizedNodes[Op];
+ }
+ }
+
+ // Note that LegalizeOp may be reentered even from single-use nodes, which
+ // means that we always must cache transformed nodes.
+ DenseMap<SDOperand, SDOperand>::iterator I = LegalizedNodes.find(Op);
+ if (I != LegalizedNodes.end()) return I->second;
+
+ SDOperand Tmp1, Tmp2, Tmp3, Tmp4;
+ SDOperand Result = Op;
+ bool isCustom = false;
+
+ switch (Node->getOpcode()) {
+ case ISD::FrameIndex:
+ case ISD::EntryToken:
+ case ISD::Register:
+ case ISD::BasicBlock:
+ case ISD::TargetFrameIndex:
+ case ISD::TargetJumpTable:
+ case ISD::TargetConstant:
+ case ISD::TargetConstantFP:
+ case ISD::TargetConstantPool:
+ case ISD::TargetGlobalAddress:
+ case ISD::TargetGlobalTLSAddress:
+ case ISD::TargetExternalSymbol:
+ case ISD::VALUETYPE:
+ case ISD::SRCVALUE:
+ case ISD::STRING:
+ case ISD::CONDCODE:
+ // Primitives must all be legal.
+ assert(TLI.isOperationLegal(Node->getValueType(0), Node->getValueType(0)) &&
+ "This must be legal!");
+ break;
+ default:
+ if (Node->getOpcode() >= ISD::BUILTIN_OP_END) {
+ // If this is a target node, legalize it by legalizing the operands then
+ // passing it through.
+ SmallVector<SDOperand, 8> Ops;
+ for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i)
+ Ops.push_back(LegalizeOp(Node->getOperand(i)));
+
+ Result = DAG.UpdateNodeOperands(Result.getValue(0), &Ops[0], Ops.size());
+
+ for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i)
+ AddLegalizedOperand(Op.getValue(i), Result.getValue(i));
+ return Result.getValue(Op.ResNo);
+ }
+ // Otherwise this is an unhandled builtin node. splat.
+#ifndef NDEBUG
+ cerr << "NODE: "; Node->dump(&DAG); cerr << "\n";
+#endif
+ assert(0 && "Do not know how to legalize this operator!");
+ abort();
+ case ISD::GLOBAL_OFFSET_TABLE:
+ case ISD::GlobalAddress:
+ case ISD::GlobalTLSAddress:
+ case ISD::ExternalSymbol:
+ case ISD::ConstantPool:
+ case ISD::JumpTable: // Nothing to do.
+ switch (TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0))) {
+ default: assert(0 && "This action is not supported yet!");
+ case TargetLowering::Custom:
+ Tmp1 = TLI.LowerOperation(Op, DAG);
+ if (Tmp1.Val) Result = Tmp1;
+ // FALLTHROUGH if the target doesn't want to lower this op after all.
+ case TargetLowering::Legal:
+ break;
+ }
+ break;
+ case ISD::FRAMEADDR:
+ case ISD::RETURNADDR:
+ case ISD::FRAME_TO_ARGS_OFFSET:
+ // The only option for these nodes is to custom lower them. If the target
+ // does not custom lower them, then return zero.
+ Tmp1 = TLI.LowerOperation(Op, DAG);
+ if (Tmp1.Val)
+ Result = Tmp1;
+ else
+ Result = DAG.getConstant(0, TLI.getPointerTy());
+ break;
+ case ISD::EXCEPTIONADDR: {
+ Tmp1 = LegalizeOp(Node->getOperand(0));
+ MVT::ValueType VT = Node->getValueType(0);
+ switch (TLI.getOperationAction(Node->getOpcode(), VT)) {
+ default: assert(0 && "This action is not supported yet!");
+ case TargetLowering::Expand: {
+ unsigned Reg = TLI.getExceptionAddressRegister();
+ Result = DAG.getCopyFromReg(Tmp1, Reg, VT).getValue(Op.ResNo);
+ }
+ break;
+ case TargetLowering::Custom:
+ Result = TLI.LowerOperation(Op, DAG);
+ if (Result.Val) break;
+ // Fall Thru
+ case TargetLowering::Legal: {
+ SDOperand Ops[] = { DAG.getConstant(0, VT), Tmp1 };
+ Result = DAG.getNode(ISD::MERGE_VALUES, DAG.getVTList(VT, MVT::Other),
+ Ops, 2).getValue(Op.ResNo);
+ break;
+ }
+ }
+ }
+ break;
+ case ISD::EHSELECTION: {
+ Tmp1 = LegalizeOp(Node->getOperand(0));
+ Tmp2 = LegalizeOp(Node->getOperand(1));
+ MVT::ValueType VT = Node->getValueType(0);
+ switch (TLI.getOperationAction(Node->getOpcode(), VT)) {
+ default: assert(0 && "This action is not supported yet!");
+ case TargetLowering::Expand: {
+ unsigned Reg = TLI.getExceptionSelectorRegister();
+ Result = DAG.getCopyFromReg(Tmp2, Reg, VT).getValue(Op.ResNo);
+ }
+ break;
+ case TargetLowering::Custom:
+ Result = TLI.LowerOperation(Op, DAG);
+ if (Result.Val) break;
+ // Fall Thru
+ case TargetLowering::Legal: {
+ SDOperand Ops[] = { DAG.getConstant(0, VT), Tmp2 };
+ Result = DAG.getNode(ISD::MERGE_VALUES, DAG.getVTList(VT, MVT::Other),
+ Ops, 2).getValue(Op.ResNo);
+ break;
+ }
+ }
+ }
+ break;
+ case ISD::EH_RETURN: {
+ MVT::ValueType VT = Node->getValueType(0);
+ // The only "good" option for this node is to custom lower it.
+ switch (TLI.getOperationAction(Node->getOpcode(), VT)) {
+ default: assert(0 && "This action is not supported at all!");
+ case TargetLowering::Custom:
+ Result = TLI.LowerOperation(Op, DAG);
+ if (Result.Val) break;
+ // Fall Thru
+ case TargetLowering::Legal:
+ // Target does not know, how to lower this, lower to noop
+ Result = LegalizeOp(Node->getOperand(0));
+ break;
+ }
+ }
+ break;
+ case ISD::AssertSext:
+ case ISD::AssertZext:
+ Tmp1 = LegalizeOp(Node->getOperand(0));
+ Result = DAG.UpdateNodeOperands(Result, Tmp1, Node->getOperand(1));
+ break;
+ case ISD::MERGE_VALUES:
+ // Legalize eliminates MERGE_VALUES nodes.
+ Result = Node->getOperand(Op.ResNo);
+ break;
+ case ISD::CopyFromReg:
+ Tmp1 = LegalizeOp(Node->getOperand(0));
+ Result = Op.getValue(0);
+ if (Node->getNumValues() == 2) {
+ Result = DAG.UpdateNodeOperands(Result, Tmp1, Node->getOperand(1));
+ } else {
+ assert(Node->getNumValues() == 3 && "Invalid copyfromreg!");
+ if (Node->getNumOperands() == 3) {
+ Tmp2 = LegalizeOp(Node->getOperand(2));
+ Result = DAG.UpdateNodeOperands(Result, Tmp1, Node->getOperand(1),Tmp2);
+ } else {
+ Result = DAG.UpdateNodeOperands(Result, Tmp1, Node->getOperand(1));
+ }
+ AddLegalizedOperand(Op.getValue(2), Result.getValue(2));
+ }
+ // Since CopyFromReg produces two values, make sure to remember that we
+ // legalized both of them.
+ AddLegalizedOperand(Op.getValue(0), Result);
+ AddLegalizedOperand(Op.getValue(1), Result.getValue(1));
+ return Result.getValue(Op.ResNo);
+ case ISD::UNDEF: {
+ MVT::ValueType VT = Op.getValueType();
+ switch (TLI.getOperationAction(ISD::UNDEF, VT)) {
+ default: assert(0 && "This action is not supported yet!");
+ case TargetLowering::Expand:
+ if (MVT::isInteger(VT))
+ Result = DAG.getConstant(0, VT);
+ else if (MVT::isFloatingPoint(VT))
+ Result = DAG.getConstantFP(0, VT);
+ else
+ assert(0 && "Unknown value type!");
+ break;
+ case TargetLowering::Legal:
+ break;
+ }
+ break;
+ }
+
+ case ISD::INTRINSIC_W_CHAIN:
+ case ISD::INTRINSIC_WO_CHAIN:
+ case ISD::INTRINSIC_VOID: {
+ SmallVector<SDOperand, 8> Ops;
+ for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i)
+ Ops.push_back(LegalizeOp(Node->getOperand(i)));
+ Result = DAG.UpdateNodeOperands(Result, &Ops[0], Ops.size());
+
+ // Allow the target to custom lower its intrinsics if it wants to.
+ if (TLI.getOperationAction(Node->getOpcode(), MVT::Other) ==
+ TargetLowering::Custom) {
+ Tmp3 = TLI.LowerOperation(Result, DAG);
+ if (Tmp3.Val) Result = Tmp3;
+ }
+
+ if (Result.Val->getNumValues() == 1) break;
+
+ // Must have return value and chain result.
+ assert(Result.Val->getNumValues() == 2 &&
+ "Cannot return more than two values!");
+
+ // Since loads produce two values, make sure to remember that we
+ // legalized both of them.
+ AddLegalizedOperand(SDOperand(Node, 0), Result.getValue(0));
+ AddLegalizedOperand(SDOperand(Node, 1), Result.getValue(1));
+ return Result.getValue(Op.ResNo);
+ }
+
+ case ISD::LOCATION:
+ assert(Node->getNumOperands() == 5 && "Invalid LOCATION node!");
+ Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the input chain.
+
+ switch (TLI.getOperationAction(ISD::LOCATION, MVT::Other)) {
+ case TargetLowering::Promote:
+ default: assert(0 && "This action is not supported yet!");
+ case TargetLowering::Expand: {
+ MachineModuleInfo *MMI = DAG.getMachineModuleInfo();
+ bool useDEBUG_LOC = TLI.isOperationLegal(ISD::DEBUG_LOC, MVT::Other);
+ bool useLABEL = TLI.isOperationLegal(ISD::LABEL, MVT::Other);
+
+ if (MMI && (useDEBUG_LOC || useLABEL)) {
+ const std::string &FName =
+ cast<StringSDNode>(Node->getOperand(3))->getValue();
+ const std::string &DirName =
+ cast<StringSDNode>(Node->getOperand(4))->getValue();
+ unsigned SrcFile = MMI->RecordSource(DirName, FName);
+
+ SmallVector<SDOperand, 8> Ops;
+ Ops.push_back(Tmp1); // chain
+ SDOperand LineOp = Node->getOperand(1);
+ SDOperand ColOp = Node->getOperand(2);
+
+ if (useDEBUG_LOC) {
+ Ops.push_back(LineOp); // line #
+ Ops.push_back(ColOp); // col #
+ Ops.push_back(DAG.getConstant(SrcFile, MVT::i32)); // source file id
+ Result = DAG.getNode(ISD::DEBUG_LOC, MVT::Other, &Ops[0], Ops.size());
+ } else {
+ unsigned Line = cast<ConstantSDNode>(LineOp)->getValue();
+ unsigned Col = cast<ConstantSDNode>(ColOp)->getValue();
+ unsigned ID = MMI->RecordLabel(Line, Col, SrcFile);
+ Ops.push_back(DAG.getConstant(ID, MVT::i32));
+ Result = DAG.getNode(ISD::LABEL, MVT::Other,&Ops[0],Ops.size());
+ }
+ } else {
+ Result = Tmp1; // chain
+ }
+ break;
+ }
+ case TargetLowering::Legal:
+ if (Tmp1 != Node->getOperand(0) ||
+ getTypeAction(Node->getOperand(1).getValueType()) == Promote) {
+ SmallVector<SDOperand, 8> Ops;
+ Ops.push_back(Tmp1);
+ if (getTypeAction(Node->getOperand(1).getValueType()) == Legal) {
+ Ops.push_back(Node->getOperand(1)); // line # must be legal.
+ Ops.push_back(Node->getOperand(2)); // col # must be legal.
+ } else {
+ // Otherwise promote them.
+ Ops.push_back(PromoteOp(Node->getOperand(1)));
+ Ops.push_back(PromoteOp(Node->getOperand(2)));
+ }
+ Ops.push_back(Node->getOperand(3)); // filename must be legal.
+ Ops.push_back(Node->getOperand(4)); // working dir # must be legal.
+ Result = DAG.UpdateNodeOperands(Result, &Ops[0], Ops.size());
+ }
+ break;
+ }
+ break;
+
+ case ISD::DEBUG_LOC:
+ assert(Node->getNumOperands() == 4 && "Invalid DEBUG_LOC node!");
+ switch (TLI.getOperationAction(ISD::DEBUG_LOC, MVT::Other)) {
+ default: assert(0 && "This action is not supported yet!");
+ case TargetLowering::Legal:
+ Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the chain.
+ Tmp2 = LegalizeOp(Node->getOperand(1)); // Legalize the line #.
+ Tmp3 = LegalizeOp(Node->getOperand(2)); // Legalize the col #.
+ Tmp4 = LegalizeOp(Node->getOperand(3)); // Legalize the source file id.
+ Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3, Tmp4);
+ break;
+ }
+ break;
+
+ case ISD::LABEL:
+ assert(Node->getNumOperands() == 2 && "Invalid LABEL node!");
+ switch (TLI.getOperationAction(ISD::LABEL, MVT::Other)) {
+ default: assert(0 && "This action is not supported yet!");
+ case TargetLowering::Legal:
+ Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the chain.
+ Tmp2 = LegalizeOp(Node->getOperand(1)); // Legalize the label id.
+ Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2);
+ break;
+ case TargetLowering::Expand:
+ Result = LegalizeOp(Node->getOperand(0));
+ break;
+ }
+ break;
+
+ case ISD::Constant:
+ // We know we don't need to expand constants here, constants only have one
+ // value and we check that it is fine above.
+
+ // FIXME: Maybe we should handle things like targets that don't support full
+ // 32-bit immediates?
+ break;
+ case ISD::ConstantFP: {
+ // Spill FP immediates to the constant pool if the target cannot directly
+ // codegen them. Targets often have some immediate values that can be
+ // efficiently generated into an FP register without a load. We explicitly
+ // leave these constants as ConstantFP nodes for the target to deal with.
+ ConstantFPSDNode *CFP = cast<ConstantFPSDNode>(Node);
+
+ // Check to see if this FP immediate is already legal.
+ bool isLegal = false;
+ for (TargetLowering::legal_fpimm_iterator I = TLI.legal_fpimm_begin(),
+ E = TLI.legal_fpimm_end(); I != E; ++I)
+ if (CFP->isExactlyValue(*I)) {
+ isLegal = true;
+ break;
+ }
+
+ // If this is a legal constant, turn it into a TargetConstantFP node.
+ if (isLegal) {
+ Result = DAG.getTargetConstantFP(CFP->getValue(), CFP->getValueType(0));
+ break;
+ }
+
+ switch (TLI.getOperationAction(ISD::ConstantFP, CFP->getValueType(0))) {
+ default: assert(0 && "This action is not supported yet!");
+ case TargetLowering::Custom:
+ Tmp3 = TLI.LowerOperation(Result, DAG);
+ if (Tmp3.Val) {
+ Result = Tmp3;
+ break;
+ }
+ // FALLTHROUGH
+ case TargetLowering::Expand:
+ Result = ExpandConstantFP(CFP, true, DAG, TLI);
+ }
+ break;
+ }
+ case ISD::TokenFactor:
+ if (Node->getNumOperands() == 2) {
+ Tmp1 = LegalizeOp(Node->getOperand(0));
+ Tmp2 = LegalizeOp(Node->getOperand(1));
+ Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2);
+ } else if (Node->getNumOperands() == 3) {
+ Tmp1 = LegalizeOp(Node->getOperand(0));
+ Tmp2 = LegalizeOp(Node->getOperand(1));
+ Tmp3 = LegalizeOp(Node->getOperand(2));
+ Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3);
+ } else {
+ SmallVector<SDOperand, 8> Ops;
+ // Legalize the operands.
+ for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i)
+ Ops.push_back(LegalizeOp(Node->getOperand(i)));
+ Result = DAG.UpdateNodeOperands(Result, &Ops[0], Ops.size());
+ }
+ break;
+
+ case ISD::FORMAL_ARGUMENTS:
+ case ISD::CALL:
+ // The only option for this is to custom lower it.
+ Tmp3 = TLI.LowerOperation(Result.getValue(0), DAG);
+ assert(Tmp3.Val && "Target didn't custom lower this node!");
+ assert(Tmp3.Val->getNumValues() == Result.Val->getNumValues() &&
+ "Lowering call/formal_arguments produced unexpected # results!");
+
+ // Since CALL/FORMAL_ARGUMENTS nodes produce multiple values, make sure to
+ // remember that we legalized all of them, so it doesn't get relegalized.
+ for (unsigned i = 0, e = Tmp3.Val->getNumValues(); i != e; ++i) {
+ Tmp1 = LegalizeOp(Tmp3.getValue(i));
+ if (Op.ResNo == i)
+ Tmp2 = Tmp1;
+ AddLegalizedOperand(SDOperand(Node, i), Tmp1);
+ }
+ return Tmp2;
+
+ case ISD::BUILD_VECTOR:
+ switch (TLI.getOperationAction(ISD::BUILD_VECTOR, Node->getValueType(0))) {
+ default: assert(0 && "This action is not supported yet!");
+ case TargetLowering::Custom:
+ Tmp3 = TLI.LowerOperation(Result, DAG);
+ if (Tmp3.Val) {
+ Result = Tmp3;
+ break;
+ }
+ // FALLTHROUGH
+ case TargetLowering::Expand:
+ Result = ExpandBUILD_VECTOR(Result.Val);
+ break;
+ }
+ break;
+ case ISD::INSERT_VECTOR_ELT:
+ Tmp1 = LegalizeOp(Node->getOperand(0)); // InVec
+ Tmp2 = LegalizeOp(Node->getOperand(1)); // InVal
+ Tmp3 = LegalizeOp(Node->getOperand(2)); // InEltNo
+ Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3);
+
+ switch (TLI.getOperationAction(ISD::INSERT_VECTOR_ELT,
+ Node->getValueType(0))) {
+ default: assert(0 && "This action is not supported yet!");
+ case TargetLowering::Legal:
+ break;
+ case TargetLowering::Custom:
+ Tmp3 = TLI.LowerOperation(Result, DAG);
+ if (Tmp3.Val) {
+ Result = Tmp3;
+ break;
+ }
+ // FALLTHROUGH
+ case TargetLowering::Expand: {
+ // If the insert index is a constant, codegen this as a scalar_to_vector,
+ // then a shuffle that inserts it into the right position in the vector.
+ if (ConstantSDNode *InsertPos = dyn_cast<ConstantSDNode>(Tmp3)) {
+ SDOperand ScVec = DAG.getNode(ISD::SCALAR_TO_VECTOR,
+ Tmp1.getValueType(), Tmp2);
+
+ unsigned NumElts = MVT::getVectorNumElements(Tmp1.getValueType());
+ MVT::ValueType ShufMaskVT = MVT::getIntVectorWithNumElements(NumElts);
+ MVT::ValueType ShufMaskEltVT = MVT::getVectorElementType(ShufMaskVT);
+
+ // We generate a shuffle of InVec and ScVec, so the shuffle mask should
+ // be 0,1,2,3,4,5... with the appropriate element replaced with elt 0 of
+ // the RHS.
+ SmallVector<SDOperand, 8> ShufOps;
+ for (unsigned i = 0; i != NumElts; ++i) {
+ if (i != InsertPos->getValue())
+ ShufOps.push_back(DAG.getConstant(i, ShufMaskEltVT));
+ else
+ ShufOps.push_back(DAG.getConstant(NumElts, ShufMaskEltVT));
+ }
+ SDOperand ShufMask = DAG.getNode(ISD::BUILD_VECTOR, ShufMaskVT,
+ &ShufOps[0], ShufOps.size());
+
+ Result = DAG.getNode(ISD::VECTOR_SHUFFLE, Tmp1.getValueType(),
+ Tmp1, ScVec, ShufMask);
+ Result = LegalizeOp(Result);
+ break;
+ }
+
+ // If the target doesn't support this, we have to spill the input vector
+ // to a temporary stack slot, update the element, then reload it. This is
+ // badness. We could also load the value into a vector register (either
+ // with a "move to register" or "extload into register" instruction, then
+ // permute it into place, if the idx is a constant and if the idx is
+ // supported by the target.
+ MVT::ValueType VT = Tmp1.getValueType();
+ MVT::ValueType EltVT = Tmp2.getValueType();
+ MVT::ValueType IdxVT = Tmp3.getValueType();
+ MVT::ValueType PtrVT = TLI.getPointerTy();
+ SDOperand StackPtr = CreateStackTemporary(VT);
+ // Store the vector.
+ SDOperand Ch = DAG.getStore(DAG.getEntryNode(), Tmp1, StackPtr, NULL, 0);
+
+ // Truncate or zero extend offset to target pointer type.
+ unsigned CastOpc = (IdxVT > PtrVT) ? ISD::TRUNCATE : ISD::ZERO_EXTEND;
+ Tmp3 = DAG.getNode(CastOpc, PtrVT, Tmp3);
+ // Add the offset to the index.
+ unsigned EltSize = MVT::getSizeInBits(EltVT)/8;
+ Tmp3 = DAG.getNode(ISD::MUL, IdxVT, Tmp3,DAG.getConstant(EltSize, IdxVT));
+ SDOperand StackPtr2 = DAG.getNode(ISD::ADD, IdxVT, Tmp3, StackPtr);
+ // Store the scalar value.
+ Ch = DAG.getStore(Ch, Tmp2, StackPtr2, NULL, 0);
+ // Load the updated vector.
+ Result = DAG.getLoad(VT, Ch, StackPtr, NULL, 0);
+ break;
+ }
+ }
+ break;
+ case ISD::SCALAR_TO_VECTOR:
+ if (!TLI.isTypeLegal(Node->getOperand(0).getValueType())) {
+ Result = LegalizeOp(ExpandSCALAR_TO_VECTOR(Node));
+ break;
+ }
+
+ Tmp1 = LegalizeOp(Node->getOperand(0)); // InVal
+ Result = DAG.UpdateNodeOperands(Result, Tmp1);
+ switch (TLI.getOperationAction(ISD::SCALAR_TO_VECTOR,
+ Node->getValueType(0))) {
+ default: assert(0 && "This action is not supported yet!");
+ case TargetLowering::Legal:
+ break;
+ case TargetLowering::Custom:
+ Tmp3 = TLI.LowerOperation(Result, DAG);
+ if (Tmp3.Val) {
+ Result = Tmp3;
+ break;
+ }
+ // FALLTHROUGH
+ case TargetLowering::Expand:
+ Result = LegalizeOp(ExpandSCALAR_TO_VECTOR(Node));
+ break;
+ }
+ break;
+ case ISD::VECTOR_SHUFFLE:
+ Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the input vectors,
+ Tmp2 = LegalizeOp(Node->getOperand(1)); // but not the shuffle mask.
+ Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Node->getOperand(2));
+
+ // Allow targets to custom lower the SHUFFLEs they support.
+ switch (TLI.getOperationAction(ISD::VECTOR_SHUFFLE,Result.getValueType())) {
+ default: assert(0 && "Unknown operation action!");
+ case TargetLowering::Legal:
+ assert(isShuffleLegal(Result.getValueType(), Node->getOperand(2)) &&
+ "vector shuffle should not be created if not legal!");
+ break;
+ case TargetLowering::Custom:
+ Tmp3 = TLI.LowerOperation(Result, DAG);
+ if (Tmp3.Val) {
+ Result = Tmp3;
+ break;
+ }
+ // FALLTHROUGH
+ case TargetLowering::Expand: {
+ MVT::ValueType VT = Node->getValueType(0);
+ MVT::ValueType EltVT = MVT::getVectorElementType(VT);
+ MVT::ValueType PtrVT = TLI.getPointerTy();
+ SDOperand Mask = Node->getOperand(2);
+ unsigned NumElems = Mask.getNumOperands();
+ SmallVector<SDOperand,8> Ops;
+ for (unsigned i = 0; i != NumElems; ++i) {
+ SDOperand Arg = Mask.getOperand(i);
+ if (Arg.getOpcode() == ISD::UNDEF) {
+ Ops.push_back(DAG.getNode(ISD::UNDEF, EltVT));
+ } else {
+ assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!");
+ unsigned Idx = cast<ConstantSDNode>(Arg)->getValue();
+ if (Idx < NumElems)
+ Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, EltVT, Tmp1,
+ DAG.getConstant(Idx, PtrVT)));
+ else
+ Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, EltVT, Tmp2,
+ DAG.getConstant(Idx - NumElems, PtrVT)));
+ }
+ }
+ Result = DAG.getNode(ISD::BUILD_VECTOR, VT, &Ops[0], Ops.size());
+ break;
+ }
+ case TargetLowering::Promote: {
+ // Change base type to a different vector type.
+ MVT::ValueType OVT = Node->getValueType(0);
+ MVT::ValueType NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), OVT);
+
+ // Cast the two input vectors.
+ Tmp1 = DAG.getNode(ISD::BIT_CONVERT, NVT, Tmp1);
+ Tmp2 = DAG.getNode(ISD::BIT_CONVERT, NVT, Tmp2);
+
+ // Convert the shuffle mask to the right # elements.
+ Tmp3 = SDOperand(isShuffleLegal(OVT, Node->getOperand(2)), 0);
+ assert(Tmp3.Val && "Shuffle not legal?");
+ Result = DAG.getNode(ISD::VECTOR_SHUFFLE, NVT, Tmp1, Tmp2, Tmp3);
+ Result = DAG.getNode(ISD::BIT_CONVERT, OVT, Result);
+ break;
+ }
+ }
+ break;
+
+ case ISD::EXTRACT_VECTOR_ELT:
+ Tmp1 = Node->getOperand(0);
+ Tmp2 = LegalizeOp(Node->getOperand(1));
+ Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2);
+ Result = ExpandEXTRACT_VECTOR_ELT(Result);
+ break;
+
+ case ISD::EXTRACT_SUBVECTOR:
+ Tmp1 = Node->getOperand(0);
+ Tmp2 = LegalizeOp(Node->getOperand(1));
+ Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2);
+ Result = ExpandEXTRACT_SUBVECTOR(Result);
+ break;
+
+ case ISD::CALLSEQ_START: {
+ SDNode *CallEnd = FindCallEndFromCallStart(Node);
+
+ // Recursively Legalize all of the inputs of the call end that do not lead
+ // to this call start. This ensures that any libcalls that need be inserted
+ // are inserted *before* the CALLSEQ_START.
+ {SmallPtrSet<SDNode*, 32> NodesLeadingTo;
+ for (unsigned i = 0, e = CallEnd->getNumOperands(); i != e; ++i)
+ LegalizeAllNodesNotLeadingTo(CallEnd->getOperand(i).Val, Node,
+ NodesLeadingTo);
+ }
+
+ // Now that we legalized all of the inputs (which may have inserted
+ // libcalls) create the new CALLSEQ_START node.
+ Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the chain.
+
+ // Merge in the last call, to ensure that this call start after the last
+ // call ended.
+ if (LastCALLSEQ_END.getOpcode() != ISD::EntryToken) {
+ Tmp1 = DAG.getNode(ISD::TokenFactor, MVT::Other, Tmp1, LastCALLSEQ_END);
+ Tmp1 = LegalizeOp(Tmp1);
+ }
+
+ // Do not try to legalize the target-specific arguments (#1+).
+ if (Tmp1 != Node->getOperand(0)) {
+ SmallVector<SDOperand, 8> Ops(Node->op_begin(), Node->op_end());
+ Ops[0] = Tmp1;
+ Result = DAG.UpdateNodeOperands(Result, &Ops[0], Ops.size());
+ }
+
+ // Remember that the CALLSEQ_START is legalized.
+ AddLegalizedOperand(Op.getValue(0), Result);
+ if (Node->getNumValues() == 2) // If this has a flag result, remember it.
+ AddLegalizedOperand(Op.getValue(1), Result.getValue(1));
+
+ // Now that the callseq_start and all of the non-call nodes above this call
+ // sequence have been legalized, legalize the call itself. During this
+ // process, no libcalls can/will be inserted, guaranteeing that no calls
+ // can overlap.
+ assert(!IsLegalizingCall && "Inconsistent sequentialization of calls!");
+ SDOperand InCallSEQ = LastCALLSEQ_END;
+ // Note that we are selecting this call!
+ LastCALLSEQ_END = SDOperand(CallEnd, 0);
+ IsLegalizingCall = true;
+
+ // Legalize the call, starting from the CALLSEQ_END.
+ LegalizeOp(LastCALLSEQ_END);
+ assert(!IsLegalizingCall && "CALLSEQ_END should have cleared this!");
+ return Result;
+ }
+ case ISD::CALLSEQ_END:
+ // If the CALLSEQ_START node hasn't been legalized first, legalize it. This
+ // will cause this node to be legalized as well as handling libcalls right.
+ if (LastCALLSEQ_END.Val != Node) {
+ LegalizeOp(SDOperand(FindCallStartFromCallEnd(Node), 0));
+ DenseMap<SDOperand, SDOperand>::iterator I = LegalizedNodes.find(Op);
+ assert(I != LegalizedNodes.end() &&
+ "Legalizing the call start should have legalized this node!");
+ return I->second;
+ }
+
+ // Otherwise, the call start has been legalized and everything is going
+ // according to plan. Just legalize ourselves normally here.
+ Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the chain.
+ // Do not try to legalize the target-specific arguments (#1+), except for
+ // an optional flag input.
+ if (Node->getOperand(Node->getNumOperands()-1).getValueType() != MVT::Flag){
+ if (Tmp1 != Node->getOperand(0)) {
+ SmallVector<SDOperand, 8> Ops(Node->op_begin(), Node->op_end());
+ Ops[0] = Tmp1;
+ Result = DAG.UpdateNodeOperands(Result, &Ops[0], Ops.size());
+ }
+ } else {
+ Tmp2 = LegalizeOp(Node->getOperand(Node->getNumOperands()-1));
+ if (Tmp1 != Node->getOperand(0) ||
+ Tmp2 != Node->getOperand(Node->getNumOperands()-1)) {
+ SmallVector<SDOperand, 8> Ops(Node->op_begin(), Node->op_end());
+ Ops[0] = Tmp1;
+ Ops.back() = Tmp2;
+ Result = DAG.UpdateNodeOperands(Result, &Ops[0], Ops.size());
+ }
+ }
+ assert(IsLegalizingCall && "Call sequence imbalance between start/end?");
+ // This finishes up call legalization.
+ IsLegalizingCall = false;
+
+ // If the CALLSEQ_END node has a flag, remember that we legalized it.
+ AddLegalizedOperand(SDOperand(Node, 0), Result.getValue(0));
+ if (Node->getNumValues() == 2)
+ AddLegalizedOperand(SDOperand(Node, 1), Result.getValue(1));
+ return Result.getValue(Op.ResNo);
+ case ISD::DYNAMIC_STACKALLOC: {
+ Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the chain.
+ Tmp2 = LegalizeOp(Node->getOperand(1)); // Legalize the size.
+ Tmp3 = LegalizeOp(Node->getOperand(2)); // Legalize the alignment.
+ Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3);
+
+ Tmp1 = Result.getValue(0);
+ Tmp2 = Result.getValue(1);
+ switch (TLI.getOperationAction(Node->getOpcode(),
+ Node->getValueType(0))) {
+ default: assert(0 && "This action is not supported yet!");
+ case TargetLowering::Expand: {
+ unsigned SPReg = TLI.getStackPointerRegisterToSaveRestore();
+ assert(SPReg && "Target cannot require DYNAMIC_STACKALLOC expansion and"
+ " not tell us which reg is the stack pointer!");
+ SDOperand Chain = Tmp1.getOperand(0);
+ SDOperand Size = Tmp2.getOperand(1);
+ SDOperand SP = DAG.getCopyFromReg(Chain, SPReg, Node->getValueType(0));
+ Tmp1 = DAG.getNode(ISD::SUB, Node->getValueType(0), SP, Size); // Value
+ Tmp2 = DAG.getCopyToReg(SP.getValue(1), SPReg, Tmp1); // Output chain
+ Tmp1 = LegalizeOp(Tmp1);
+ Tmp2 = LegalizeOp(Tmp2);
+ break;
+ }
+ case TargetLowering::Custom:
+ Tmp3 = TLI.LowerOperation(Tmp1, DAG);
+ if (Tmp3.Val) {
+ Tmp1 = LegalizeOp(Tmp3);
+ Tmp2 = LegalizeOp(Tmp3.getValue(1));
+ }
+ break;
+ case TargetLowering::Legal:
+ break;
+ }
+ // Since this op produce two values, make sure to remember that we
+ // legalized both of them.
+ AddLegalizedOperand(SDOperand(Node, 0), Tmp1);
+ AddLegalizedOperand(SDOperand(Node, 1), Tmp2);
+ return Op.ResNo ? Tmp2 : Tmp1;
+ }
+ case ISD::INLINEASM: {
+ SmallVector<SDOperand, 8> Ops(Node->op_begin(), Node->op_end());
+ bool Changed = false;
+ // Legalize all of the operands of the inline asm, in case they are nodes
+ // that need to be expanded or something. Note we skip the asm string and
+ // all of the TargetConstant flags.
+ SDOperand Op = LegalizeOp(Ops[0]);
+ Changed = Op != Ops[0];
+ Ops[0] = Op;
+
+ bool HasInFlag = Ops.back().getValueType() == MVT::Flag;
+ for (unsigned i = 2, e = Ops.size()-HasInFlag; i < e; ) {
+ unsigned NumVals = cast<ConstantSDNode>(Ops[i])->getValue() >> 3;
+ for (++i; NumVals; ++i, --NumVals) {
+ SDOperand Op = LegalizeOp(Ops[i]);
+ if (Op != Ops[i]) {
+ Changed = true;
+ Ops[i] = Op;
+ }
+ }
+ }
+
+ if (HasInFlag) {
+ Op = LegalizeOp(Ops.back());
+ Changed |= Op != Ops.back();
+ Ops.back() = Op;
+ }
+
+ if (Changed)
+ Result = DAG.UpdateNodeOperands(Result, &Ops[0], Ops.size());
+
+ // INLINE asm returns a chain and flag, make sure to add both to the map.
+ AddLegalizedOperand(SDOperand(Node, 0), Result.getValue(0));
+ AddLegalizedOperand(SDOperand(Node, 1), Result.getValue(1));
+ return Result.getValue(Op.ResNo);
+ }
+ case ISD::BR:
+ Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the chain.
+ // Ensure that libcalls are emitted before a branch.
+ Tmp1 = DAG.getNode(ISD::TokenFactor, MVT::Other, Tmp1, LastCALLSEQ_END);
+ Tmp1 = LegalizeOp(Tmp1);
+ LastCALLSEQ_END = DAG.getEntryNode();
+
+ Result = DAG.UpdateNodeOperands(Result, Tmp1, Node->getOperand(1));
+ break;
+ case ISD::BRIND:
+ Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the chain.
+ // Ensure that libcalls are emitted before a branch.
+ Tmp1 = DAG.getNode(ISD::TokenFactor, MVT::Other, Tmp1, LastCALLSEQ_END);
+ Tmp1 = LegalizeOp(Tmp1);
+ LastCALLSEQ_END = DAG.getEntryNode();
+
+ switch (getTypeAction(Node->getOperand(1).getValueType())) {
+ default: assert(0 && "Indirect target must be legal type (pointer)!");
+ case Legal:
+ Tmp2 = LegalizeOp(Node->getOperand(1)); // Legalize the condition.
+ break;
+ }
+ Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2);
+ break;
+ case ISD::BR_JT:
+ Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the chain.
+ // Ensure that libcalls are emitted before a branch.
+ Tmp1 = DAG.getNode(ISD::TokenFactor, MVT::Other, Tmp1, LastCALLSEQ_END);
+ Tmp1 = LegalizeOp(Tmp1);
+ LastCALLSEQ_END = DAG.getEntryNode();
+
+ Tmp2 = LegalizeOp(Node->getOperand(1)); // Legalize the jumptable node.
+ Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Node->getOperand(2));
+
+ switch (TLI.getOperationAction(ISD::BR_JT, MVT::Other)) {
+ default: assert(0 && "This action is not supported yet!");
+ case TargetLowering::Legal: break;
+ case TargetLowering::Custom:
+ Tmp1 = TLI.LowerOperation(Result, DAG);
+ if (Tmp1.Val) Result = Tmp1;
+ break;
+ case TargetLowering::Expand: {
+ SDOperand Chain = Result.getOperand(0);
+ SDOperand Table = Result.getOperand(1);
+ SDOperand Index = Result.getOperand(2);
+
+ MVT::ValueType PTy = TLI.getPointerTy();
+ MachineFunction &MF = DAG.getMachineFunction();
+ unsigned EntrySize = MF.getJumpTableInfo()->getEntrySize();
+ Index= DAG.getNode(ISD::MUL, PTy, Index, DAG.getConstant(EntrySize, PTy));
+ SDOperand Addr = DAG.getNode(ISD::ADD, PTy, Index, Table);
+
+ SDOperand LD;
+ switch (EntrySize) {
+ default: assert(0 && "Size of jump table not supported yet."); break;
+ case 4: LD = DAG.getLoad(MVT::i32, Chain, Addr, NULL, 0); break;
+ case 8: LD = DAG.getLoad(MVT::i64, Chain, Addr, NULL, 0); break;
+ }
+
+ if (TLI.getTargetMachine().getRelocationModel() == Reloc::PIC_) {
+ // For PIC, the sequence is:
+ // BRIND(load(Jumptable + index) + RelocBase)
+ // RelocBase is the JumpTable on PPC and X86, GOT on Alpha
+ SDOperand Reloc;
+ if (TLI.usesGlobalOffsetTable())
+ Reloc = DAG.getNode(ISD::GLOBAL_OFFSET_TABLE, PTy);
+ else
+ Reloc = Table;
+ Addr = (PTy != MVT::i32) ? DAG.getNode(ISD::SIGN_EXTEND, PTy, LD) : LD;
+ Addr = DAG.getNode(ISD::ADD, PTy, Addr, Reloc);
+ Result = DAG.getNode(ISD::BRIND, MVT::Other, LD.getValue(1), Addr);
+ } else {
+ Result = DAG.getNode(ISD::BRIND, MVT::Other, LD.getValue(1), LD);
+ }
+ }
+ }
+ break;
+ case ISD::BRCOND:
+ Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the chain.
+ // Ensure that libcalls are emitted before a return.
+ Tmp1 = DAG.getNode(ISD::TokenFactor, MVT::Other, Tmp1, LastCALLSEQ_END);
+ Tmp1 = LegalizeOp(Tmp1);
+ LastCALLSEQ_END = DAG.getEntryNode();
+
+ switch (getTypeAction(Node->getOperand(1).getValueType())) {
+ case Expand: assert(0 && "It's impossible to expand bools");
+ case Legal:
+ Tmp2 = LegalizeOp(Node->getOperand(1)); // Legalize the condition.
+ break;
+ case Promote:
+ Tmp2 = PromoteOp(Node->getOperand(1)); // Promote the condition.
+
+ // The top bits of the promoted condition are not necessarily zero, ensure
+ // that the value is properly zero extended.
+ if (!DAG.MaskedValueIsZero(Tmp2,
+ MVT::getIntVTBitMask(Tmp2.getValueType())^1))
+ Tmp2 = DAG.getZeroExtendInReg(Tmp2, MVT::i1);
+ break;
+ }
+
+ // Basic block destination (Op#2) is always legal.
+ Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Node->getOperand(2));
+
+ switch (TLI.getOperationAction(ISD::BRCOND, MVT::Other)) {
+ default: assert(0 && "This action is not supported yet!");
+ case TargetLowering::Legal: break;
+ case TargetLowering::Custom:
+ Tmp1 = TLI.LowerOperation(Result, DAG);
+ if (Tmp1.Val) Result = Tmp1;
+ break;
+ case TargetLowering::Expand:
+ // Expand brcond's setcc into its constituent parts and create a BR_CC
+ // Node.
+ if (Tmp2.getOpcode() == ISD::SETCC) {
+ Result = DAG.getNode(ISD::BR_CC, MVT::Other, Tmp1, Tmp2.getOperand(2),
+ Tmp2.getOperand(0), Tmp2.getOperand(1),
+ Node->getOperand(2));
+ } else {
+ Result = DAG.getNode(ISD::BR_CC, MVT::Other, Tmp1,
+ DAG.getCondCode(ISD::SETNE), Tmp2,
+ DAG.getConstant(0, Tmp2.getValueType()),
+ Node->getOperand(2));
+ }
+ break;
+ }
+ break;
+ case ISD::BR_CC:
+ Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the chain.
+ // Ensure that libcalls are emitted before a branch.
+ Tmp1 = DAG.getNode(ISD::TokenFactor, MVT::Other, Tmp1, LastCALLSEQ_END);
+ Tmp1 = LegalizeOp(Tmp1);
+ Tmp2 = Node->getOperand(2); // LHS
+ Tmp3 = Node->getOperand(3); // RHS
+ Tmp4 = Node->getOperand(1); // CC
+
+ LegalizeSetCCOperands(Tmp2, Tmp3, Tmp4);
+ LastCALLSEQ_END = DAG.getEntryNode();
+
+ // If we didn't get both a LHS and RHS back from LegalizeSetCCOperands,
+ // the LHS is a legal SETCC itself. In this case, we need to compare
+ // the result against zero to select between true and false values.
+ if (Tmp3.Val == 0) {
+ Tmp3 = DAG.getConstant(0, Tmp2.getValueType());
+ Tmp4 = DAG.getCondCode(ISD::SETNE);
+ }
+
+ Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp4, Tmp2, Tmp3,
+ Node->getOperand(4));
+
+ switch (TLI.getOperationAction(ISD::BR_CC, Tmp3.getValueType())) {
+ default: assert(0 && "Unexpected action for BR_CC!");
+ case TargetLowering::Legal: break;
+ case TargetLowering::Custom:
+ Tmp4 = TLI.LowerOperation(Result, DAG);
+ if (Tmp4.Val) Result = Tmp4;
+ break;
+ }
+ break;
+ case ISD::LOAD: {
+ LoadSDNode *LD = cast<LoadSDNode>(Node);
+ Tmp1 = LegalizeOp(LD->getChain()); // Legalize the chain.
+ Tmp2 = LegalizeOp(LD->getBasePtr()); // Legalize the base pointer.
+
+ ISD::LoadExtType ExtType = LD->getExtensionType();
+ if (ExtType == ISD::NON_EXTLOAD) {
+ MVT::ValueType VT = Node->getValueType(0);
+ Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, LD->getOffset());
+ Tmp3 = Result.getValue(0);
+ Tmp4 = Result.getValue(1);
+
+ switch (TLI.getOperationAction(Node->getOpcode(), VT)) {
+ default: assert(0 && "This action is not supported yet!");
+ case TargetLowering::Legal: break;
+ case TargetLowering::Custom:
+ Tmp1 = TLI.LowerOperation(Tmp3, DAG);
+ if (Tmp1.Val) {
+ Tmp3 = LegalizeOp(Tmp1);
+ Tmp4 = LegalizeOp(Tmp1.getValue(1));
+ }
+ break;
+ case TargetLowering::Promote: {
+ // Only promote a load of vector type to another.
+ assert(MVT::isVector(VT) && "Cannot promote this load!");
+ // Change base type to a different vector type.
+ MVT::ValueType NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VT);
+
+ Tmp1 = DAG.getLoad(NVT, Tmp1, Tmp2, LD->getSrcValue(),
+ LD->getSrcValueOffset(),
+ LD->isVolatile(), LD->getAlignment());
+ Tmp3 = LegalizeOp(DAG.getNode(ISD::BIT_CONVERT, VT, Tmp1));
+ Tmp4 = LegalizeOp(Tmp1.getValue(1));
+ break;
+ }
+ }
+ // Since loads produce two values, make sure to remember that we
+ // legalized both of them.
+ AddLegalizedOperand(SDOperand(Node, 0), Tmp3);
+ AddLegalizedOperand(SDOperand(Node, 1), Tmp4);
+ return Op.ResNo ? Tmp4 : Tmp3;
+ } else {
+ MVT::ValueType SrcVT = LD->getLoadedVT();
+ switch (TLI.getLoadXAction(ExtType, SrcVT)) {
+ default: assert(0 && "This action is not supported yet!");
+ case TargetLowering::Promote:
+ assert(SrcVT == MVT::i1 &&
+ "Can only promote extending LOAD from i1 -> i8!");
+ Result = DAG.getExtLoad(ExtType, Node->getValueType(0), Tmp1, Tmp2,
+ LD->getSrcValue(), LD->getSrcValueOffset(),
+ MVT::i8, LD->isVolatile(), LD->getAlignment());
+ Tmp1 = Result.getValue(0);
+ Tmp2 = Result.getValue(1);
+ break;
+ case TargetLowering::Custom:
+ isCustom = true;
+ // FALLTHROUGH
+ case TargetLowering::Legal:
+ Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, LD->getOffset());
+ Tmp1 = Result.getValue(0);
+ Tmp2 = Result.getValue(1);
+
+ if (isCustom) {
+ Tmp3 = TLI.LowerOperation(Result, DAG);
+ if (Tmp3.Val) {
+ Tmp1 = LegalizeOp(Tmp3);
+ Tmp2 = LegalizeOp(Tmp3.getValue(1));
+ }
+ }
+ break;
+ case TargetLowering::Expand:
+ // f64 = EXTLOAD f32 should expand to LOAD, FP_EXTEND
+ if (SrcVT == MVT::f32 && Node->getValueType(0) == MVT::f64) {
+ SDOperand Load = DAG.getLoad(SrcVT, Tmp1, Tmp2, LD->getSrcValue(),
+ LD->getSrcValueOffset(),
+ LD->isVolatile(), LD->getAlignment());
+ Result = DAG.getNode(ISD::FP_EXTEND, Node->getValueType(0), Load);
+ Tmp1 = LegalizeOp(Result); // Relegalize new nodes.
+ Tmp2 = LegalizeOp(Load.getValue(1));
+ break;
+ }
+ assert(ExtType != ISD::EXTLOAD &&"EXTLOAD should always be supported!");
+ // Turn the unsupported load into an EXTLOAD followed by an explicit
+ // zero/sign extend inreg.
+ Result = DAG.getExtLoad(ISD::EXTLOAD, Node->getValueType(0),
+ Tmp1, Tmp2, LD->getSrcValue(),
+ LD->getSrcValueOffset(), SrcVT,
+ LD->isVolatile(), LD->getAlignment());
+ SDOperand ValRes;
+ if (ExtType == ISD::SEXTLOAD)
+ ValRes = DAG.getNode(ISD::SIGN_EXTEND_INREG, Result.getValueType(),
+ Result, DAG.getValueType(SrcVT));
+ else
+ ValRes = DAG.getZeroExtendInReg(Result, SrcVT);
+ Tmp1 = LegalizeOp(ValRes); // Relegalize new nodes.
+ Tmp2 = LegalizeOp(Result.getValue(1)); // Relegalize new nodes.
+ break;
+ }
+ // Since loads produce two values, make sure to remember that we legalized
+ // both of them.
+ AddLegalizedOperand(SDOperand(Node, 0), Tmp1);
+ AddLegalizedOperand(SDOperand(Node, 1), Tmp2);
+ return Op.ResNo ? Tmp2 : Tmp1;
+ }
+ }
+ case ISD::EXTRACT_ELEMENT: {
+ MVT::ValueType OpTy = Node->getOperand(0).getValueType();
+ switch (getTypeAction(OpTy)) {
+ default: assert(0 && "EXTRACT_ELEMENT action for type unimplemented!");
+ case Legal:
+ if (cast<ConstantSDNode>(Node->getOperand(1))->getValue()) {
+ // 1 -> Hi
+ Result = DAG.getNode(ISD::SRL, OpTy, Node->getOperand(0),
+ DAG.getConstant(MVT::getSizeInBits(OpTy)/2,
+ TLI.getShiftAmountTy()));
+ Result = DAG.getNode(ISD::TRUNCATE, Node->getValueType(0), Result);
+ } else {
+ // 0 -> Lo
+ Result = DAG.getNode(ISD::TRUNCATE, Node->getValueType(0),
+ Node->getOperand(0));
+ }
+ break;
+ case Expand:
+ // Get both the low and high parts.
+ ExpandOp(Node->getOperand(0), Tmp1, Tmp2);
+ if (cast<ConstantSDNode>(Node->getOperand(1))->getValue())
+ Result = Tmp2; // 1 -> Hi
+ else
+ Result = Tmp1; // 0 -> Lo
+ break;
+ }
+ break;
+ }
+
+ case ISD::CopyToReg:
+ Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the chain.
+
+ assert(isTypeLegal(Node->getOperand(2).getValueType()) &&
+ "Register type must be legal!");
+ // Legalize the incoming value (must be a legal type).
+ Tmp2 = LegalizeOp(Node->getOperand(2));
+ if (Node->getNumValues() == 1) {
+ Result = DAG.UpdateNodeOperands(Result, Tmp1, Node->getOperand(1), Tmp2);
+ } else {
+ assert(Node->getNumValues() == 2 && "Unknown CopyToReg");
+ if (Node->getNumOperands() == 4) {
+ Tmp3 = LegalizeOp(Node->getOperand(3));
+ Result = DAG.UpdateNodeOperands(Result, Tmp1, Node->getOperand(1), Tmp2,
+ Tmp3);
+ } else {
+ Result = DAG.UpdateNodeOperands(Result, Tmp1, Node->getOperand(1),Tmp2);
+ }
+
+ // Since this produces two values, make sure to remember that we legalized
+ // both of them.
+ AddLegalizedOperand(SDOperand(Node, 0), Result.getValue(0));
+ AddLegalizedOperand(SDOperand(Node, 1), Result.getValue(1));
+ return Result;
+ }
+ break;
+
+ case ISD::RET:
+ Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the chain.
+
+ // Ensure that libcalls are emitted before a return.
+ Tmp1 = DAG.getNode(ISD::TokenFactor, MVT::Other, Tmp1, LastCALLSEQ_END);
+ Tmp1 = LegalizeOp(Tmp1);
+ LastCALLSEQ_END = DAG.getEntryNode();
+
+ switch (Node->getNumOperands()) {
+ case 3: // ret val
+ Tmp2 = Node->getOperand(1);
+ Tmp3 = Node->getOperand(2); // Signness
+ switch (getTypeAction(Tmp2.getValueType())) {
+ case Legal:
+ Result = DAG.UpdateNodeOperands(Result, Tmp1, LegalizeOp(Tmp2), Tmp3);
+ break;
+ case Expand:
+ if (!MVT::isVector(Tmp2.getValueType())) {
+ SDOperand Lo, Hi;
+ ExpandOp(Tmp2, Lo, Hi);
+
+ // Big endian systems want the hi reg first.
+ if (!TLI.isLittleEndian())
+ std::swap(Lo, Hi);
+
+ if (Hi.Val)
+ Result = DAG.getNode(ISD::RET, MVT::Other, Tmp1, Lo, Tmp3, Hi,Tmp3);
+ else
+ Result = DAG.getNode(ISD::RET, MVT::Other, Tmp1, Lo, Tmp3);
+ Result = LegalizeOp(Result);
+ } else {
+ SDNode *InVal = Tmp2.Val;
+ unsigned NumElems = MVT::getVectorNumElements(InVal->getValueType(0));
+ MVT::ValueType EVT = MVT::getVectorElementType(InVal->getValueType(0));
+
+ // Figure out if there is a simple type corresponding to this Vector
+ // type. If so, convert to the vector type.
+ MVT::ValueType TVT = MVT::getVectorType(EVT, NumElems);
+ if (TLI.isTypeLegal(TVT)) {
+ // Turn this into a return of the vector type.
+ Tmp2 = LegalizeOp(Tmp2);
+ Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3);
+ } else if (NumElems == 1) {
+ // Turn this into a return of the scalar type.
+ Tmp2 = ScalarizeVectorOp(Tmp2);
+ Tmp2 = LegalizeOp(Tmp2);
+ Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3);
+
+ // FIXME: Returns of gcc generic vectors smaller than a legal type
+ // should be returned in integer registers!
+
+ // The scalarized value type may not be legal, e.g. it might require
+ // promotion or expansion. Relegalize the return.
+ Result = LegalizeOp(Result);
+ } else {
+ // FIXME: Returns of gcc generic vectors larger than a legal vector
+ // type should be returned by reference!
+ SDOperand Lo, Hi;
+ SplitVectorOp(Tmp2, Lo, Hi);
+ Result = DAG.getNode(ISD::RET, MVT::Other, Tmp1, Lo, Tmp3, Hi,Tmp3);
+ Result = LegalizeOp(Result);
+ }
+ }
+ break;
+ case Promote:
+ Tmp2 = PromoteOp(Node->getOperand(1));
+ Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3);
+ Result = LegalizeOp(Result);
+ break;
+ }
+ break;
+ case 1: // ret void
+ Result = DAG.UpdateNodeOperands(Result, Tmp1);
+ break;
+ default: { // ret <values>
+ SmallVector<SDOperand, 8> NewValues;
+ NewValues.push_back(Tmp1);
+ for (unsigned i = 1, e = Node->getNumOperands(); i < e; i += 2)
+ switch (getTypeAction(Node->getOperand(i).getValueType())) {
+ case Legal:
+ NewValues.push_back(LegalizeOp(Node->getOperand(i)));
+ NewValues.push_back(Node->getOperand(i+1));
+ break;
+ case Expand: {
+ SDOperand Lo, Hi;
+ assert(!MVT::isExtendedVT(Node->getOperand(i).getValueType()) &&
+ "FIXME: TODO: implement returning non-legal vector types!");
+ ExpandOp(Node->getOperand(i), Lo, Hi);
+ NewValues.push_back(Lo);
+ NewValues.push_back(Node->getOperand(i+1));
+ if (Hi.Val) {
+ NewValues.push_back(Hi);
+ NewValues.push_back(Node->getOperand(i+1));
+ }
+ break;
+ }
+ case Promote:
+ assert(0 && "Can't promote multiple return value yet!");
+ }
+
+ if (NewValues.size() == Node->getNumOperands())
+ Result = DAG.UpdateNodeOperands(Result, &NewValues[0],NewValues.size());
+ else
+ Result = DAG.getNode(ISD::RET, MVT::Other,
+ &NewValues[0], NewValues.size());
+ break;
+ }
+ }
+
+ if (Result.getOpcode() == ISD::RET) {
+ switch (TLI.getOperationAction(Result.getOpcode(), MVT::Other)) {
+ default: assert(0 && "This action is not supported yet!");
+ case TargetLowering::Legal: break;
+ case TargetLowering::Custom:
+ Tmp1 = TLI.LowerOperation(Result, DAG);
+ if (Tmp1.Val) Result = Tmp1;
+ break;
+ }
+ }
+ break;
+ case ISD::STORE: {
+ StoreSDNode *ST = cast<StoreSDNode>(Node);
+ Tmp1 = LegalizeOp(ST->getChain()); // Legalize the chain.
+ Tmp2 = LegalizeOp(ST->getBasePtr()); // Legalize the pointer.
+ int SVOffset = ST->getSrcValueOffset();
+ unsigned Alignment = ST->getAlignment();
+ bool isVolatile = ST->isVolatile();
+
+ if (!ST->isTruncatingStore()) {
+ // Turn 'store float 1.0, Ptr' -> 'store int 0x12345678, Ptr'
+ // FIXME: We shouldn't do this for TargetConstantFP's.
+ // FIXME: move this to the DAG Combiner! Note that we can't regress due
+ // to phase ordering between legalized code and the dag combiner. This
+ // probably means that we need to integrate dag combiner and legalizer
+ // together.
+ if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(ST->getValue())) {
+ if (CFP->getValueType(0) == MVT::f32) {
+ Tmp3 = DAG.getConstant(FloatToBits(CFP->getValue()), MVT::i32);
+ } else {
+ assert(CFP->getValueType(0) == MVT::f64 && "Unknown FP type!");
+ Tmp3 = DAG.getConstant(DoubleToBits(CFP->getValue()), MVT::i64);
+ }
+ Result = DAG.getStore(Tmp1, Tmp3, Tmp2, ST->getSrcValue(),
+ SVOffset, isVolatile, Alignment);
+ break;
+ }
+
+ switch (getTypeAction(ST->getStoredVT())) {
+ case Legal: {
+ Tmp3 = LegalizeOp(ST->getValue());
+ Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp3, Tmp2,
+ ST->getOffset());
+
+ MVT::ValueType VT = Tmp3.getValueType();
+ switch (TLI.getOperationAction(ISD::STORE, VT)) {
+ default: assert(0 && "This action is not supported yet!");
+ case TargetLowering::Legal: break;
+ case TargetLowering::Custom:
+ Tmp1 = TLI.LowerOperation(Result, DAG);
+ if (Tmp1.Val) Result = Tmp1;
+ break;
+ case TargetLowering::Promote:
+ assert(MVT::isVector(VT) && "Unknown legal promote case!");
+ Tmp3 = DAG.getNode(ISD::BIT_CONVERT,
+ TLI.getTypeToPromoteTo(ISD::STORE, VT), Tmp3);
+ Result = DAG.getStore(Tmp1, Tmp3, Tmp2,
+ ST->getSrcValue(), SVOffset, isVolatile,
+ Alignment);
+ break;
+ }
+ break;
+ }
+ case Promote:
+ // Truncate the value and store the result.
+ Tmp3 = PromoteOp(ST->getValue());
+ Result = DAG.getTruncStore(Tmp1, Tmp3, Tmp2, ST->getSrcValue(),
+ SVOffset, ST->getStoredVT(),
+ isVolatile, Alignment);
+ break;
+
+ case Expand:
+ unsigned IncrementSize = 0;
+ SDOperand Lo, Hi;
+
+ // If this is a vector type, then we have to calculate the increment as
+ // the product of the element size in bytes, and the number of elements
+ // in the high half of the vector.
+ if (MVT::isVector(ST->getValue().getValueType())) {
+ SDNode *InVal = ST->getValue().Val;
+ unsigned NumElems = MVT::getVectorNumElements(InVal->getValueType(0));
+ MVT::ValueType EVT = MVT::getVectorElementType(InVal->getValueType(0));
+
+ // Figure out if there is a simple type corresponding to this Vector
+ // type. If so, convert to the vector type.
+ MVT::ValueType TVT = MVT::getVectorType(EVT, NumElems);
+ if (TLI.isTypeLegal(TVT)) {
+ // Turn this into a normal store of the vector type.
+ Tmp3 = LegalizeOp(Node->getOperand(1));
+ Result = DAG.getStore(Tmp1, Tmp3, Tmp2, ST->getSrcValue(),
+ SVOffset, isVolatile, Alignment);
+ Result = LegalizeOp(Result);
+ break;
+ } else if (NumElems == 1) {
+ // Turn this into a normal store of the scalar type.
+ Tmp3 = ScalarizeVectorOp(Node->getOperand(1));
+ Result = DAG.getStore(Tmp1, Tmp3, Tmp2, ST->getSrcValue(),
+ SVOffset, isVolatile, Alignment);
+ // The scalarized value type may not be legal, e.g. it might require
+ // promotion or expansion. Relegalize the scalar store.
+ Result = LegalizeOp(Result);
+ break;
+ } else {
+ SplitVectorOp(Node->getOperand(1), Lo, Hi);
+ IncrementSize = NumElems/2 * MVT::getSizeInBits(EVT)/8;
+ }
+ } else {
+ ExpandOp(Node->getOperand(1), Lo, Hi);
+ IncrementSize = Hi.Val ? MVT::getSizeInBits(Hi.getValueType())/8 : 0;
+
+ if (!TLI.isLittleEndian())
+ std::swap(Lo, Hi);
+ }
+
+ Lo = DAG.getStore(Tmp1, Lo, Tmp2, ST->getSrcValue(),
+ SVOffset, isVolatile, Alignment);
+
+ if (Hi.Val == NULL) {
+ // Must be int <-> float one-to-one expansion.
+ Result = Lo;
+ break;
+ }
+
+ Tmp2 = DAG.getNode(ISD::ADD, Tmp2.getValueType(), Tmp2,
+ getIntPtrConstant(IncrementSize));
+ assert(isTypeLegal(Tmp2.getValueType()) &&
+ "Pointers must be legal!");
+ SVOffset += IncrementSize;
+ if (Alignment > IncrementSize)
+ Alignment = IncrementSize;
+ Hi = DAG.getStore(Tmp1, Hi, Tmp2, ST->getSrcValue(),
+ SVOffset, isVolatile, Alignment);
+ Result = DAG.getNode(ISD::TokenFactor, MVT::Other, Lo, Hi);
+ break;
+ }
+ } else {
+ // Truncating store
+ assert(isTypeLegal(ST->getValue().getValueType()) &&
+ "Cannot handle illegal TRUNCSTORE yet!");
+ Tmp3 = LegalizeOp(ST->getValue());
+
+ // The only promote case we handle is TRUNCSTORE:i1 X into
+ // -> TRUNCSTORE:i8 (and X, 1)
+ if (ST->getStoredVT() == MVT::i1 &&
+ TLI.getStoreXAction(MVT::i1) == TargetLowering::Promote) {
+ // Promote the bool to a mask then store.
+ Tmp3 = DAG.getNode(ISD::AND, Tmp3.getValueType(), Tmp3,
+ DAG.getConstant(1, Tmp3.getValueType()));
+ Result = DAG.getTruncStore(Tmp1, Tmp3, Tmp2, ST->getSrcValue(),
+ SVOffset, MVT::i8,
+ isVolatile, Alignment);
+ } else if (Tmp1 != ST->getChain() || Tmp3 != ST->getValue() ||
+ Tmp2 != ST->getBasePtr()) {
+ Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp3, Tmp2,
+ ST->getOffset());
+ }
+
+ MVT::ValueType StVT = cast<StoreSDNode>(Result.Val)->getStoredVT();
+ switch (TLI.getStoreXAction(StVT)) {
+ default: assert(0 && "This action is not supported yet!");
+ case TargetLowering::Legal: break;
+ case TargetLowering::Custom:
+ Tmp1 = TLI.LowerOperation(Result, DAG);
+ if (Tmp1.Val) Result = Tmp1;
+ break;
+ }
+ }
+ break;
+ }
+ case ISD::PCMARKER:
+ Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the chain.
+ Result = DAG.UpdateNodeOperands(Result, Tmp1, Node->getOperand(1));
+ break;
+ case ISD::STACKSAVE:
+ Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the chain.
+ Result = DAG.UpdateNodeOperands(Result, Tmp1);
+ Tmp1 = Result.getValue(0);
+ Tmp2 = Result.getValue(1);
+
+ switch (TLI.getOperationAction(ISD::STACKSAVE, MVT::Other)) {
+ default: assert(0 && "This action is not supported yet!");
+ case TargetLowering::Legal: break;
+ case TargetLowering::Custom:
+ Tmp3 = TLI.LowerOperation(Result, DAG);
+ if (Tmp3.Val) {
+ Tmp1 = LegalizeOp(Tmp3);
+ Tmp2 = LegalizeOp(Tmp3.getValue(1));
+ }
+ break;
+ case TargetLowering::Expand:
+ // Expand to CopyFromReg if the target set
+ // StackPointerRegisterToSaveRestore.
+ if (unsigned SP = TLI.getStackPointerRegisterToSaveRestore()) {
+ Tmp1 = DAG.getCopyFromReg(Result.getOperand(0), SP,
+ Node->getValueType(0));
+ Tmp2 = Tmp1.getValue(1);
+ } else {
+ Tmp1 = DAG.getNode(ISD::UNDEF, Node->getValueType(0));
+ Tmp2 = Node->getOperand(0);
+ }
+ break;
+ }
+
+ // Since stacksave produce two values, make sure to remember that we
+ // legalized both of them.
+ AddLegalizedOperand(SDOperand(Node, 0), Tmp1);
+ AddLegalizedOperand(SDOperand(Node, 1), Tmp2);
+ return Op.ResNo ? Tmp2 : Tmp1;
+
+ case ISD::STACKRESTORE:
+ Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the chain.
+ Tmp2 = LegalizeOp(Node->getOperand(1)); // Legalize the pointer.
+ Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2);
+
+ switch (TLI.getOperationAction(ISD::STACKRESTORE, MVT::Other)) {
+ default: assert(0 && "This action is not supported yet!");
+ case TargetLowering::Legal: break;
+ case TargetLowering::Custom:
+ Tmp1 = TLI.LowerOperation(Result, DAG);
+ if (Tmp1.Val) Result = Tmp1;
+ break;
+ case TargetLowering::Expand:
+ // Expand to CopyToReg if the target set
+ // StackPointerRegisterToSaveRestore.
+ if (unsigned SP = TLI.getStackPointerRegisterToSaveRestore()) {
+ Result = DAG.getCopyToReg(Tmp1, SP, Tmp2);
+ } else {
+ Result = Tmp1;
+ }
+ break;
+ }
+ break;
+
+ case ISD::READCYCLECOUNTER:
+ Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the chain
+ Result = DAG.UpdateNodeOperands(Result, Tmp1);
+ switch (TLI.getOperationAction(ISD::READCYCLECOUNTER,
+ Node->getValueType(0))) {
+ default: assert(0 && "This action is not supported yet!");
+ case TargetLowering::Legal:
+ Tmp1 = Result.getValue(0);
+ Tmp2 = Result.getValue(1);
+ break;
+ case TargetLowering::Custom:
+ Result = TLI.LowerOperation(Result, DAG);
+ Tmp1 = LegalizeOp(Result.getValue(0));
+ Tmp2 = LegalizeOp(Result.getValue(1));
+ break;
+ }
+
+ // Since rdcc produce two values, make sure to remember that we legalized
+ // both of them.
+ AddLegalizedOperand(SDOperand(Node, 0), Tmp1);
+ AddLegalizedOperand(SDOperand(Node, 1), Tmp2);
+ return Result;
+
+ case ISD::SELECT:
+ switch (getTypeAction(Node->getOperand(0).getValueType())) {
+ case Expand: assert(0 && "It's impossible to expand bools");
+ case Legal:
+ Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the condition.
+ break;
+ case Promote:
+ Tmp1 = PromoteOp(Node->getOperand(0)); // Promote the condition.
+ // Make sure the condition is either zero or one.
+ if (!DAG.MaskedValueIsZero(Tmp1,
+ MVT::getIntVTBitMask(Tmp1.getValueType())^1))
+ Tmp1 = DAG.getZeroExtendInReg(Tmp1, MVT::i1);
+ break;
+ }
+ Tmp2 = LegalizeOp(Node->getOperand(1)); // TrueVal
+ Tmp3 = LegalizeOp(Node->getOperand(2)); // FalseVal
+
+ Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3);
+
+ switch (TLI.getOperationAction(ISD::SELECT, Tmp2.getValueType())) {
+ default: assert(0 && "This action is not supported yet!");
+ case TargetLowering::Legal: break;
+ case TargetLowering::Custom: {
+ Tmp1 = TLI.LowerOperation(Result, DAG);
+ if (Tmp1.Val) Result = Tmp1;
+ break;
+ }
+ case TargetLowering::Expand:
+ if (Tmp1.getOpcode() == ISD::SETCC) {
+ Result = DAG.getSelectCC(Tmp1.getOperand(0), Tmp1.getOperand(1),
+ Tmp2, Tmp3,
+ cast<CondCodeSDNode>(Tmp1.getOperand(2))->get());
+ } else {
+ Result = DAG.getSelectCC(Tmp1,
+ DAG.getConstant(0, Tmp1.getValueType()),
+ Tmp2, Tmp3, ISD::SETNE);
+ }
+ break;
+ case TargetLowering::Promote: {
+ MVT::ValueType NVT =
+ TLI.getTypeToPromoteTo(ISD::SELECT, Tmp2.getValueType());
+ unsigned ExtOp, TruncOp;
+ if (MVT::isVector(Tmp2.getValueType())) {
+ ExtOp = ISD::BIT_CONVERT;
+ TruncOp = ISD::BIT_CONVERT;
+ } else if (MVT::isInteger(Tmp2.getValueType())) {
+ ExtOp = ISD::ANY_EXTEND;
+ TruncOp = ISD::TRUNCATE;
+ } else {
+ ExtOp = ISD::FP_EXTEND;
+ TruncOp = ISD::FP_ROUND;
+ }
+ // Promote each of the values to the new type.
+ Tmp2 = DAG.getNode(ExtOp, NVT, Tmp2);
+ Tmp3 = DAG.getNode(ExtOp, NVT, Tmp3);
+ // Perform the larger operation, then round down.
+ Result = DAG.getNode(ISD::SELECT, NVT, Tmp1, Tmp2,Tmp3);
+ Result = DAG.getNode(TruncOp, Node->getValueType(0), Result);
+ break;
+ }
+ }
+ break;
+ case ISD::SELECT_CC: {
+ Tmp1 = Node->getOperand(0); // LHS
+ Tmp2 = Node->getOperand(1); // RHS
+ Tmp3 = LegalizeOp(Node->getOperand(2)); // True
+ Tmp4 = LegalizeOp(Node->getOperand(3)); // False
+ SDOperand CC = Node->getOperand(4);
+
+ LegalizeSetCCOperands(Tmp1, Tmp2, CC);
+
+ // If we didn't get both a LHS and RHS back from LegalizeSetCCOperands,
+ // the LHS is a legal SETCC itself. In this case, we need to compare
+ // the result against zero to select between true and false values.
+ if (Tmp2.Val == 0) {
+ Tmp2 = DAG.getConstant(0, Tmp1.getValueType());
+ CC = DAG.getCondCode(ISD::SETNE);
+ }
+ Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3, Tmp4, CC);
+
+ // Everything is legal, see if we should expand this op or something.
+ switch (TLI.getOperationAction(ISD::SELECT_CC, Tmp3.getValueType())) {
+ default: assert(0 && "This action is not supported yet!");
+ case TargetLowering::Legal: break;
+ case TargetLowering::Custom:
+ Tmp1 = TLI.LowerOperation(Result, DAG);
+ if (Tmp1.Val) Result = Tmp1;
+ break;
+ }
+ break;
+ }
+ case ISD::SETCC:
+ Tmp1 = Node->getOperand(0);
+ Tmp2 = Node->getOperand(1);
+ Tmp3 = Node->getOperand(2);
+ LegalizeSetCCOperands(Tmp1, Tmp2, Tmp3);
+
+ // If we had to Expand the SetCC operands into a SELECT node, then it may
+ // not always be possible to return a true LHS & RHS. In this case, just
+ // return the value we legalized, returned in the LHS
+ if (Tmp2.Val == 0) {
+ Result = Tmp1;
+ break;
+ }
+
+ switch (TLI.getOperationAction(ISD::SETCC, Tmp1.getValueType())) {
+ default: assert(0 && "Cannot handle this action for SETCC yet!");
+ case TargetLowering::Custom:
+ isCustom = true;
+ // FALLTHROUGH.
+ case TargetLowering::Legal:
+ Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3);
+ if (isCustom) {
+ Tmp4 = TLI.LowerOperation(Result, DAG);
+ if (Tmp4.Val) Result = Tmp4;
+ }
+ break;
+ case TargetLowering::Promote: {
+ // First step, figure out the appropriate operation to use.
+ // Allow SETCC to not be supported for all legal data types
+ // Mostly this targets FP
+ MVT::ValueType NewInTy = Node->getOperand(0).getValueType();
+ MVT::ValueType OldVT = NewInTy; OldVT = OldVT;
+
+ // Scan for the appropriate larger type to use.
+ while (1) {
+ NewInTy = (MVT::ValueType)(NewInTy+1);
+
+ assert(MVT::isInteger(NewInTy) == MVT::isInteger(OldVT) &&
+ "Fell off of the edge of the integer world");
+ assert(MVT::isFloatingPoint(NewInTy) == MVT::isFloatingPoint(OldVT) &&
+ "Fell off of the edge of the floating point world");
+
+ // If the target supports SETCC of this type, use it.
+ if (TLI.isOperationLegal(ISD::SETCC, NewInTy))
+ break;
+ }
+ if (MVT::isInteger(NewInTy))
+ assert(0 && "Cannot promote Legal Integer SETCC yet");
+ else {
+ Tmp1 = DAG.getNode(ISD::FP_EXTEND, NewInTy, Tmp1);
+ Tmp2 = DAG.getNode(ISD::FP_EXTEND, NewInTy, Tmp2);
+ }
+ Tmp1 = LegalizeOp(Tmp1);
+ Tmp2 = LegalizeOp(Tmp2);
+ Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3);
+ Result = LegalizeOp(Result);
+ break;
+ }
+ case TargetLowering::Expand:
+ // Expand a setcc node into a select_cc of the same condition, lhs, and
+ // rhs that selects between const 1 (true) and const 0 (false).
+ MVT::ValueType VT = Node->getValueType(0);
+ Result = DAG.getNode(ISD::SELECT_CC, VT, Tmp1, Tmp2,
+ DAG.getConstant(1, VT), DAG.getConstant(0, VT),
+ Tmp3);
+ break;
+ }
+ break;
+ case ISD::MEMSET:
+ case ISD::MEMCPY:
+ case ISD::MEMMOVE: {
+ Tmp1 = LegalizeOp(Node->getOperand(0)); // Chain
+ Tmp2 = LegalizeOp(Node->getOperand(1)); // Pointer
+
+ if (Node->getOpcode() == ISD::MEMSET) { // memset = ubyte
+ switch (getTypeAction(Node->getOperand(2).getValueType())) {
+ case Expand: assert(0 && "Cannot expand a byte!");
+ case Legal:
+ Tmp3 = LegalizeOp(Node->getOperand(2));
+ break;
+ case Promote:
+ Tmp3 = PromoteOp(Node->getOperand(2));
+ break;
+ }
+ } else {
+ Tmp3 = LegalizeOp(Node->getOperand(2)); // memcpy/move = pointer,
+ }
+
+ SDOperand Tmp4;
+ switch (getTypeAction(Node->getOperand(3).getValueType())) {
+ case Expand: {
+ // Length is too big, just take the lo-part of the length.
+ SDOperand HiPart;
+ ExpandOp(Node->getOperand(3), Tmp4, HiPart);
+ break;
+ }
+ case Legal:
+ Tmp4 = LegalizeOp(Node->getOperand(3));
+ break;
+ case Promote:
+ Tmp4 = PromoteOp(Node->getOperand(3));
+ break;
+ }
+
+ SDOperand Tmp5;
+ switch (getTypeAction(Node->getOperand(4).getValueType())) { // uint
+ case Expand: assert(0 && "Cannot expand this yet!");
+ case Legal:
+ Tmp5 = LegalizeOp(Node->getOperand(4));
+ break;
+ case Promote:
+ Tmp5 = PromoteOp(Node->getOperand(4));
+ break;
+ }
+
+ switch (TLI.getOperationAction(Node->getOpcode(), MVT::Other)) {
+ default: assert(0 && "This action not implemented for this operation!");
+ case TargetLowering::Custom:
+ isCustom = true;
+ // FALLTHROUGH
+ case TargetLowering::Legal:
+ Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3, Tmp4, Tmp5);
+ if (isCustom) {
+ Tmp1 = TLI.LowerOperation(Result, DAG);
+ if (Tmp1.Val) Result = Tmp1;
+ }
+ break;
+ case TargetLowering::Expand: {
+ // Otherwise, the target does not support this operation. Lower the
+ // operation to an explicit libcall as appropriate.
+ MVT::ValueType IntPtr = TLI.getPointerTy();
+ const Type *IntPtrTy = TLI.getTargetData()->getIntPtrType();
+ TargetLowering::ArgListTy Args;
+ TargetLowering::ArgListEntry Entry;
+
+ const char *FnName = 0;
+ if (Node->getOpcode() == ISD::MEMSET) {
+ Entry.Node = Tmp2; Entry.Ty = IntPtrTy;
+ Args.push_back(Entry);
+ // Extend the (previously legalized) ubyte argument to be an int value
+ // for the call.
+ if (Tmp3.getValueType() > MVT::i32)
+ Tmp3 = DAG.getNode(ISD::TRUNCATE, MVT::i32, Tmp3);
+ else
+ Tmp3 = DAG.getNode(ISD::ZERO_EXTEND, MVT::i32, Tmp3);
+ Entry.Node = Tmp3; Entry.Ty = Type::Int32Ty; Entry.isSExt = true;
+ Args.push_back(Entry);
+ Entry.Node = Tmp4; Entry.Ty = IntPtrTy; Entry.isSExt = false;
+ Args.push_back(Entry);
+
+ FnName = "memset";
+ } else if (Node->getOpcode() == ISD::MEMCPY ||
+ Node->getOpcode() == ISD::MEMMOVE) {
+ Entry.Ty = IntPtrTy;
+ Entry.Node = Tmp2; Args.push_back(Entry);
+ Entry.Node = Tmp3; Args.push_back(Entry);
+ Entry.Node = Tmp4; Args.push_back(Entry);
+ FnName = Node->getOpcode() == ISD::MEMMOVE ? "memmove" : "memcpy";
+ } else {
+ assert(0 && "Unknown op!");
+ }
+
+ std::pair<SDOperand,SDOperand> CallResult =
+ TLI.LowerCallTo(Tmp1, Type::VoidTy, false, false, CallingConv::C, false,
+ DAG.getExternalSymbol(FnName, IntPtr), Args, DAG);
+ Result = CallResult.second;
+ break;
+ }
+ }
+ break;
+ }
+
+ case ISD::SHL_PARTS:
+ case ISD::SRA_PARTS:
+ case ISD::SRL_PARTS: {
+ SmallVector<SDOperand, 8> Ops;
+ bool Changed = false;
+ for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) {
+ Ops.push_back(LegalizeOp(Node->getOperand(i)));
+ Changed |= Ops.back() != Node->getOperand(i);
+ }
+ if (Changed)
+ Result = DAG.UpdateNodeOperands(Result, &Ops[0], Ops.size());
+
+ switch (TLI.getOperationAction(Node->getOpcode(),
+ Node->getValueType(0))) {
+ default: assert(0 && "This action is not supported yet!");
+ case TargetLowering::Legal: break;
+ case TargetLowering::Custom:
+ Tmp1 = TLI.LowerOperation(Result, DAG);
+ if (Tmp1.Val) {
+ SDOperand Tmp2, RetVal(0, 0);
+ for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i) {
+ Tmp2 = LegalizeOp(Tmp1.getValue(i));
+ AddLegalizedOperand(SDOperand(Node, i), Tmp2);
+ if (i == Op.ResNo)
+ RetVal = Tmp2;
+ }
+ assert(RetVal.Val && "Illegal result number");
+ return RetVal;
+ }
+ break;
+ }
+
+ // Since these produce multiple values, make sure to remember that we
+ // legalized all of them.
+ for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i)
+ AddLegalizedOperand(SDOperand(Node, i), Result.getValue(i));
+ return Result.getValue(Op.ResNo);
+ }
+
+ // Binary operators
+ case ISD::ADD:
+ case ISD::SUB:
+ case ISD::MUL:
+ case ISD::MULHS:
+ case ISD::MULHU:
+ case ISD::UDIV:
+ case ISD::SDIV:
+ case ISD::AND:
+ case ISD::OR:
+ case ISD::XOR:
+ case ISD::SHL:
+ case ISD::SRL:
+ case ISD::SRA:
+ case ISD::FADD:
+ case ISD::FSUB:
+ case ISD::FMUL:
+ case ISD::FDIV:
+ Tmp1 = LegalizeOp(Node->getOperand(0)); // LHS
+ switch (getTypeAction(Node->getOperand(1).getValueType())) {
+ case Expand: assert(0 && "Not possible");
+ case Legal:
+ Tmp2 = LegalizeOp(Node->getOperand(1)); // Legalize the RHS.
+ break;
+ case Promote:
+ Tmp2 = PromoteOp(Node->getOperand(1)); // Promote the RHS.
+ break;
+ }
+
+ Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2);
+
+ switch (TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0))) {
+ default: assert(0 && "BinOp legalize operation not supported");
+ case TargetLowering::Legal: break;
+ case TargetLowering::Custom:
+ Tmp1 = TLI.LowerOperation(Result, DAG);
+ if (Tmp1.Val) Result = Tmp1;
+ break;
+ case TargetLowering::Expand: {
+ if (Node->getValueType(0) == MVT::i32) {
+ switch (Node->getOpcode()) {
+ default: assert(0 && "Do not know how to expand this integer BinOp!");
+ case ISD::UDIV:
+ case ISD::SDIV:
+ RTLIB::Libcall LC = Node->getOpcode() == ISD::UDIV
+ ? RTLIB::UDIV_I32 : RTLIB::SDIV_I32;
+ SDOperand Dummy;
+ bool isSigned = Node->getOpcode() == ISD::SDIV;
+ Result = ExpandLibCall(TLI.getLibcallName(LC), Node, isSigned, Dummy);
+ };
+ break;
+ }
+
+ assert(MVT::isVector(Node->getValueType(0)) &&
+ "Cannot expand this binary operator!");
+ // Expand the operation into a bunch of nasty scalar code.
+ SmallVector<SDOperand, 8> Ops;
+ MVT::ValueType EltVT = MVT::getVectorElementType(Node->getValueType(0));
+ MVT::ValueType PtrVT = TLI.getPointerTy();
+ for (unsigned i = 0, e = MVT::getVectorNumElements(Node->getValueType(0));
+ i != e; ++i) {
+ SDOperand Idx = DAG.getConstant(i, PtrVT);
+ SDOperand LHS = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, EltVT, Tmp1, Idx);
+ SDOperand RHS = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, EltVT, Tmp2, Idx);
+ Ops.push_back(DAG.getNode(Node->getOpcode(), EltVT, LHS, RHS));
+ }
+ Result = DAG.getNode(ISD::BUILD_VECTOR, Node->getValueType(0),
+ &Ops[0], Ops.size());
+ break;
+ }
+ case TargetLowering::Promote: {
+ switch (Node->getOpcode()) {
+ default: assert(0 && "Do not know how to promote this BinOp!");
+ case ISD::AND:
+ case ISD::OR:
+ case ISD::XOR: {
+ MVT::ValueType OVT = Node->getValueType(0);
+ MVT::ValueType NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), OVT);
+ assert(MVT::isVector(OVT) && "Cannot promote this BinOp!");
+ // Bit convert each of the values to the new type.
+ Tmp1 = DAG.getNode(ISD::BIT_CONVERT, NVT, Tmp1);
+ Tmp2 = DAG.getNode(ISD::BIT_CONVERT, NVT, Tmp2);
+ Result = DAG.getNode(Node->getOpcode(), NVT, Tmp1, Tmp2);
+ // Bit convert the result back the original type.
+ Result = DAG.getNode(ISD::BIT_CONVERT, OVT, Result);
+ break;
+ }
+ }
+ }
+ }
+ break;
+
+ case ISD::FCOPYSIGN: // FCOPYSIGN does not require LHS/RHS to match type!
+ Tmp1 = LegalizeOp(Node->getOperand(0)); // LHS
+ switch (getTypeAction(Node->getOperand(1).getValueType())) {
+ case Expand: assert(0 && "Not possible");
+ case Legal:
+ Tmp2 = LegalizeOp(Node->getOperand(1)); // Legalize the RHS.
+ break;
+ case Promote:
+ Tmp2 = PromoteOp(Node->getOperand(1)); // Promote the RHS.
+ break;
+ }
+
+ Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2);
+
+ switch (TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0))) {
+ default: assert(0 && "Operation not supported");
+ case TargetLowering::Custom:
+ Tmp1 = TLI.LowerOperation(Result, DAG);
+ if (Tmp1.Val) Result = Tmp1;
+ break;
+ case TargetLowering::Legal: break;
+ case TargetLowering::Expand: {
+ // If this target supports fabs/fneg natively and select is cheap,
+ // do this efficiently.
+ if (!TLI.isSelectExpensive() &&
+ TLI.getOperationAction(ISD::FABS, Tmp1.getValueType()) ==
+ TargetLowering::Legal &&
+ TLI.getOperationAction(ISD::FNEG, Tmp1.getValueType()) ==
+ TargetLowering::Legal) {
+ // Get the sign bit of the RHS.
+ MVT::ValueType IVT =
+ Tmp2.getValueType() == MVT::f32 ? MVT::i32 : MVT::i64;
+ SDOperand SignBit = DAG.getNode(ISD::BIT_CONVERT, IVT, Tmp2);
+ SignBit = DAG.getSetCC(TLI.getSetCCResultTy(),
+ SignBit, DAG.getConstant(0, IVT), ISD::SETLT);
+ // Get the absolute value of the result.
+ SDOperand AbsVal = DAG.getNode(ISD::FABS, Tmp1.getValueType(), Tmp1);
+ // Select between the nabs and abs value based on the sign bit of
+ // the input.
+ Result = DAG.getNode(ISD::SELECT, AbsVal.getValueType(), SignBit,
+ DAG.getNode(ISD::FNEG, AbsVal.getValueType(),
+ AbsVal),
+ AbsVal);
+ Result = LegalizeOp(Result);
+ break;
+ }
+
+ // Otherwise, do bitwise ops!
+ MVT::ValueType NVT =
+ Node->getValueType(0) == MVT::f32 ? MVT::i32 : MVT::i64;
+ Result = ExpandFCOPYSIGNToBitwiseOps(Node, NVT, DAG, TLI);
+ Result = DAG.getNode(ISD::BIT_CONVERT, Node->getValueType(0), Result);
+ Result = LegalizeOp(Result);
+ break;
+ }
+ }
+ break;
+
+ case ISD::ADDC:
+ case ISD::SUBC:
+ Tmp1 = LegalizeOp(Node->getOperand(0));
+ Tmp2 = LegalizeOp(Node->getOperand(1));
+ Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2);
+ // Since this produces two values, make sure to remember that we legalized
+ // both of them.
+ AddLegalizedOperand(SDOperand(Node, 0), Result.getValue(0));
+ AddLegalizedOperand(SDOperand(Node, 1), Result.getValue(1));
+ return Result;
+
+ case ISD::ADDE:
+ case ISD::SUBE:
+ Tmp1 = LegalizeOp(Node->getOperand(0));
+ Tmp2 = LegalizeOp(Node->getOperand(1));
+ Tmp3 = LegalizeOp(Node->getOperand(2));
+ Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3);
+ // Since this produces two values, make sure to remember that we legalized
+ // both of them.
+ AddLegalizedOperand(SDOperand(Node, 0), Result.getValue(0));
+ AddLegalizedOperand(SDOperand(Node, 1), Result.getValue(1));
+ return Result;
+
+ case ISD::BUILD_PAIR: {
+ MVT::ValueType PairTy = Node->getValueType(0);
+ // TODO: handle the case where the Lo and Hi operands are not of legal type
+ Tmp1 = LegalizeOp(Node->getOperand(0)); // Lo
+ Tmp2 = LegalizeOp(Node->getOperand(1)); // Hi
+ switch (TLI.getOperationAction(ISD::BUILD_PAIR, PairTy)) {
+ case TargetLowering::Promote:
+ case TargetLowering::Custom:
+ assert(0 && "Cannot promote/custom this yet!");
+ case TargetLowering::Legal:
+ if (Tmp1 != Node->getOperand(0) || Tmp2 != Node->getOperand(1))
+ Result = DAG.getNode(ISD::BUILD_PAIR, PairTy, Tmp1, Tmp2);
+ break;
+ case TargetLowering::Expand:
+ Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, PairTy, Tmp1);
+ Tmp2 = DAG.getNode(ISD::ANY_EXTEND, PairTy, Tmp2);
+ Tmp2 = DAG.getNode(ISD::SHL, PairTy, Tmp2,
+ DAG.getConstant(MVT::getSizeInBits(PairTy)/2,
+ TLI.getShiftAmountTy()));
+ Result = DAG.getNode(ISD::OR, PairTy, Tmp1, Tmp2);
+ break;
+ }
+ break;
+ }
+
+ case ISD::UREM:
+ case ISD::SREM:
+ case ISD::FREM:
+ Tmp1 = LegalizeOp(Node->getOperand(0)); // LHS
+ Tmp2 = LegalizeOp(Node->getOperand(1)); // RHS
+
+ switch (TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0))) {
+ case TargetLowering::Promote: assert(0 && "Cannot promote this yet!");
+ case TargetLowering::Custom:
+ isCustom = true;
+ // FALLTHROUGH
+ case TargetLowering::Legal:
+ Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2);
+ if (isCustom) {
+ Tmp1 = TLI.LowerOperation(Result, DAG);
+ if (Tmp1.Val) Result = Tmp1;
+ }
+ break;
+ case TargetLowering::Expand:
+ unsigned DivOpc= (Node->getOpcode() == ISD::UREM) ? ISD::UDIV : ISD::SDIV;
+ bool isSigned = DivOpc == ISD::SDIV;
+ if (MVT::isInteger(Node->getValueType(0))) {
+ if (TLI.getOperationAction(DivOpc, Node->getValueType(0)) ==
+ TargetLowering::Legal) {
+ // X % Y -> X-X/Y*Y
+ MVT::ValueType VT = Node->getValueType(0);
+ Result = DAG.getNode(DivOpc, VT, Tmp1, Tmp2);
+ Result = DAG.getNode(ISD::MUL, VT, Result, Tmp2);
+ Result = DAG.getNode(ISD::SUB, VT, Tmp1, Result);
+ } else {
+ assert(Node->getValueType(0) == MVT::i32 &&
+ "Cannot expand this binary operator!");
+ RTLIB::Libcall LC = Node->getOpcode() == ISD::UREM
+ ? RTLIB::UREM_I32 : RTLIB::SREM_I32;
+ SDOperand Dummy;
+ Result = ExpandLibCall(TLI.getLibcallName(LC), Node, isSigned, Dummy);
+ }
+ } else {
+ // Floating point mod -> fmod libcall.
+ RTLIB::Libcall LC = Node->getValueType(0) == MVT::f32
+ ? RTLIB::REM_F32 : RTLIB::REM_F64;
+ SDOperand Dummy;
+ Result = ExpandLibCall(TLI.getLibcallName(LC), Node,
+ false/*sign irrelevant*/, Dummy);
+ }
+ break;
+ }
+ break;
+ case ISD::VAARG: {
+ Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the chain.
+ Tmp2 = LegalizeOp(Node->getOperand(1)); // Legalize the pointer.
+
+ MVT::ValueType VT = Node->getValueType(0);
+ switch (TLI.getOperationAction(Node->getOpcode(), MVT::Other)) {
+ default: assert(0 && "This action is not supported yet!");
+ case TargetLowering::Custom:
+ isCustom = true;
+ // FALLTHROUGH
+ case TargetLowering::Legal:
+ Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Node->getOperand(2));
+ Result = Result.getValue(0);
+ Tmp1 = Result.getValue(1);
+
+ if (isCustom) {
+ Tmp2 = TLI.LowerOperation(Result, DAG);
+ if (Tmp2.Val) {
+ Result = LegalizeOp(Tmp2);
+ Tmp1 = LegalizeOp(Tmp2.getValue(1));
+ }
+ }
+ break;
+ case TargetLowering::Expand: {
+ SrcValueSDNode *SV = cast<SrcValueSDNode>(Node->getOperand(2));
+ SDOperand VAList = DAG.getLoad(TLI.getPointerTy(), Tmp1, Tmp2,
+ SV->getValue(), SV->getOffset());
+ // Increment the pointer, VAList, to the next vaarg
+ Tmp3 = DAG.getNode(ISD::ADD, TLI.getPointerTy(), VAList,
+ DAG.getConstant(MVT::getSizeInBits(VT)/8,
+ TLI.getPointerTy()));
+ // Store the incremented VAList to the legalized pointer
+ Tmp3 = DAG.getStore(VAList.getValue(1), Tmp3, Tmp2, SV->getValue(),
+ SV->getOffset());
+ // Load the actual argument out of the pointer VAList
+ Result = DAG.getLoad(VT, Tmp3, VAList, NULL, 0);
+ Tmp1 = LegalizeOp(Result.getValue(1));
+ Result = LegalizeOp(Result);
+ break;
+ }
+ }
+ // Since VAARG produces two values, make sure to remember that we
+ // legalized both of them.
+ AddLegalizedOperand(SDOperand(Node, 0), Result);
+ AddLegalizedOperand(SDOperand(Node, 1), Tmp1);
+ return Op.ResNo ? Tmp1 : Result;
+ }
+
+ case ISD::VACOPY:
+ Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the chain.
+ Tmp2 = LegalizeOp(Node->getOperand(1)); // Legalize the dest pointer.
+ Tmp3 = LegalizeOp(Node->getOperand(2)); // Legalize the source pointer.
+
+ switch (TLI.getOperationAction(ISD::VACOPY, MVT::Other)) {
+ default: assert(0 && "This action is not supported yet!");
+ case TargetLowering::Custom:
+ isCustom = true;
+ // FALLTHROUGH
+ case TargetLowering::Legal:
+ Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3,
+ Node->getOperand(3), Node->getOperand(4));
+ if (isCustom) {
+ Tmp1 = TLI.LowerOperation(Result, DAG);
+ if (Tmp1.Val) Result = Tmp1;
+ }
+ break;
+ case TargetLowering::Expand:
+ // This defaults to loading a pointer from the input and storing it to the
+ // output, returning the chain.
+ SrcValueSDNode *SVD = cast<SrcValueSDNode>(Node->getOperand(3));
+ SrcValueSDNode *SVS = cast<SrcValueSDNode>(Node->getOperand(4));
+ Tmp4 = DAG.getLoad(TLI.getPointerTy(), Tmp1, Tmp3, SVD->getValue(),
+ SVD->getOffset());
+ Result = DAG.getStore(Tmp4.getValue(1), Tmp4, Tmp2, SVS->getValue(),
+ SVS->getOffset());
+ break;
+ }
+ break;
+
+ case ISD::VAEND:
+ Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the chain.
+ Tmp2 = LegalizeOp(Node->getOperand(1)); // Legalize the pointer.
+
+ switch (TLI.getOperationAction(ISD::VAEND, MVT::Other)) {
+ default: assert(0 && "This action is not supported yet!");
+ case TargetLowering::Custom:
+ isCustom = true;
+ // FALLTHROUGH
+ case TargetLowering::Legal:
+ Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Node->getOperand(2));
+ if (isCustom) {
+ Tmp1 = TLI.LowerOperation(Tmp1, DAG);
+ if (Tmp1.Val) Result = Tmp1;
+ }
+ break;
+ case TargetLowering::Expand:
+ Result = Tmp1; // Default to a no-op, return the chain
+ break;
+ }
+ break;
+
+ case ISD::VASTART:
+ Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the chain.
+ Tmp2 = LegalizeOp(Node->getOperand(1)); // Legalize the pointer.
+
+ Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Node->getOperand(2));
+
+ switch (TLI.getOperationAction(ISD::VASTART, MVT::Other)) {
+ default: assert(0 && "This action is not supported yet!");
+ case TargetLowering::Legal: break;
+ case TargetLowering::Custom:
+ Tmp1 = TLI.LowerOperation(Result, DAG);
+ if (Tmp1.Val) Result = Tmp1;
+ break;
+ }
+ break;
+
+ case ISD::ROTL:
+ case ISD::ROTR:
+ Tmp1 = LegalizeOp(Node->getOperand(0)); // LHS
+ Tmp2 = LegalizeOp(Node->getOperand(1)); // RHS
+ Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2);
+ switch (TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0))) {
+ default:
+ assert(0 && "ROTL/ROTR legalize operation not supported");
+ break;
+ case TargetLowering::Legal:
+ break;
+ case TargetLowering::Custom:
+ Tmp1 = TLI.LowerOperation(Result, DAG);
+ if (Tmp1.Val) Result = Tmp1;
+ break;
+ case TargetLowering::Promote:
+ assert(0 && "Do not know how to promote ROTL/ROTR");
+ break;
+ case TargetLowering::Expand:
+ assert(0 && "Do not know how to expand ROTL/ROTR");
+ break;
+ }
+ break;
+
+ case ISD::BSWAP:
+ Tmp1 = LegalizeOp(Node->getOperand(0)); // Op
+ switch (TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0))) {
+ case TargetLowering::Custom:
+ assert(0 && "Cannot custom legalize this yet!");
+ case TargetLowering::Legal:
+ Result = DAG.UpdateNodeOperands(Result, Tmp1);
+ break;
+ case TargetLowering::Promote: {
+ MVT::ValueType OVT = Tmp1.getValueType();
+ MVT::ValueType NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), OVT);
+ unsigned DiffBits = MVT::getSizeInBits(NVT) - MVT::getSizeInBits(OVT);
+
+ Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, NVT, Tmp1);
+ Tmp1 = DAG.getNode(ISD::BSWAP, NVT, Tmp1);
+ Result = DAG.getNode(ISD::SRL, NVT, Tmp1,
+ DAG.getConstant(DiffBits, TLI.getShiftAmountTy()));
+ break;
+ }
+ case TargetLowering::Expand:
+ Result = ExpandBSWAP(Tmp1);
+ break;
+ }
+ break;
+
+ case ISD::CTPOP:
+ case ISD::CTTZ:
+ case ISD::CTLZ:
+ Tmp1 = LegalizeOp(Node->getOperand(0)); // Op
+ switch (TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0))) {
+ case TargetLowering::Custom: assert(0 && "Cannot custom handle this yet!");
+ case TargetLowering::Legal:
+ Result = DAG.UpdateNodeOperands(Result, Tmp1);
+ break;
+ case TargetLowering::Promote: {
+ MVT::ValueType OVT = Tmp1.getValueType();
+ MVT::ValueType NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), OVT);
+
+ // Zero extend the argument.
+ Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, NVT, Tmp1);
+ // Perform the larger operation, then subtract if needed.
+ Tmp1 = DAG.getNode(Node->getOpcode(), Node->getValueType(0), Tmp1);
+ switch (Node->getOpcode()) {
+ case ISD::CTPOP:
+ Result = Tmp1;
+ break;
+ case ISD::CTTZ:
+ //if Tmp1 == sizeinbits(NVT) then Tmp1 = sizeinbits(Old VT)
+ Tmp2 = DAG.getSetCC(TLI.getSetCCResultTy(), Tmp1,
+ DAG.getConstant(MVT::getSizeInBits(NVT), NVT),
+ ISD::SETEQ);
+ Result = DAG.getNode(ISD::SELECT, NVT, Tmp2,
+ DAG.getConstant(MVT::getSizeInBits(OVT),NVT), Tmp1);
+ break;
+ case ISD::CTLZ:
+ // Tmp1 = Tmp1 - (sizeinbits(NVT) - sizeinbits(Old VT))
+ Result = DAG.getNode(ISD::SUB, NVT, Tmp1,
+ DAG.getConstant(MVT::getSizeInBits(NVT) -
+ MVT::getSizeInBits(OVT), NVT));
+ break;
+ }
+ break;
+ }
+ case TargetLowering::Expand:
+ Result = ExpandBitCount(Node->getOpcode(), Tmp1);
+ break;
+ }
+ break;
+
+ // Unary operators
+ case ISD::FABS:
+ case ISD::FNEG:
+ case ISD::FSQRT:
+ case ISD::FSIN:
+ case ISD::FCOS:
+ Tmp1 = LegalizeOp(Node->getOperand(0));
+ switch (TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0))) {
+ case TargetLowering::Promote:
+ case TargetLowering::Custom:
+ isCustom = true;
+ // FALLTHROUGH
+ case TargetLowering::Legal:
+ Result = DAG.UpdateNodeOperands(Result, Tmp1);
+ if (isCustom) {
+ Tmp1 = TLI.LowerOperation(Result, DAG);
+ if (Tmp1.Val) Result = Tmp1;
+ }
+ break;
+ case TargetLowering::Expand:
+ switch (Node->getOpcode()) {
+ default: assert(0 && "Unreachable!");
+ case ISD::FNEG:
+ // Expand Y = FNEG(X) -> Y = SUB -0.0, X
+ Tmp2 = DAG.getConstantFP(-0.0, Node->getValueType(0));
+ Result = DAG.getNode(ISD::FSUB, Node->getValueType(0), Tmp2, Tmp1);
+ break;
+ case ISD::FABS: {
+ // Expand Y = FABS(X) -> Y = (X >u 0.0) ? X : fneg(X).
+ MVT::ValueType VT = Node->getValueType(0);
+ Tmp2 = DAG.getConstantFP(0.0, VT);
+ Tmp2 = DAG.getSetCC(TLI.getSetCCResultTy(), Tmp1, Tmp2, ISD::SETUGT);
+ Tmp3 = DAG.getNode(ISD::FNEG, VT, Tmp1);
+ Result = DAG.getNode(ISD::SELECT, VT, Tmp2, Tmp1, Tmp3);
+ break;
+ }
+ case ISD::FSQRT:
+ case ISD::FSIN:
+ case ISD::FCOS: {
+ MVT::ValueType VT = Node->getValueType(0);
+ RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
+ switch(Node->getOpcode()) {
+ case ISD::FSQRT:
+ LC = VT == MVT::f32 ? RTLIB::SQRT_F32 : RTLIB::SQRT_F64;
+ break;
+ case ISD::FSIN:
+ LC = VT == MVT::f32 ? RTLIB::SIN_F32 : RTLIB::SIN_F64;
+ break;
+ case ISD::FCOS:
+ LC = VT == MVT::f32 ? RTLIB::COS_F32 : RTLIB::COS_F64;
+ break;
+ default: assert(0 && "Unreachable!");
+ }
+ SDOperand Dummy;
+ Result = ExpandLibCall(TLI.getLibcallName(LC), Node,
+ false/*sign irrelevant*/, Dummy);
+ break;
+ }
+ }
+ break;
+ }
+ break;
+ case ISD::FPOWI: {
+ // We always lower FPOWI into a libcall. No target support it yet.
+ RTLIB::Libcall LC = Node->getValueType(0) == MVT::f32
+ ? RTLIB::POWI_F32 : RTLIB::POWI_F64;
+ SDOperand Dummy;
+ Result = ExpandLibCall(TLI.getLibcallName(LC), Node,
+ false/*sign irrelevant*/, Dummy);
+ break;
+ }
+ case ISD::BIT_CONVERT:
+ if (!isTypeLegal(Node->getOperand(0).getValueType())) {
+ Result = ExpandBIT_CONVERT(Node->getValueType(0), Node->getOperand(0));
+ } else if (MVT::isVector(Op.getOperand(0).getValueType())) {
+ // The input has to be a vector type, we have to either scalarize it, pack
+ // it, or convert it based on whether the input vector type is legal.
+ SDNode *InVal = Node->getOperand(0).Val;
+ unsigned NumElems = MVT::getVectorNumElements(InVal->getValueType(0));
+ MVT::ValueType EVT = MVT::getVectorElementType(InVal->getValueType(0));
+
+ // Figure out if there is a simple type corresponding to this Vector
+ // type. If so, convert to the vector type.
+ MVT::ValueType TVT = MVT::getVectorType(EVT, NumElems);
+ if (TLI.isTypeLegal(TVT)) {
+ // Turn this into a bit convert of the vector input.
+ Result = DAG.getNode(ISD::BIT_CONVERT, Node->getValueType(0),
+ LegalizeOp(Node->getOperand(0)));
+ break;
+ } else if (NumElems == 1) {
+ // Turn this into a bit convert of the scalar input.
+ Result = DAG.getNode(ISD::BIT_CONVERT, Node->getValueType(0),
+ ScalarizeVectorOp(Node->getOperand(0)));
+ break;
+ } else {
+ // FIXME: UNIMP! Store then reload
+ assert(0 && "Cast from unsupported vector type not implemented yet!");
+ }
+ } else {
+ switch (TLI.getOperationAction(ISD::BIT_CONVERT,
+ Node->getOperand(0).getValueType())) {
+ default: assert(0 && "Unknown operation action!");
+ case TargetLowering::Expand:
+ Result = ExpandBIT_CONVERT(Node->getValueType(0), Node->getOperand(0));
+ break;
+ case TargetLowering::Legal:
+ Tmp1 = LegalizeOp(Node->getOperand(0));
+ Result = DAG.UpdateNodeOperands(Result, Tmp1);
+ break;
+ }
+ }
+ break;
+
+ // Conversion operators. The source and destination have different types.
+ case ISD::SINT_TO_FP:
+ case ISD::UINT_TO_FP: {
+ bool isSigned = Node->getOpcode() == ISD::SINT_TO_FP;
+ switch (getTypeAction(Node->getOperand(0).getValueType())) {
+ case Legal:
+ switch (TLI.getOperationAction(Node->getOpcode(),
+ Node->getOperand(0).getValueType())) {
+ default: assert(0 && "Unknown operation action!");
+ case TargetLowering::Custom:
+ isCustom = true;
+ // FALLTHROUGH
+ case TargetLowering::Legal:
+ Tmp1 = LegalizeOp(Node->getOperand(0));
+ Result = DAG.UpdateNodeOperands(Result, Tmp1);
+ if (isCustom) {
+ Tmp1 = TLI.LowerOperation(Result, DAG);
+ if (Tmp1.Val) Result = Tmp1;
+ }
+ break;
+ case TargetLowering::Expand:
+ Result = ExpandLegalINT_TO_FP(isSigned,
+ LegalizeOp(Node->getOperand(0)),
+ Node->getValueType(0));
+ break;
+ case TargetLowering::Promote:
+ Result = PromoteLegalINT_TO_FP(LegalizeOp(Node->getOperand(0)),
+ Node->getValueType(0),
+ isSigned);
+ break;
+ }
+ break;
+ case Expand:
+ Result = ExpandIntToFP(Node->getOpcode() == ISD::SINT_TO_FP,
+ Node->getValueType(0), Node->getOperand(0));
+ break;
+ case Promote:
+ Tmp1 = PromoteOp(Node->getOperand(0));
+ if (isSigned) {
+ Tmp1 = DAG.getNode(ISD::SIGN_EXTEND_INREG, Tmp1.getValueType(),
+ Tmp1, DAG.getValueType(Node->getOperand(0).getValueType()));
+ } else {
+ Tmp1 = DAG.getZeroExtendInReg(Tmp1,
+ Node->getOperand(0).getValueType());
+ }
+ Result = DAG.UpdateNodeOperands(Result, Tmp1);
+ Result = LegalizeOp(Result); // The 'op' is not necessarily legal!
+ break;
+ }
+ break;
+ }
+ case ISD::TRUNCATE:
+ switch (getTypeAction(Node->getOperand(0).getValueType())) {
+ case Legal:
+ Tmp1 = LegalizeOp(Node->getOperand(0));
+ Result = DAG.UpdateNodeOperands(Result, Tmp1);
+ break;
+ case Expand:
+ ExpandOp(Node->getOperand(0), Tmp1, Tmp2);
+
+ // Since the result is legal, we should just be able to truncate the low
+ // part of the source.
+ Result = DAG.getNode(ISD::TRUNCATE, Node->getValueType(0), Tmp1);
+ break;
+ case Promote:
+ Result = PromoteOp(Node->getOperand(0));
+ Result = DAG.getNode(ISD::TRUNCATE, Op.getValueType(), Result);
+ break;
+ }
+ break;
+
+ case ISD::FP_TO_SINT:
+ case ISD::FP_TO_UINT:
+ switch (getTypeAction(Node->getOperand(0).getValueType())) {
+ case Legal:
+ Tmp1 = LegalizeOp(Node->getOperand(0));
+
+ switch (TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0))){
+ default: assert(0 && "Unknown operation action!");
+ case TargetLowering::Custom:
+ isCustom = true;
+ // FALLTHROUGH
+ case TargetLowering::Legal:
+ Result = DAG.UpdateNodeOperands(Result, Tmp1);
+ if (isCustom) {
+ Tmp1 = TLI.LowerOperation(Result, DAG);
+ if (Tmp1.Val) Result = Tmp1;
+ }
+ break;
+ case TargetLowering::Promote:
+ Result = PromoteLegalFP_TO_INT(Tmp1, Node->getValueType(0),
+ Node->getOpcode() == ISD::FP_TO_SINT);
+ break;
+ case TargetLowering::Expand:
+ if (Node->getOpcode() == ISD::FP_TO_UINT) {
+ SDOperand True, False;
+ MVT::ValueType VT = Node->getOperand(0).getValueType();
+ MVT::ValueType NVT = Node->getValueType(0);
+ unsigned ShiftAmt = MVT::getSizeInBits(Node->getValueType(0))-1;
+ Tmp2 = DAG.getConstantFP((double)(1ULL << ShiftAmt), VT);
+ Tmp3 = DAG.getSetCC(TLI.getSetCCResultTy(),
+ Node->getOperand(0), Tmp2, ISD::SETLT);
+ True = DAG.getNode(ISD::FP_TO_SINT, NVT, Node->getOperand(0));
+ False = DAG.getNode(ISD::FP_TO_SINT, NVT,
+ DAG.getNode(ISD::FSUB, VT, Node->getOperand(0),
+ Tmp2));
+ False = DAG.getNode(ISD::XOR, NVT, False,
+ DAG.getConstant(1ULL << ShiftAmt, NVT));
+ Result = DAG.getNode(ISD::SELECT, NVT, Tmp3, True, False);
+ break;
+ } else {
+ assert(0 && "Do not know how to expand FP_TO_SINT yet!");
+ }
+ break;
+ }
+ break;
+ case Expand: {
+ // Convert f32 / f64 to i32 / i64.
+ MVT::ValueType VT = Op.getValueType();
+ RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
+ switch (Node->getOpcode()) {
+ case ISD::FP_TO_SINT:
+ if (Node->getOperand(0).getValueType() == MVT::f32)
+ LC = (VT == MVT::i32)
+ ? RTLIB::FPTOSINT_F32_I32 : RTLIB::FPTOSINT_F32_I64;
+ else
+ LC = (VT == MVT::i32)
+ ? RTLIB::FPTOSINT_F64_I32 : RTLIB::FPTOSINT_F64_I64;
+ break;
+ case ISD::FP_TO_UINT:
+ if (Node->getOperand(0).getValueType() == MVT::f32)
+ LC = (VT == MVT::i32)
+ ? RTLIB::FPTOUINT_F32_I32 : RTLIB::FPTOSINT_F32_I64;
+ else
+ LC = (VT == MVT::i32)
+ ? RTLIB::FPTOUINT_F64_I32 : RTLIB::FPTOSINT_F64_I64;
+ break;
+ default: assert(0 && "Unreachable!");
+ }
+ SDOperand Dummy;
+ Result = ExpandLibCall(TLI.getLibcallName(LC), Node,
+ false/*sign irrelevant*/, Dummy);
+ break;
+ }
+ case Promote:
+ Tmp1 = PromoteOp(Node->getOperand(0));
+ Result = DAG.UpdateNodeOperands(Result, LegalizeOp(Tmp1));
+ Result = LegalizeOp(Result);
+ break;
+ }
+ break;
+
+ case ISD::FP_ROUND:
+ if (TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)) ==
+ TargetLowering::Expand) {
+ // The only way we can lower this is to turn it into a TRUNCSTORE,
+ // EXTLOAD pair, targetting a temporary location (a stack slot).
+
+ // NOTE: there is a choice here between constantly creating new stack
+ // slots and always reusing the same one. We currently always create
+ // new ones, as reuse may inhibit scheduling.
+ MVT::ValueType VT = Op.getValueType(); // 32
+ const Type *Ty = MVT::getTypeForValueType(VT);
+ uint64_t TySize = TLI.getTargetData()->getTypeSize(Ty);
+ unsigned Align = TLI.getTargetData()->getPrefTypeAlignment(Ty);
+ MachineFunction &MF = DAG.getMachineFunction();
+ int SSFI =
+ MF.getFrameInfo()->CreateStackObject(TySize, Align);
+ SDOperand StackSlot = DAG.getFrameIndex(SSFI, TLI.getPointerTy());
+ Result = DAG.getTruncStore(DAG.getEntryNode(), Node->getOperand(0),
+ StackSlot, NULL, 0, VT);
+ Result = DAG.getLoad(VT, Result, StackSlot, NULL, 0, VT);
+ break;
+ }
+ // FALL THROUGH
+ case ISD::ANY_EXTEND:
+ case ISD::ZERO_EXTEND:
+ case ISD::SIGN_EXTEND:
+ case ISD::FP_EXTEND:
+ switch (getTypeAction(Node->getOperand(0).getValueType())) {
+ case Expand: assert(0 && "Shouldn't need to expand other operators here!");
+ case Legal:
+ Tmp1 = LegalizeOp(Node->getOperand(0));
+ Result = DAG.UpdateNodeOperands(Result, Tmp1);
+ break;
+ case Promote:
+ switch (Node->getOpcode()) {
+ case ISD::ANY_EXTEND:
+ Tmp1 = PromoteOp(Node->getOperand(0));
+ Result = DAG.getNode(ISD::ANY_EXTEND, Op.getValueType(), Tmp1);
+ break;
+ case ISD::ZERO_EXTEND:
+ Result = PromoteOp(Node->getOperand(0));
+ Result = DAG.getNode(ISD::ANY_EXTEND, Op.getValueType(), Result);
+ Result = DAG.getZeroExtendInReg(Result,
+ Node->getOperand(0).getValueType());
+ break;
+ case ISD::SIGN_EXTEND:
+ Result = PromoteOp(Node->getOperand(0));
+ Result = DAG.getNode(ISD::ANY_EXTEND, Op.getValueType(), Result);
+ Result = DAG.getNode(ISD::SIGN_EXTEND_INREG, Result.getValueType(),
+ Result,
+ DAG.getValueType(Node->getOperand(0).getValueType()));
+ break;
+ case ISD::FP_EXTEND:
+ Result = PromoteOp(Node->getOperand(0));
+ if (Result.getValueType() != Op.getValueType())
+ // Dynamically dead while we have only 2 FP types.
+ Result = DAG.getNode(ISD::FP_EXTEND, Op.getValueType(), Result);
+ break;
+ case ISD::FP_ROUND:
+ Result = PromoteOp(Node->getOperand(0));
+ Result = DAG.getNode(Node->getOpcode(), Op.getValueType(), Result);
+ break;
+ }
+ }
+ break;
+ case ISD::FP_ROUND_INREG:
+ case ISD::SIGN_EXTEND_INREG: {
+ Tmp1 = LegalizeOp(Node->getOperand(0));
+ MVT::ValueType ExtraVT = cast<VTSDNode>(Node->getOperand(1))->getVT();
+
+ // If this operation is not supported, convert it to a shl/shr or load/store
+ // pair.
+ switch (TLI.getOperationAction(Node->getOpcode(), ExtraVT)) {
+ default: assert(0 && "This action not supported for this op yet!");
+ case TargetLowering::Legal:
+ Result = DAG.UpdateNodeOperands(Result, Tmp1, Node->getOperand(1));
+ break;
+ case TargetLowering::Expand:
+ // If this is an integer extend and shifts are supported, do that.
+ if (Node->getOpcode() == ISD::SIGN_EXTEND_INREG) {
+ // NOTE: we could fall back on load/store here too for targets without
+ // SAR. However, it is doubtful that any exist.
+ unsigned BitsDiff = MVT::getSizeInBits(Node->getValueType(0)) -
+ MVT::getSizeInBits(ExtraVT);
+ SDOperand ShiftCst = DAG.getConstant(BitsDiff, TLI.getShiftAmountTy());
+ Result = DAG.getNode(ISD::SHL, Node->getValueType(0),
+ Node->getOperand(0), ShiftCst);
+ Result = DAG.getNode(ISD::SRA, Node->getValueType(0),
+ Result, ShiftCst);
+ } else if (Node->getOpcode() == ISD::FP_ROUND_INREG) {
+ // The only way we can lower this is to turn it into a TRUNCSTORE,
+ // EXTLOAD pair, targetting a temporary location (a stack slot).
+
+ // NOTE: there is a choice here between constantly creating new stack
+ // slots and always reusing the same one. We currently always create
+ // new ones, as reuse may inhibit scheduling.
+ const Type *Ty = MVT::getTypeForValueType(ExtraVT);
+ uint64_t TySize = TLI.getTargetData()->getTypeSize(Ty);
+ unsigned Align = TLI.getTargetData()->getPrefTypeAlignment(Ty);
+ MachineFunction &MF = DAG.getMachineFunction();
+ int SSFI =
+ MF.getFrameInfo()->CreateStackObject(TySize, Align);
+ SDOperand StackSlot = DAG.getFrameIndex(SSFI, TLI.getPointerTy());
+ Result = DAG.getTruncStore(DAG.getEntryNode(), Node->getOperand(0),
+ StackSlot, NULL, 0, ExtraVT);
+ Result = DAG.getExtLoad(ISD::EXTLOAD, Node->getValueType(0),
+ Result, StackSlot, NULL, 0, ExtraVT);
+ } else {
+ assert(0 && "Unknown op");
+ }
+ break;
+ }
+ break;
+ }
+ }
+
+ assert(Result.getValueType() == Op.getValueType() &&
+ "Bad legalization!");
+
+ // Make sure that the generated code is itself legal.
+ if (Result != Op)
+ Result = LegalizeOp(Result);
+
+ // Note that LegalizeOp may be reentered even from single-use nodes, which
+ // means that we always must cache transformed nodes.
+ AddLegalizedOperand(Op, Result);
+ return Result;
+}
+
+/// PromoteOp - Given an operation that produces a value in an invalid type,
+/// promote it to compute the value into a larger type. The produced value will
+/// have the correct bits for the low portion of the register, but no guarantee
+/// is made about the top bits: it may be zero, sign-extended, or garbage.
+SDOperand SelectionDAGLegalize::PromoteOp(SDOperand Op) {
+ MVT::ValueType VT = Op.getValueType();
+ MVT::ValueType NVT = TLI.getTypeToTransformTo(VT);
+ assert(getTypeAction(VT) == Promote &&
+ "Caller should expand or legalize operands that are not promotable!");
+ assert(NVT > VT && MVT::isInteger(NVT) == MVT::isInteger(VT) &&
+ "Cannot promote to smaller type!");
+
+ SDOperand Tmp1, Tmp2, Tmp3;
+ SDOperand Result;
+ SDNode *Node = Op.Val;
+
+ DenseMap<SDOperand, SDOperand>::iterator I = PromotedNodes.find(Op);
+ if (I != PromotedNodes.end()) return I->second;
+
+ switch (Node->getOpcode()) {
+ case ISD::CopyFromReg:
+ assert(0 && "CopyFromReg must be legal!");
+ default:
+#ifndef NDEBUG
+ cerr << "NODE: "; Node->dump(&DAG); cerr << "\n";
+#endif
+ assert(0 && "Do not know how to promote this operator!");
+ abort();
+ case ISD::UNDEF:
+ Result = DAG.getNode(ISD::UNDEF, NVT);
+ break;
+ case ISD::Constant:
+ if (VT != MVT::i1)
+ Result = DAG.getNode(ISD::SIGN_EXTEND, NVT, Op);
+ else
+ Result = DAG.getNode(ISD::ZERO_EXTEND, NVT, Op);
+ assert(isa<ConstantSDNode>(Result) && "Didn't constant fold zext?");
+ break;
+ case ISD::ConstantFP:
+ Result = DAG.getNode(ISD::FP_EXTEND, NVT, Op);
+ assert(isa<ConstantFPSDNode>(Result) && "Didn't constant fold fp_extend?");
+ break;
+
+ case ISD::SETCC:
+ assert(isTypeLegal(TLI.getSetCCResultTy()) && "SetCC type is not legal??");
+ Result = DAG.getNode(ISD::SETCC, TLI.getSetCCResultTy(),Node->getOperand(0),
+ Node->getOperand(1), Node->getOperand(2));
+ break;
+
+ case ISD::TRUNCATE:
+ switch (getTypeAction(Node->getOperand(0).getValueType())) {
+ case Legal:
+ Result = LegalizeOp(Node->getOperand(0));
+ assert(Result.getValueType() >= NVT &&
+ "This truncation doesn't make sense!");
+ if (Result.getValueType() > NVT) // Truncate to NVT instead of VT
+ Result = DAG.getNode(ISD::TRUNCATE, NVT, Result);
+ break;
+ case Promote:
+ // The truncation is not required, because we don't guarantee anything
+ // about high bits anyway.
+ Result = PromoteOp(Node->getOperand(0));
+ break;
+ case Expand:
+ ExpandOp(Node->getOperand(0), Tmp1, Tmp2);
+ // Truncate the low part of the expanded value to the result type
+ Result = DAG.getNode(ISD::TRUNCATE, NVT, Tmp1);
+ }
+ break;
+ case ISD::SIGN_EXTEND:
+ case ISD::ZERO_EXTEND:
+ case ISD::ANY_EXTEND:
+ switch (getTypeAction(Node->getOperand(0).getValueType())) {
+ case Expand: assert(0 && "BUG: Smaller reg should have been promoted!");
+ case Legal:
+ // Input is legal? Just do extend all the way to the larger type.
+ Result = DAG.getNode(Node->getOpcode(), NVT, Node->getOperand(0));
+ break;
+ case Promote:
+ // Promote the reg if it's smaller.
+ Result = PromoteOp(Node->getOperand(0));
+ // The high bits are not guaranteed to be anything. Insert an extend.
+ if (Node->getOpcode() == ISD::SIGN_EXTEND)
+ Result = DAG.getNode(ISD::SIGN_EXTEND_INREG, NVT, Result,
+ DAG.getValueType(Node->getOperand(0).getValueType()));
+ else if (Node->getOpcode() == ISD::ZERO_EXTEND)
+ Result = DAG.getZeroExtendInReg(Result,
+ Node->getOperand(0).getValueType());
+ break;
+ }
+ break;
+ case ISD::BIT_CONVERT:
+ Result = ExpandBIT_CONVERT(Node->getValueType(0), Node->getOperand(0));
+ Result = PromoteOp(Result);
+ break;
+
+ case ISD::FP_EXTEND:
+ assert(0 && "Case not implemented. Dynamically dead with 2 FP types!");
+ case ISD::FP_ROUND:
+ switch (getTypeAction(Node->getOperand(0).getValueType())) {
+ case Expand: assert(0 && "BUG: Cannot expand FP regs!");
+ case Promote: assert(0 && "Unreachable with 2 FP types!");
+ case Legal:
+ // Input is legal? Do an FP_ROUND_INREG.
+ Result = DAG.getNode(ISD::FP_ROUND_INREG, NVT, Node->getOperand(0),
+ DAG.getValueType(VT));
+ break;
+ }
+ break;
+
+ case ISD::SINT_TO_FP:
+ case ISD::UINT_TO_FP:
+ switch (getTypeAction(Node->getOperand(0).getValueType())) {
+ case Legal:
+ // No extra round required here.
+ Result = DAG.getNode(Node->getOpcode(), NVT, Node->getOperand(0));
+ break;
+
+ case Promote:
+ Result = PromoteOp(Node->getOperand(0));
+ if (Node->getOpcode() == ISD::SINT_TO_FP)
+ Result = DAG.getNode(ISD::SIGN_EXTEND_INREG, Result.getValueType(),
+ Result,
+ DAG.getValueType(Node->getOperand(0).getValueType()));
+ else
+ Result = DAG.getZeroExtendInReg(Result,
+ Node->getOperand(0).getValueType());
+ // No extra round required here.
+ Result = DAG.getNode(Node->getOpcode(), NVT, Result);
+ break;
+ case Expand:
+ Result = ExpandIntToFP(Node->getOpcode() == ISD::SINT_TO_FP, NVT,
+ Node->getOperand(0));
+ // Round if we cannot tolerate excess precision.
+ if (NoExcessFPPrecision)
+ Result = DAG.getNode(ISD::FP_ROUND_INREG, NVT, Result,
+ DAG.getValueType(VT));
+ break;
+ }
+ break;
+
+ case ISD::SIGN_EXTEND_INREG:
+ Result = PromoteOp(Node->getOperand(0));
+ Result = DAG.getNode(ISD::SIGN_EXTEND_INREG, NVT, Result,
+ Node->getOperand(1));
+ break;
+ case ISD::FP_TO_SINT:
+ case ISD::FP_TO_UINT:
+ switch (getTypeAction(Node->getOperand(0).getValueType())) {
+ case Legal:
+ case Expand:
+ Tmp1 = Node->getOperand(0);
+ break;
+ case Promote:
+ // The input result is prerounded, so we don't have to do anything
+ // special.
+ Tmp1 = PromoteOp(Node->getOperand(0));
+ break;
+ }
+ // If we're promoting a UINT to a larger size, check to see if the new node
+ // will be legal. If it isn't, check to see if FP_TO_SINT is legal, since
+ // we can use that instead. This allows us to generate better code for
+ // FP_TO_UINT for small destination sizes on targets where FP_TO_UINT is not
+ // legal, such as PowerPC.
+ if (Node->getOpcode() == ISD::FP_TO_UINT &&
+ !TLI.isOperationLegal(ISD::FP_TO_UINT, NVT) &&
+ (TLI.isOperationLegal(ISD::FP_TO_SINT, NVT) ||
+ TLI.getOperationAction(ISD::FP_TO_SINT, NVT)==TargetLowering::Custom)){
+ Result = DAG.getNode(ISD::FP_TO_SINT, NVT, Tmp1);
+ } else {
+ Result = DAG.getNode(Node->getOpcode(), NVT, Tmp1);
+ }
+ break;
+
+ case ISD::FABS:
+ case ISD::FNEG:
+ Tmp1 = PromoteOp(Node->getOperand(0));
+ assert(Tmp1.getValueType() == NVT);
+ Result = DAG.getNode(Node->getOpcode(), NVT, Tmp1);
+ // NOTE: we do not have to do any extra rounding here for
+ // NoExcessFPPrecision, because we know the input will have the appropriate
+ // precision, and these operations don't modify precision at all.
+ break;
+
+ case ISD::FSQRT:
+ case ISD::FSIN:
+ case ISD::FCOS:
+ Tmp1 = PromoteOp(Node->getOperand(0));
+ assert(Tmp1.getValueType() == NVT);
+ Result = DAG.getNode(Node->getOpcode(), NVT, Tmp1);
+ if (NoExcessFPPrecision)
+ Result = DAG.getNode(ISD::FP_ROUND_INREG, NVT, Result,
+ DAG.getValueType(VT));
+ break;
+
+ case ISD::FPOWI: {
+ // Promote f32 powi to f64 powi. Note that this could insert a libcall
+ // directly as well, which may be better.
+ Tmp1 = PromoteOp(Node->getOperand(0));
+ assert(Tmp1.getValueType() == NVT);
+ Result = DAG.getNode(ISD::FPOWI, NVT, Tmp1, Node->getOperand(1));
+ if (NoExcessFPPrecision)
+ Result = DAG.getNode(ISD::FP_ROUND_INREG, NVT, Result,
+ DAG.getValueType(VT));
+ break;
+ }
+
+ case ISD::AND:
+ case ISD::OR:
+ case ISD::XOR:
+ case ISD::ADD:
+ case ISD::SUB:
+ case ISD::MUL:
+ // The input may have strange things in the top bits of the registers, but
+ // these operations don't care. They may have weird bits going out, but
+ // that too is okay if they are integer operations.
+ Tmp1 = PromoteOp(Node->getOperand(0));
+ Tmp2 = PromoteOp(Node->getOperand(1));
+ assert(Tmp1.getValueType() == NVT && Tmp2.getValueType() == NVT);
+ Result = DAG.getNode(Node->getOpcode(), NVT, Tmp1, Tmp2);
+ break;
+ case ISD::FADD:
+ case ISD::FSUB:
+ case ISD::FMUL:
+ Tmp1 = PromoteOp(Node->getOperand(0));
+ Tmp2 = PromoteOp(Node->getOperand(1));
+ assert(Tmp1.getValueType() == NVT && Tmp2.getValueType() == NVT);
+ Result = DAG.getNode(Node->getOpcode(), NVT, Tmp1, Tmp2);
+
+ // Floating point operations will give excess precision that we may not be
+ // able to tolerate. If we DO allow excess precision, just leave it,
+ // otherwise excise it.
+ // FIXME: Why would we need to round FP ops more than integer ones?
+ // Is Round(Add(Add(A,B),C)) != Round(Add(Round(Add(A,B)), C))
+ if (NoExcessFPPrecision)
+ Result = DAG.getNode(ISD::FP_ROUND_INREG, NVT, Result,
+ DAG.getValueType(VT));
+ break;
+
+ case ISD::SDIV:
+ case ISD::SREM:
+ // These operators require that their input be sign extended.
+ Tmp1 = PromoteOp(Node->getOperand(0));
+ Tmp2 = PromoteOp(Node->getOperand(1));
+ if (MVT::isInteger(NVT)) {
+ Tmp1 = DAG.getNode(ISD::SIGN_EXTEND_INREG, NVT, Tmp1,
+ DAG.getValueType(VT));
+ Tmp2 = DAG.getNode(ISD::SIGN_EXTEND_INREG, NVT, Tmp2,
+ DAG.getValueType(VT));
+ }
+ Result = DAG.getNode(Node->getOpcode(), NVT, Tmp1, Tmp2);
+
+ // Perform FP_ROUND: this is probably overly pessimistic.
+ if (MVT::isFloatingPoint(NVT) && NoExcessFPPrecision)
+ Result = DAG.getNode(ISD::FP_ROUND_INREG, NVT, Result,
+ DAG.getValueType(VT));
+ break;
+ case ISD::FDIV:
+ case ISD::FREM:
+ case ISD::FCOPYSIGN:
+ // These operators require that their input be fp extended.
+ switch (getTypeAction(Node->getOperand(0).getValueType())) {
+ case Legal:
+ Tmp1 = LegalizeOp(Node->getOperand(0));
+ break;
+ case Promote:
+ Tmp1 = PromoteOp(Node->getOperand(0));
+ break;
+ case Expand:
+ assert(0 && "not implemented");
+ }
+ switch (getTypeAction(Node->getOperand(1).getValueType())) {
+ case Legal:
+ Tmp2 = LegalizeOp(Node->getOperand(1));
+ break;
+ case Promote:
+ Tmp2 = PromoteOp(Node->getOperand(1));
+ break;
+ case Expand:
+ assert(0 && "not implemented");
+ }
+ Result = DAG.getNode(Node->getOpcode(), NVT, Tmp1, Tmp2);
+
+ // Perform FP_ROUND: this is probably overly pessimistic.
+ if (NoExcessFPPrecision && Node->getOpcode() != ISD::FCOPYSIGN)
+ Result = DAG.getNode(ISD::FP_ROUND_INREG, NVT, Result,
+ DAG.getValueType(VT));
+ break;
+
+ case ISD::UDIV:
+ case ISD::UREM:
+ // These operators require that their input be zero extended.
+ Tmp1 = PromoteOp(Node->getOperand(0));
+ Tmp2 = PromoteOp(Node->getOperand(1));
+ assert(MVT::isInteger(NVT) && "Operators don't apply to FP!");
+ Tmp1 = DAG.getZeroExtendInReg(Tmp1, VT);
+ Tmp2 = DAG.getZeroExtendInReg(Tmp2, VT);
+ Result = DAG.getNode(Node->getOpcode(), NVT, Tmp1, Tmp2);
+ break;
+
+ case ISD::SHL:
+ Tmp1 = PromoteOp(Node->getOperand(0));
+ Result = DAG.getNode(ISD::SHL, NVT, Tmp1, Node->getOperand(1));
+ break;
+ case ISD::SRA:
+ // The input value must be properly sign extended.
+ Tmp1 = PromoteOp(Node->getOperand(0));
+ Tmp1 = DAG.getNode(ISD::SIGN_EXTEND_INREG, NVT, Tmp1,
+ DAG.getValueType(VT));
+ Result = DAG.getNode(ISD::SRA, NVT, Tmp1, Node->getOperand(1));
+ break;
+ case ISD::SRL:
+ // The input value must be properly zero extended.
+ Tmp1 = PromoteOp(Node->getOperand(0));
+ Tmp1 = DAG.getZeroExtendInReg(Tmp1, VT);
+ Result = DAG.getNode(ISD::SRL, NVT, Tmp1, Node->getOperand(1));
+ break;
+
+ case ISD::VAARG:
+ Tmp1 = Node->getOperand(0); // Get the chain.
+ Tmp2 = Node->getOperand(1); // Get the pointer.
+ if (TLI.getOperationAction(ISD::VAARG, VT) == TargetLowering::Custom) {
+ Tmp3 = DAG.getVAArg(VT, Tmp1, Tmp2, Node->getOperand(2));
+ Result = TLI.CustomPromoteOperation(Tmp3, DAG);
+ } else {
+ SrcValueSDNode *SV = cast<SrcValueSDNode>(Node->getOperand(2));
+ SDOperand VAList = DAG.getLoad(TLI.getPointerTy(), Tmp1, Tmp2,
+ SV->getValue(), SV->getOffset());
+ // Increment the pointer, VAList, to the next vaarg
+ Tmp3 = DAG.getNode(ISD::ADD, TLI.getPointerTy(), VAList,
+ DAG.getConstant(MVT::getSizeInBits(VT)/8,
+ TLI.getPointerTy()));
+ // Store the incremented VAList to the legalized pointer
+ Tmp3 = DAG.getStore(VAList.getValue(1), Tmp3, Tmp2, SV->getValue(),
+ SV->getOffset());
+ // Load the actual argument out of the pointer VAList
+ Result = DAG.getExtLoad(ISD::EXTLOAD, NVT, Tmp3, VAList, NULL, 0, VT);
+ }
+ // Remember that we legalized the chain.
+ AddLegalizedOperand(Op.getValue(1), LegalizeOp(Result.getValue(1)));
+ break;
+
+ case ISD::LOAD: {
+ LoadSDNode *LD = cast<LoadSDNode>(Node);
+ ISD::LoadExtType ExtType = ISD::isNON_EXTLoad(Node)
+ ? ISD::EXTLOAD : LD->getExtensionType();
+ Result = DAG.getExtLoad(ExtType, NVT,
+ LD->getChain(), LD->getBasePtr(),
+ LD->getSrcValue(), LD->getSrcValueOffset(),
+ LD->getLoadedVT(),
+ LD->isVolatile(),
+ LD->getAlignment());
+ // Remember that we legalized the chain.
+ AddLegalizedOperand(Op.getValue(1), LegalizeOp(Result.getValue(1)));
+ break;
+ }
+ case ISD::SELECT:
+ Tmp2 = PromoteOp(Node->getOperand(1)); // Legalize the op0
+ Tmp3 = PromoteOp(Node->getOperand(2)); // Legalize the op1
+ Result = DAG.getNode(ISD::SELECT, NVT, Node->getOperand(0), Tmp2, Tmp3);
+ break;
+ case ISD::SELECT_CC:
+ Tmp2 = PromoteOp(Node->getOperand(2)); // True
+ Tmp3 = PromoteOp(Node->getOperand(3)); // False
+ Result = DAG.getNode(ISD::SELECT_CC, NVT, Node->getOperand(0),
+ Node->getOperand(1), Tmp2, Tmp3, Node->getOperand(4));
+ break;
+ case ISD::BSWAP:
+ Tmp1 = Node->getOperand(0);
+ Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, NVT, Tmp1);
+ Tmp1 = DAG.getNode(ISD::BSWAP, NVT, Tmp1);
+ Result = DAG.getNode(ISD::SRL, NVT, Tmp1,
+ DAG.getConstant(MVT::getSizeInBits(NVT) -
+ MVT::getSizeInBits(VT),
+ TLI.getShiftAmountTy()));
+ break;
+ case ISD::CTPOP:
+ case ISD::CTTZ:
+ case ISD::CTLZ:
+ // Zero extend the argument
+ Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, NVT, Node->getOperand(0));
+ // Perform the larger operation, then subtract if needed.
+ Tmp1 = DAG.getNode(Node->getOpcode(), NVT, Tmp1);
+ switch(Node->getOpcode()) {
+ case ISD::CTPOP:
+ Result = Tmp1;
+ break;
+ case ISD::CTTZ:
+ // if Tmp1 == sizeinbits(NVT) then Tmp1 = sizeinbits(Old VT)
+ Tmp2 = DAG.getSetCC(TLI.getSetCCResultTy(), Tmp1,
+ DAG.getConstant(MVT::getSizeInBits(NVT), NVT),
+ ISD::SETEQ);
+ Result = DAG.getNode(ISD::SELECT, NVT, Tmp2,
+ DAG.getConstant(MVT::getSizeInBits(VT), NVT), Tmp1);
+ break;
+ case ISD::CTLZ:
+ //Tmp1 = Tmp1 - (sizeinbits(NVT) - sizeinbits(Old VT))
+ Result = DAG.getNode(ISD::SUB, NVT, Tmp1,
+ DAG.getConstant(MVT::getSizeInBits(NVT) -
+ MVT::getSizeInBits(VT), NVT));
+ break;
+ }
+ break;
+ case ISD::EXTRACT_SUBVECTOR:
+ Result = PromoteOp(ExpandEXTRACT_SUBVECTOR(Op));
+ break;
+ case ISD::EXTRACT_VECTOR_ELT:
+ Result = PromoteOp(ExpandEXTRACT_VECTOR_ELT(Op));
+ break;
+ }
+
+ assert(Result.Val && "Didn't set a result!");
+
+ // Make sure the result is itself legal.
+ Result = LegalizeOp(Result);
+
+ // Remember that we promoted this!
+ AddPromotedOperand(Op, Result);
+ return Result;
+}
+
+/// ExpandEXTRACT_VECTOR_ELT - Expand an EXTRACT_VECTOR_ELT operation into
+/// a legal EXTRACT_VECTOR_ELT operation, scalar code, or memory traffic,
+/// based on the vector type. The return type of this matches the element type
+/// of the vector, which may not be legal for the target.
+SDOperand SelectionDAGLegalize::ExpandEXTRACT_VECTOR_ELT(SDOperand Op) {
+ // We know that operand #0 is the Vec vector. If the index is a constant
+ // or if the invec is a supported hardware type, we can use it. Otherwise,
+ // lower to a store then an indexed load.
+ SDOperand Vec = Op.getOperand(0);
+ SDOperand Idx = Op.getOperand(1);
+
+ SDNode *InVal = Vec.Val;
+ MVT::ValueType TVT = InVal->getValueType(0);
+ unsigned NumElems = MVT::getVectorNumElements(TVT);
+
+ switch (TLI.getOperationAction(ISD::EXTRACT_VECTOR_ELT, TVT)) {
+ default: assert(0 && "This action is not supported yet!");
+ case TargetLowering::Custom: {
+ Vec = LegalizeOp(Vec);
+ Op = DAG.UpdateNodeOperands(Op, Vec, Idx);
+ SDOperand Tmp3 = TLI.LowerOperation(Op, DAG);
+ if (Tmp3.Val)
+ return Tmp3;
+ break;
+ }
+ case TargetLowering::Legal:
+ if (isTypeLegal(TVT)) {
+ Vec = LegalizeOp(Vec);
+ Op = DAG.UpdateNodeOperands(Op, Vec, Idx);
+ Op = LegalizeOp(Op);
+ }
+ break;
+ case TargetLowering::Expand:
+ break;
+ }
+
+ if (NumElems == 1) {
+ // This must be an access of the only element. Return it.
+ Op = ScalarizeVectorOp(Vec);
+ } else if (!TLI.isTypeLegal(TVT) && isa<ConstantSDNode>(Idx)) {
+ ConstantSDNode *CIdx = cast<ConstantSDNode>(Idx);
+ SDOperand Lo, Hi;
+ SplitVectorOp(Vec, Lo, Hi);
+ if (CIdx->getValue() < NumElems/2) {
+ Vec = Lo;
+ } else {
+ Vec = Hi;
+ Idx = DAG.getConstant(CIdx->getValue() - NumElems/2,
+ Idx.getValueType());
+ }
+
+ // It's now an extract from the appropriate high or low part. Recurse.
+ Op = DAG.UpdateNodeOperands(Op, Vec, Idx);
+ Op = ExpandEXTRACT_VECTOR_ELT(Op);
+ } else {
+ // Store the value to a temporary stack slot, then LOAD the scalar
+ // element back out.
+ SDOperand StackPtr = CreateStackTemporary(Vec.getValueType());
+ SDOperand Ch = DAG.getStore(DAG.getEntryNode(), Vec, StackPtr, NULL, 0);
+
+ // Add the offset to the index.
+ unsigned EltSize = MVT::getSizeInBits(Op.getValueType())/8;
+ Idx = DAG.getNode(ISD::MUL, Idx.getValueType(), Idx,
+ DAG.getConstant(EltSize, Idx.getValueType()));
+ StackPtr = DAG.getNode(ISD::ADD, Idx.getValueType(), Idx, StackPtr);
+
+ Op = DAG.getLoad(Op.getValueType(), Ch, StackPtr, NULL, 0);
+ }
+ return Op;
+}
+
+/// ExpandEXTRACT_SUBVECTOR - Expand a EXTRACT_SUBVECTOR operation. For now
+/// we assume the operation can be split if it is not already legal.
+SDOperand SelectionDAGLegalize::ExpandEXTRACT_SUBVECTOR(SDOperand Op) {
+ // We know that operand #0 is the Vec vector. For now we assume the index
+ // is a constant and that the extracted result is a supported hardware type.
+ SDOperand Vec = Op.getOperand(0);
+ SDOperand Idx = LegalizeOp(Op.getOperand(1));
+
+ unsigned NumElems = MVT::getVectorNumElements(Vec.getValueType());
+
+ if (NumElems == MVT::getVectorNumElements(Op.getValueType())) {
+ // This must be an access of the desired vector length. Return it.
+ return Vec;
+ }
+
+ ConstantSDNode *CIdx = cast<ConstantSDNode>(Idx);
+ SDOperand Lo, Hi;
+ SplitVectorOp(Vec, Lo, Hi);
+ if (CIdx->getValue() < NumElems/2) {
+ Vec = Lo;
+ } else {
+ Vec = Hi;
+ Idx = DAG.getConstant(CIdx->getValue() - NumElems/2, Idx.getValueType());
+ }
+
+ // It's now an extract from the appropriate high or low part. Recurse.
+ Op = DAG.UpdateNodeOperands(Op, Vec, Idx);
+ return ExpandEXTRACT_SUBVECTOR(Op);
+}
+
+/// LegalizeSetCCOperands - Attempts to create a legal LHS and RHS for a SETCC
+/// with condition CC on the current target. This usually involves legalizing
+/// or promoting the arguments. In the case where LHS and RHS must be expanded,
+/// there may be no choice but to create a new SetCC node to represent the
+/// legalized value of setcc lhs, rhs. In this case, the value is returned in
+/// LHS, and the SDOperand returned in RHS has a nil SDNode value.
+void SelectionDAGLegalize::LegalizeSetCCOperands(SDOperand &LHS,
+ SDOperand &RHS,
+ SDOperand &CC) {
+ SDOperand Tmp1, Tmp2, Result;
+
+ switch (getTypeAction(LHS.getValueType())) {
+ case Legal:
+ Tmp1 = LegalizeOp(LHS); // LHS
+ Tmp2 = LegalizeOp(RHS); // RHS
+ break;
+ case Promote:
+ Tmp1 = PromoteOp(LHS); // LHS
+ Tmp2 = PromoteOp(RHS); // RHS
+
+ // If this is an FP compare, the operands have already been extended.
+ if (MVT::isInteger(LHS.getValueType())) {
+ MVT::ValueType VT = LHS.getValueType();
+ MVT::ValueType NVT = TLI.getTypeToTransformTo(VT);
+
+ // Otherwise, we have to insert explicit sign or zero extends. Note
+ // that we could insert sign extends for ALL conditions, but zero extend
+ // is cheaper on many machines (an AND instead of two shifts), so prefer
+ // it.
+ switch (cast<CondCodeSDNode>(CC)->get()) {
+ default: assert(0 && "Unknown integer comparison!");
+ case ISD::SETEQ:
+ case ISD::SETNE:
+ case ISD::SETUGE:
+ case ISD::SETUGT:
+ case ISD::SETULE:
+ case ISD::SETULT:
+ // ALL of these operations will work if we either sign or zero extend
+ // the operands (including the unsigned comparisons!). Zero extend is
+ // usually a simpler/cheaper operation, so prefer it.
+ Tmp1 = DAG.getZeroExtendInReg(Tmp1, VT);
+ Tmp2 = DAG.getZeroExtendInReg(Tmp2, VT);
+ break;
+ case ISD::SETGE:
+ case ISD::SETGT:
+ case ISD::SETLT:
+ case ISD::SETLE:
+ Tmp1 = DAG.getNode(ISD::SIGN_EXTEND_INREG, NVT, Tmp1,
+ DAG.getValueType(VT));
+ Tmp2 = DAG.getNode(ISD::SIGN_EXTEND_INREG, NVT, Tmp2,
+ DAG.getValueType(VT));
+ break;
+ }
+ }
+ break;
+ case Expand: {
+ MVT::ValueType VT = LHS.getValueType();
+ if (VT == MVT::f32 || VT == MVT::f64) {
+ // Expand into one or more soft-fp libcall(s).
+ RTLIB::Libcall LC1, LC2 = RTLIB::UNKNOWN_LIBCALL;
+ switch (cast<CondCodeSDNode>(CC)->get()) {
+ case ISD::SETEQ:
+ case ISD::SETOEQ:
+ LC1 = (VT == MVT::f32) ? RTLIB::OEQ_F32 : RTLIB::OEQ_F64;
+ break;
+ case ISD::SETNE:
+ case ISD::SETUNE:
+ LC1 = (VT == MVT::f32) ? RTLIB::UNE_F32 : RTLIB::UNE_F64;
+ break;
+ case ISD::SETGE:
+ case ISD::SETOGE:
+ LC1 = (VT == MVT::f32) ? RTLIB::OGE_F32 : RTLIB::OGE_F64;
+ break;
+ case ISD::SETLT:
+ case ISD::SETOLT:
+ LC1 = (VT == MVT::f32) ? RTLIB::OLT_F32 : RTLIB::OLT_F64;
+ break;
+ case ISD::SETLE:
+ case ISD::SETOLE:
+ LC1 = (VT == MVT::f32) ? RTLIB::OLE_F32 : RTLIB::OLE_F64;
+ break;
+ case ISD::SETGT:
+ case ISD::SETOGT:
+ LC1 = (VT == MVT::f32) ? RTLIB::OGT_F32 : RTLIB::OGT_F64;
+ break;
+ case ISD::SETUO:
+ LC1 = (VT == MVT::f32) ? RTLIB::UO_F32 : RTLIB::UO_F64;
+ break;
+ case ISD::SETO:
+ LC1 = (VT == MVT::f32) ? RTLIB::O_F32 : RTLIB::O_F64;
+ break;
+ default:
+ LC1 = (VT == MVT::f32) ? RTLIB::UO_F32 : RTLIB::UO_F64;
+ switch (cast<CondCodeSDNode>(CC)->get()) {
+ case ISD::SETONE:
+ // SETONE = SETOLT | SETOGT
+ LC1 = (VT == MVT::f32) ? RTLIB::OLT_F32 : RTLIB::OLT_F64;
+ // Fallthrough
+ case ISD::SETUGT:
+ LC2 = (VT == MVT::f32) ? RTLIB::OGT_F32 : RTLIB::OGT_F64;
+ break;
+ case ISD::SETUGE:
+ LC2 = (VT == MVT::f32) ? RTLIB::OGE_F32 : RTLIB::OGE_F64;
+ break;
+ case ISD::SETULT:
+ LC2 = (VT == MVT::f32) ? RTLIB::OLT_F32 : RTLIB::OLT_F64;
+ break;
+ case ISD::SETULE:
+ LC2 = (VT == MVT::f32) ? RTLIB::OLE_F32 : RTLIB::OLE_F64;
+ break;
+ case ISD::SETUEQ:
+ LC2 = (VT == MVT::f32) ? RTLIB::OEQ_F32 : RTLIB::OEQ_F64;
+ break;
+ default: assert(0 && "Unsupported FP setcc!");
+ }
+ }
+
+ SDOperand Dummy;
+ Tmp1 = ExpandLibCall(TLI.getLibcallName(LC1),
+ DAG.getNode(ISD::MERGE_VALUES, VT, LHS, RHS).Val,
+ false /*sign irrelevant*/, Dummy);
+ Tmp2 = DAG.getConstant(0, MVT::i32);
+ CC = DAG.getCondCode(TLI.getCmpLibcallCC(LC1));
+ if (LC2 != RTLIB::UNKNOWN_LIBCALL) {
+ Tmp1 = DAG.getNode(ISD::SETCC, TLI.getSetCCResultTy(), Tmp1, Tmp2, CC);
+ LHS = ExpandLibCall(TLI.getLibcallName(LC2),
+ DAG.getNode(ISD::MERGE_VALUES, VT, LHS, RHS).Val,
+ false /*sign irrelevant*/, Dummy);
+ Tmp2 = DAG.getNode(ISD::SETCC, TLI.getSetCCResultTy(), LHS, Tmp2,
+ DAG.getCondCode(TLI.getCmpLibcallCC(LC2)));
+ Tmp1 = DAG.getNode(ISD::OR, Tmp1.getValueType(), Tmp1, Tmp2);
+ Tmp2 = SDOperand();
+ }
+ LHS = Tmp1;
+ RHS = Tmp2;
+ return;
+ }
+
+ SDOperand LHSLo, LHSHi, RHSLo, RHSHi;
+ ExpandOp(LHS, LHSLo, LHSHi);
+ ExpandOp(RHS, RHSLo, RHSHi);
+ switch (cast<CondCodeSDNode>(CC)->get()) {
+ case ISD::SETEQ:
+ case ISD::SETNE:
+ if (RHSLo == RHSHi)
+ if (ConstantSDNode *RHSCST = dyn_cast<ConstantSDNode>(RHSLo))
+ if (RHSCST->isAllOnesValue()) {
+ // Comparison to -1.
+ Tmp1 = DAG.getNode(ISD::AND, LHSLo.getValueType(), LHSLo, LHSHi);
+ Tmp2 = RHSLo;
+ break;
+ }
+
+ Tmp1 = DAG.getNode(ISD::XOR, LHSLo.getValueType(), LHSLo, RHSLo);
+ Tmp2 = DAG.getNode(ISD::XOR, LHSLo.getValueType(), LHSHi, RHSHi);
+ Tmp1 = DAG.getNode(ISD::OR, Tmp1.getValueType(), Tmp1, Tmp2);
+ Tmp2 = DAG.getConstant(0, Tmp1.getValueType());
+ break;
+ default:
+ // If this is a comparison of the sign bit, just look at the top part.
+ // X > -1, x < 0
+ if (ConstantSDNode *CST = dyn_cast<ConstantSDNode>(RHS))
+ if ((cast<CondCodeSDNode>(CC)->get() == ISD::SETLT &&
+ CST->getValue() == 0) || // X < 0
+ (cast<CondCodeSDNode>(CC)->get() == ISD::SETGT &&
+ CST->isAllOnesValue())) { // X > -1
+ Tmp1 = LHSHi;
+ Tmp2 = RHSHi;
+ break;
+ }
+
+ // FIXME: This generated code sucks.
+ ISD::CondCode LowCC;
+ ISD::CondCode CCCode = cast<CondCodeSDNode>(CC)->get();
+ switch (CCCode) {
+ default: assert(0 && "Unknown integer setcc!");
+ case ISD::SETLT:
+ case ISD::SETULT: LowCC = ISD::SETULT; break;
+ case ISD::SETGT:
+ case ISD::SETUGT: LowCC = ISD::SETUGT; break;
+ case ISD::SETLE:
+ case ISD::SETULE: LowCC = ISD::SETULE; break;
+ case ISD::SETGE:
+ case ISD::SETUGE: LowCC = ISD::SETUGE; break;
+ }
+
+ // Tmp1 = lo(op1) < lo(op2) // Always unsigned comparison
+ // Tmp2 = hi(op1) < hi(op2) // Signedness depends on operands
+ // dest = hi(op1) == hi(op2) ? Tmp1 : Tmp2;
+
+ // NOTE: on targets without efficient SELECT of bools, we can always use
+ // this identity: (B1 ? B2 : B3) --> (B1 & B2)|(!B1&B3)
+ TargetLowering::DAGCombinerInfo DagCombineInfo(DAG, false, true, NULL);
+ Tmp1 = TLI.SimplifySetCC(TLI.getSetCCResultTy(), LHSLo, RHSLo, LowCC,
+ false, DagCombineInfo);
+ if (!Tmp1.Val)
+ Tmp1 = DAG.getSetCC(TLI.getSetCCResultTy(), LHSLo, RHSLo, LowCC);
+ Tmp2 = TLI.SimplifySetCC(TLI.getSetCCResultTy(), LHSHi, RHSHi,
+ CCCode, false, DagCombineInfo);
+ if (!Tmp2.Val)
+ Tmp2 = DAG.getNode(ISD::SETCC, TLI.getSetCCResultTy(), LHSHi, RHSHi, CC);
+
+ ConstantSDNode *Tmp1C = dyn_cast<ConstantSDNode>(Tmp1.Val);
+ ConstantSDNode *Tmp2C = dyn_cast<ConstantSDNode>(Tmp2.Val);
+ if ((Tmp1C && Tmp1C->getValue() == 0) ||
+ (Tmp2C && Tmp2C->getValue() == 0 &&
+ (CCCode == ISD::SETLE || CCCode == ISD::SETGE ||
+ CCCode == ISD::SETUGE || CCCode == ISD::SETULE)) ||
+ (Tmp2C && Tmp2C->getValue() == 1 &&
+ (CCCode == ISD::SETLT || CCCode == ISD::SETGT ||
+ CCCode == ISD::SETUGT || CCCode == ISD::SETULT))) {
+ // low part is known false, returns high part.
+ // For LE / GE, if high part is known false, ignore the low part.
+ // For LT / GT, if high part is known true, ignore the low part.
+ Tmp1 = Tmp2;
+ Tmp2 = SDOperand();
+ } else {
+ Result = TLI.SimplifySetCC(TLI.getSetCCResultTy(), LHSHi, RHSHi,
+ ISD::SETEQ, false, DagCombineInfo);
+ if (!Result.Val)
+ Result=DAG.getSetCC(TLI.getSetCCResultTy(), LHSHi, RHSHi, ISD::SETEQ);
+ Result = LegalizeOp(DAG.getNode(ISD::SELECT, Tmp1.getValueType(),
+ Result, Tmp1, Tmp2));
+ Tmp1 = Result;
+ Tmp2 = SDOperand();
+ }
+ }
+ }
+ }
+ LHS = Tmp1;
+ RHS = Tmp2;
+}
+
+/// ExpandBIT_CONVERT - Expand a BIT_CONVERT node into a store/load combination.
+/// The resultant code need not be legal. Note that SrcOp is the input operand
+/// to the BIT_CONVERT, not the BIT_CONVERT node itself.
+SDOperand SelectionDAGLegalize::ExpandBIT_CONVERT(MVT::ValueType DestVT,
+ SDOperand SrcOp) {
+ // Create the stack frame object.
+ SDOperand FIPtr = CreateStackTemporary(DestVT);
+
+ // Emit a store to the stack slot.
+ SDOperand Store = DAG.getStore(DAG.getEntryNode(), SrcOp, FIPtr, NULL, 0);
+ // Result is a load from the stack slot.
+ return DAG.getLoad(DestVT, Store, FIPtr, NULL, 0);
+}
+
+SDOperand SelectionDAGLegalize::ExpandSCALAR_TO_VECTOR(SDNode *Node) {
+ // Create a vector sized/aligned stack slot, store the value to element #0,
+ // then load the whole vector back out.
+ SDOperand StackPtr = CreateStackTemporary(Node->getValueType(0));
+ SDOperand Ch = DAG.getStore(DAG.getEntryNode(), Node->getOperand(0), StackPtr,
+ NULL, 0);
+ return DAG.getLoad(Node->getValueType(0), Ch, StackPtr, NULL, 0);
+}
+
+
+/// ExpandBUILD_VECTOR - Expand a BUILD_VECTOR node on targets that don't
+/// support the operation, but do support the resultant vector type.
+SDOperand SelectionDAGLegalize::ExpandBUILD_VECTOR(SDNode *Node) {
+
+ // If the only non-undef value is the low element, turn this into a
+ // SCALAR_TO_VECTOR node. If this is { X, X, X, X }, determine X.
+ unsigned NumElems = Node->getNumOperands();
+ bool isOnlyLowElement = true;
+ SDOperand SplatValue = Node->getOperand(0);
+ std::map<SDOperand, std::vector<unsigned> > Values;
+ Values[SplatValue].push_back(0);
+ bool isConstant = true;
+ if (!isa<ConstantFPSDNode>(SplatValue) && !isa<ConstantSDNode>(SplatValue) &&
+ SplatValue.getOpcode() != ISD::UNDEF)
+ isConstant = false;
+
+ for (unsigned i = 1; i < NumElems; ++i) {
+ SDOperand V = Node->getOperand(i);
+ Values[V].push_back(i);
+ if (V.getOpcode() != ISD::UNDEF)
+ isOnlyLowElement = false;
+ if (SplatValue != V)
+ SplatValue = SDOperand(0,0);
+
+ // If this isn't a constant element or an undef, we can't use a constant
+ // pool load.
+ if (!isa<ConstantFPSDNode>(V) && !isa<ConstantSDNode>(V) &&
+ V.getOpcode() != ISD::UNDEF)
+ isConstant = false;
+ }
+
+ if (isOnlyLowElement) {
+ // If the low element is an undef too, then this whole things is an undef.
+ if (Node->getOperand(0).getOpcode() == ISD::UNDEF)
+ return DAG.getNode(ISD::UNDEF, Node->getValueType(0));
+ // Otherwise, turn this into a scalar_to_vector node.
+ return DAG.getNode(ISD::SCALAR_TO_VECTOR, Node->getValueType(0),
+ Node->getOperand(0));
+ }
+
+ // If all elements are constants, create a load from the constant pool.
+ if (isConstant) {
+ MVT::ValueType VT = Node->getValueType(0);
+ const Type *OpNTy =
+ MVT::getTypeForValueType(Node->getOperand(0).getValueType());
+ std::vector<Constant*> CV;
+ for (unsigned i = 0, e = NumElems; i != e; ++i) {
+ if (ConstantFPSDNode *V =
+ dyn_cast<ConstantFPSDNode>(Node->getOperand(i))) {
+ CV.push_back(ConstantFP::get(OpNTy, V->getValue()));
+ } else if (ConstantSDNode *V =
+ dyn_cast<ConstantSDNode>(Node->getOperand(i))) {
+ CV.push_back(ConstantInt::get(OpNTy, V->getValue()));
+ } else {
+ assert(Node->getOperand(i).getOpcode() == ISD::UNDEF);
+ CV.push_back(UndefValue::get(OpNTy));
+ }
+ }
+ Constant *CP = ConstantVector::get(CV);
+ SDOperand CPIdx = DAG.getConstantPool(CP, TLI.getPointerTy());
+ return DAG.getLoad(VT, DAG.getEntryNode(), CPIdx, NULL, 0);
+ }
+
+ if (SplatValue.Val) { // Splat of one value?
+ // Build the shuffle constant vector: <0, 0, 0, 0>
+ MVT::ValueType MaskVT =
+ MVT::getIntVectorWithNumElements(NumElems);
+ SDOperand Zero = DAG.getConstant(0, MVT::getVectorElementType(MaskVT));
+ std::vector<SDOperand> ZeroVec(NumElems, Zero);
+ SDOperand SplatMask = DAG.getNode(ISD::BUILD_VECTOR, MaskVT,
+ &ZeroVec[0], ZeroVec.size());
+
+ // If the target supports VECTOR_SHUFFLE and this shuffle mask, use it.
+ if (isShuffleLegal(Node->getValueType(0), SplatMask)) {
+ // Get the splatted value into the low element of a vector register.
+ SDOperand LowValVec =
+ DAG.getNode(ISD::SCALAR_TO_VECTOR, Node->getValueType(0), SplatValue);
+
+ // Return shuffle(LowValVec, undef, <0,0,0,0>)
+ return DAG.getNode(ISD::VECTOR_SHUFFLE, Node->getValueType(0), LowValVec,
+ DAG.getNode(ISD::UNDEF, Node->getValueType(0)),
+ SplatMask);
+ }
+ }
+
+ // If there are only two unique elements, we may be able to turn this into a
+ // vector shuffle.
+ if (Values.size() == 2) {
+ // Build the shuffle constant vector: e.g. <0, 4, 0, 4>
+ MVT::ValueType MaskVT =
+ MVT::getIntVectorWithNumElements(NumElems);
+ std::vector<SDOperand> MaskVec(NumElems);
+ unsigned i = 0;
+ for (std::map<SDOperand,std::vector<unsigned> >::iterator I=Values.begin(),
+ E = Values.end(); I != E; ++I) {
+ for (std::vector<unsigned>::iterator II = I->second.begin(),
+ EE = I->second.end(); II != EE; ++II)
+ MaskVec[*II] = DAG.getConstant(i, MVT::getVectorElementType(MaskVT));
+ i += NumElems;
+ }
+ SDOperand ShuffleMask = DAG.getNode(ISD::BUILD_VECTOR, MaskVT,
+ &MaskVec[0], MaskVec.size());
+
+ // If the target supports VECTOR_SHUFFLE and this shuffle mask, use it.
+ if (TLI.isOperationLegal(ISD::SCALAR_TO_VECTOR, Node->getValueType(0)) &&
+ isShuffleLegal(Node->getValueType(0), ShuffleMask)) {
+ SmallVector<SDOperand, 8> Ops;
+ for(std::map<SDOperand,std::vector<unsigned> >::iterator I=Values.begin(),
+ E = Values.end(); I != E; ++I) {
+ SDOperand Op = DAG.getNode(ISD::SCALAR_TO_VECTOR, Node->getValueType(0),
+ I->first);
+ Ops.push_back(Op);
+ }
+ Ops.push_back(ShuffleMask);
+
+ // Return shuffle(LoValVec, HiValVec, <0,1,0,1>)
+ return DAG.getNode(ISD::VECTOR_SHUFFLE, Node->getValueType(0),
+ &Ops[0], Ops.size());
+ }
+ }
+
+ // Otherwise, we can't handle this case efficiently. Allocate a sufficiently
+ // aligned object on the stack, store each element into it, then load
+ // the result as a vector.
+ MVT::ValueType VT = Node->getValueType(0);
+ // Create the stack frame object.
+ SDOperand FIPtr = CreateStackTemporary(VT);
+
+ // Emit a store of each element to the stack slot.
+ SmallVector<SDOperand, 8> Stores;
+ unsigned TypeByteSize =
+ MVT::getSizeInBits(Node->getOperand(0).getValueType())/8;
+ // Store (in the right endianness) the elements to memory.
+ for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) {
+ // Ignore undef elements.
+ if (Node->getOperand(i).getOpcode() == ISD::UNDEF) continue;
+
+ unsigned Offset = TypeByteSize*i;
+
+ SDOperand Idx = DAG.getConstant(Offset, FIPtr.getValueType());
+ Idx = DAG.getNode(ISD::ADD, FIPtr.getValueType(), FIPtr, Idx);
+
+ Stores.push_back(DAG.getStore(DAG.getEntryNode(), Node->getOperand(i), Idx,
+ NULL, 0));
+ }
+
+ SDOperand StoreChain;
+ if (!Stores.empty()) // Not all undef elements?
+ StoreChain = DAG.getNode(ISD::TokenFactor, MVT::Other,
+ &Stores[0], Stores.size());
+ else
+ StoreChain = DAG.getEntryNode();
+
+ // Result is a load from the stack slot.
+ return DAG.getLoad(VT, StoreChain, FIPtr, NULL, 0);
+}
+
+/// CreateStackTemporary - Create a stack temporary, suitable for holding the
+/// specified value type.
+SDOperand SelectionDAGLegalize::CreateStackTemporary(MVT::ValueType VT) {
+ MachineFrameInfo *FrameInfo = DAG.getMachineFunction().getFrameInfo();
+ unsigned ByteSize = MVT::getSizeInBits(VT)/8;
+ const Type *Ty = MVT::getTypeForValueType(VT);
+ unsigned StackAlign = (unsigned)TLI.getTargetData()->getPrefTypeAlignment(Ty);
+ int FrameIdx = FrameInfo->CreateStackObject(ByteSize, StackAlign);
+ return DAG.getFrameIndex(FrameIdx, TLI.getPointerTy());
+}
+
+void SelectionDAGLegalize::ExpandShiftParts(unsigned NodeOp,
+ SDOperand Op, SDOperand Amt,
+ SDOperand &Lo, SDOperand &Hi) {
+ // Expand the subcomponents.
+ SDOperand LHSL, LHSH;
+ ExpandOp(Op, LHSL, LHSH);
+
+ SDOperand Ops[] = { LHSL, LHSH, Amt };
+ MVT::ValueType VT = LHSL.getValueType();
+ Lo = DAG.getNode(NodeOp, DAG.getNodeValueTypes(VT, VT), 2, Ops, 3);
+ Hi = Lo.getValue(1);
+}
+
+
+/// ExpandShift - Try to find a clever way to expand this shift operation out to
+/// smaller elements. If we can't find a way that is more efficient than a
+/// libcall on this target, return false. Otherwise, return true with the
+/// low-parts expanded into Lo and Hi.
+bool SelectionDAGLegalize::ExpandShift(unsigned Opc, SDOperand Op,SDOperand Amt,
+ SDOperand &Lo, SDOperand &Hi) {
+ assert((Opc == ISD::SHL || Opc == ISD::SRA || Opc == ISD::SRL) &&
+ "This is not a shift!");
+
+ MVT::ValueType NVT = TLI.getTypeToTransformTo(Op.getValueType());
+ SDOperand ShAmt = LegalizeOp(Amt);
+ MVT::ValueType ShTy = ShAmt.getValueType();
+ unsigned VTBits = MVT::getSizeInBits(Op.getValueType());
+ unsigned NVTBits = MVT::getSizeInBits(NVT);
+
+ // Handle the case when Amt is an immediate. Other cases are currently broken
+ // and are disabled.
+ if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Amt.Val)) {
+ unsigned Cst = CN->getValue();
+ // Expand the incoming operand to be shifted, so that we have its parts
+ SDOperand InL, InH;
+ ExpandOp(Op, InL, InH);
+ switch(Opc) {
+ case ISD::SHL:
+ if (Cst > VTBits) {
+ Lo = DAG.getConstant(0, NVT);
+ Hi = DAG.getConstant(0, NVT);
+ } else if (Cst > NVTBits) {
+ Lo = DAG.getConstant(0, NVT);
+ Hi = DAG.getNode(ISD::SHL, NVT, InL, DAG.getConstant(Cst-NVTBits,ShTy));
+ } else if (Cst == NVTBits) {
+ Lo = DAG.getConstant(0, NVT);
+ Hi = InL;
+ } else {
+ Lo = DAG.getNode(ISD::SHL, NVT, InL, DAG.getConstant(Cst, ShTy));
+ Hi = DAG.getNode(ISD::OR, NVT,
+ DAG.getNode(ISD::SHL, NVT, InH, DAG.getConstant(Cst, ShTy)),
+ DAG.getNode(ISD::SRL, NVT, InL, DAG.getConstant(NVTBits-Cst, ShTy)));
+ }
+ return true;
+ case ISD::SRL:
+ if (Cst > VTBits) {
+ Lo = DAG.getConstant(0, NVT);
+ Hi = DAG.getConstant(0, NVT);
+ } else if (Cst > NVTBits) {
+ Lo = DAG.getNode(ISD::SRL, NVT, InH, DAG.getConstant(Cst-NVTBits,ShTy));
+ Hi = DAG.getConstant(0, NVT);
+ } else if (Cst == NVTBits) {
+ Lo = InH;
+ Hi = DAG.getConstant(0, NVT);
+ } else {
+ Lo = DAG.getNode(ISD::OR, NVT,
+ DAG.getNode(ISD::SRL, NVT, InL, DAG.getConstant(Cst, ShTy)),
+ DAG.getNode(ISD::SHL, NVT, InH, DAG.getConstant(NVTBits-Cst, ShTy)));
+ Hi = DAG.getNode(ISD::SRL, NVT, InH, DAG.getConstant(Cst, ShTy));
+ }
+ return true;
+ case ISD::SRA:
+ if (Cst > VTBits) {
+ Hi = Lo = DAG.getNode(ISD::SRA, NVT, InH,
+ DAG.getConstant(NVTBits-1, ShTy));
+ } else if (Cst > NVTBits) {
+ Lo = DAG.getNode(ISD::SRA, NVT, InH,
+ DAG.getConstant(Cst-NVTBits, ShTy));
+ Hi = DAG.getNode(ISD::SRA, NVT, InH,
+ DAG.getConstant(NVTBits-1, ShTy));
+ } else if (Cst == NVTBits) {
+ Lo = InH;
+ Hi = DAG.getNode(ISD::SRA, NVT, InH,
+ DAG.getConstant(NVTBits-1, ShTy));
+ } else {
+ Lo = DAG.getNode(ISD::OR, NVT,
+ DAG.getNode(ISD::SRL, NVT, InL, DAG.getConstant(Cst, ShTy)),
+ DAG.getNode(ISD::SHL, NVT, InH, DAG.getConstant(NVTBits-Cst, ShTy)));
+ Hi = DAG.getNode(ISD::SRA, NVT, InH, DAG.getConstant(Cst, ShTy));
+ }
+ return true;
+ }
+ }
+
+ // Okay, the shift amount isn't constant. However, if we can tell that it is
+ // >= 32 or < 32, we can still simplify it, without knowing the actual value.
+ uint64_t Mask = NVTBits, KnownZero, KnownOne;
+ DAG.ComputeMaskedBits(Amt, Mask, KnownZero, KnownOne);
+
+ // If we know that the high bit of the shift amount is one, then we can do
+ // this as a couple of simple shifts.
+ if (KnownOne & Mask) {
+ // Mask out the high bit, which we know is set.
+ Amt = DAG.getNode(ISD::AND, Amt.getValueType(), Amt,
+ DAG.getConstant(NVTBits-1, Amt.getValueType()));
+
+ // Expand the incoming operand to be shifted, so that we have its parts
+ SDOperand InL, InH;
+ ExpandOp(Op, InL, InH);
+ switch(Opc) {
+ case ISD::SHL:
+ Lo = DAG.getConstant(0, NVT); // Low part is zero.
+ Hi = DAG.getNode(ISD::SHL, NVT, InL, Amt); // High part from Lo part.
+ return true;
+ case ISD::SRL:
+ Hi = DAG.getConstant(0, NVT); // Hi part is zero.
+ Lo = DAG.getNode(ISD::SRL, NVT, InH, Amt); // Lo part from Hi part.
+ return true;
+ case ISD::SRA:
+ Hi = DAG.getNode(ISD::SRA, NVT, InH, // Sign extend high part.
+ DAG.getConstant(NVTBits-1, Amt.getValueType()));
+ Lo = DAG.getNode(ISD::SRA, NVT, InH, Amt); // Lo part from Hi part.
+ return true;
+ }
+ }
+
+ // If we know that the high bit of the shift amount is zero, then we can do
+ // this as a couple of simple shifts.
+ if (KnownZero & Mask) {
+ // Compute 32-amt.
+ SDOperand Amt2 = DAG.getNode(ISD::SUB, Amt.getValueType(),
+ DAG.getConstant(NVTBits, Amt.getValueType()),
+ Amt);
+
+ // Expand the incoming operand to be shifted, so that we have its parts
+ SDOperand InL, InH;
+ ExpandOp(Op, InL, InH);
+ switch(Opc) {
+ case ISD::SHL:
+ Lo = DAG.getNode(ISD::SHL, NVT, InL, Amt);
+ Hi = DAG.getNode(ISD::OR, NVT,
+ DAG.getNode(ISD::SHL, NVT, InH, Amt),
+ DAG.getNode(ISD::SRL, NVT, InL, Amt2));
+ return true;
+ case ISD::SRL:
+ Hi = DAG.getNode(ISD::SRL, NVT, InH, Amt);
+ Lo = DAG.getNode(ISD::OR, NVT,
+ DAG.getNode(ISD::SRL, NVT, InL, Amt),
+ DAG.getNode(ISD::SHL, NVT, InH, Amt2));
+ return true;
+ case ISD::SRA:
+ Hi = DAG.getNode(ISD::SRA, NVT, InH, Amt);
+ Lo = DAG.getNode(ISD::OR, NVT,
+ DAG.getNode(ISD::SRL, NVT, InL, Amt),
+ DAG.getNode(ISD::SHL, NVT, InH, Amt2));
+ return true;
+ }
+ }
+
+ return false;
+}
+
+
+// ExpandLibCall - Expand a node into a call to a libcall. If the result value
+// does not fit into a register, return the lo part and set the hi part to the
+// by-reg argument. If it does fit into a single register, return the result
+// and leave the Hi part unset.
+SDOperand SelectionDAGLegalize::ExpandLibCall(const char *Name, SDNode *Node,
+ bool isSigned, SDOperand &Hi) {
+ assert(!IsLegalizingCall && "Cannot overlap legalization of calls!");
+ // The input chain to this libcall is the entry node of the function.
+ // Legalizing the call will automatically add the previous call to the
+ // dependence.
+ SDOperand InChain = DAG.getEntryNode();
+
+ TargetLowering::ArgListTy Args;
+ TargetLowering::ArgListEntry Entry;
+ for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) {
+ MVT::ValueType ArgVT = Node->getOperand(i).getValueType();
+ const Type *ArgTy = MVT::getTypeForValueType(ArgVT);
+ Entry.Node = Node->getOperand(i); Entry.Ty = ArgTy;
+ Entry.isSExt = isSigned;
+ Args.push_back(Entry);
+ }
+ SDOperand Callee = DAG.getExternalSymbol(Name, TLI.getPointerTy());
+
+ // Splice the libcall in wherever FindInputOutputChains tells us to.
+ const Type *RetTy = MVT::getTypeForValueType(Node->getValueType(0));
+ std::pair<SDOperand,SDOperand> CallInfo =
+ TLI.LowerCallTo(InChain, RetTy, isSigned, false, CallingConv::C, false,
+ Callee, Args, DAG);
+
+ // Legalize the call sequence, starting with the chain. This will advance
+ // the LastCALLSEQ_END to the legalized version of the CALLSEQ_END node that
+ // was added by LowerCallTo (guaranteeing proper serialization of calls).
+ LegalizeOp(CallInfo.second);
+ SDOperand Result;
+ switch (getTypeAction(CallInfo.first.getValueType())) {
+ default: assert(0 && "Unknown thing");
+ case Legal:
+ Result = CallInfo.first;
+ break;
+ case Expand:
+ ExpandOp(CallInfo.first, Result, Hi);
+ break;
+ }
+ return Result;
+}
+
+
+/// ExpandIntToFP - Expand a [US]INT_TO_FP operation.
+///
+SDOperand SelectionDAGLegalize::
+ExpandIntToFP(bool isSigned, MVT::ValueType DestTy, SDOperand Source) {
+ assert(getTypeAction(Source.getValueType()) == Expand &&
+ "This is not an expansion!");
+ assert(Source.getValueType() == MVT::i64 && "Only handle expand from i64!");
+
+ if (!isSigned) {
+ assert(Source.getValueType() == MVT::i64 &&
+ "This only works for 64-bit -> FP");
+ // The 64-bit value loaded will be incorrectly if the 'sign bit' of the
+ // incoming integer is set. To handle this, we dynamically test to see if
+ // it is set, and, if so, add a fudge factor.
+ SDOperand Lo, Hi;
+ ExpandOp(Source, Lo, Hi);
+
+ // If this is unsigned, and not supported, first perform the conversion to
+ // signed, then adjust the result if the sign bit is set.
+ SDOperand SignedConv = ExpandIntToFP(true, DestTy,
+ DAG.getNode(ISD::BUILD_PAIR, Source.getValueType(), Lo, Hi));
+
+ SDOperand SignSet = DAG.getSetCC(TLI.getSetCCResultTy(), Hi,
+ DAG.getConstant(0, Hi.getValueType()),
+ ISD::SETLT);
+ SDOperand Zero = getIntPtrConstant(0), Four = getIntPtrConstant(4);
+ SDOperand CstOffset = DAG.getNode(ISD::SELECT, Zero.getValueType(),
+ SignSet, Four, Zero);
+ uint64_t FF = 0x5f800000ULL;
+ if (TLI.isLittleEndian()) FF <<= 32;
+ static Constant *FudgeFactor = ConstantInt::get(Type::Int64Ty, FF);
+
+ SDOperand CPIdx = DAG.getConstantPool(FudgeFactor, TLI.getPointerTy());
+ CPIdx = DAG.getNode(ISD::ADD, TLI.getPointerTy(), CPIdx, CstOffset);
+ SDOperand FudgeInReg;
+ if (DestTy == MVT::f32)
+ FudgeInReg = DAG.getLoad(MVT::f32, DAG.getEntryNode(), CPIdx, NULL, 0);
+ else {
+ assert(DestTy == MVT::f64 && "Unexpected conversion");
+ // FIXME: Avoid the extend by construction the right constantpool?
+ FudgeInReg = DAG.getExtLoad(ISD::EXTLOAD, MVT::f64, DAG.getEntryNode(),
+ CPIdx, NULL, 0, MVT::f32);
+ }
+ MVT::ValueType SCVT = SignedConv.getValueType();
+ if (SCVT != DestTy) {
+ // Destination type needs to be expanded as well. The FADD now we are
+ // constructing will be expanded into a libcall.
+ if (MVT::getSizeInBits(SCVT) != MVT::getSizeInBits(DestTy)) {
+ assert(SCVT == MVT::i32 && DestTy == MVT::f64);
+ SignedConv = DAG.getNode(ISD::BUILD_PAIR, MVT::i64,
+ SignedConv, SignedConv.getValue(1));
+ }
+ SignedConv = DAG.getNode(ISD::BIT_CONVERT, DestTy, SignedConv);
+ }
+ return DAG.getNode(ISD::FADD, DestTy, SignedConv, FudgeInReg);
+ }
+
+ // Check to see if the target has a custom way to lower this. If so, use it.
+ switch (TLI.getOperationAction(ISD::SINT_TO_FP, Source.getValueType())) {
+ default: assert(0 && "This action not implemented for this operation!");
+ case TargetLowering::Legal:
+ case TargetLowering::Expand:
+ break; // This case is handled below.
+ case TargetLowering::Custom: {
+ SDOperand NV = TLI.LowerOperation(DAG.getNode(ISD::SINT_TO_FP, DestTy,
+ Source), DAG);
+ if (NV.Val)
+ return LegalizeOp(NV);
+ break; // The target decided this was legal after all
+ }
+ }
+
+ // Expand the source, then glue it back together for the call. We must expand
+ // the source in case it is shared (this pass of legalize must traverse it).
+ SDOperand SrcLo, SrcHi;
+ ExpandOp(Source, SrcLo, SrcHi);
+ Source = DAG.getNode(ISD::BUILD_PAIR, Source.getValueType(), SrcLo, SrcHi);
+
+ RTLIB::Libcall LC;
+ if (DestTy == MVT::f32)
+ LC = RTLIB::SINTTOFP_I64_F32;
+ else {
+ assert(DestTy == MVT::f64 && "Unknown fp value type!");
+ LC = RTLIB::SINTTOFP_I64_F64;
+ }
+
+ assert(TLI.getLibcallName(LC) && "Don't know how to expand this SINT_TO_FP!");
+ Source = DAG.getNode(ISD::SINT_TO_FP, DestTy, Source);
+ SDOperand UnusedHiPart;
+ return ExpandLibCall(TLI.getLibcallName(LC), Source.Val, isSigned,
+ UnusedHiPart);
+}
+
+/// ExpandLegalINT_TO_FP - This function is responsible for legalizing a
+/// INT_TO_FP operation of the specified operand when the target requests that
+/// we expand it. At this point, we know that the result and operand types are
+/// legal for the target.
+SDOperand SelectionDAGLegalize::ExpandLegalINT_TO_FP(bool isSigned,
+ SDOperand Op0,
+ MVT::ValueType DestVT) {
+ if (Op0.getValueType() == MVT::i32) {
+ // simple 32-bit [signed|unsigned] integer to float/double expansion
+
+ // get the stack frame index of a 8 byte buffer, pessimistically aligned
+ MachineFunction &MF = DAG.getMachineFunction();
+ const Type *F64Type = MVT::getTypeForValueType(MVT::f64);
+ unsigned StackAlign =
+ (unsigned)TLI.getTargetData()->getPrefTypeAlignment(F64Type);
+ int SSFI = MF.getFrameInfo()->CreateStackObject(8, StackAlign);
+ // get address of 8 byte buffer
+ SDOperand StackSlot = DAG.getFrameIndex(SSFI, TLI.getPointerTy());
+ // word offset constant for Hi/Lo address computation
+ SDOperand WordOff = DAG.getConstant(sizeof(int), TLI.getPointerTy());
+ // set up Hi and Lo (into buffer) address based on endian
+ SDOperand Hi = StackSlot;
+ SDOperand Lo = DAG.getNode(ISD::ADD, TLI.getPointerTy(), StackSlot,WordOff);
+ if (TLI.isLittleEndian())
+ std::swap(Hi, Lo);
+
+ // if signed map to unsigned space
+ SDOperand Op0Mapped;
+ if (isSigned) {
+ // constant used to invert sign bit (signed to unsigned mapping)
+ SDOperand SignBit = DAG.getConstant(0x80000000u, MVT::i32);
+ Op0Mapped = DAG.getNode(ISD::XOR, MVT::i32, Op0, SignBit);
+ } else {
+ Op0Mapped = Op0;
+ }
+ // store the lo of the constructed double - based on integer input
+ SDOperand Store1 = DAG.getStore(DAG.getEntryNode(),
+ Op0Mapped, Lo, NULL, 0);
+ // initial hi portion of constructed double
+ SDOperand InitialHi = DAG.getConstant(0x43300000u, MVT::i32);
+ // store the hi of the constructed double - biased exponent
+ SDOperand Store2=DAG.getStore(Store1, InitialHi, Hi, NULL, 0);
+ // load the constructed double
+ SDOperand Load = DAG.getLoad(MVT::f64, Store2, StackSlot, NULL, 0);
+ // FP constant to bias correct the final result
+ SDOperand Bias = DAG.getConstantFP(isSigned ?
+ BitsToDouble(0x4330000080000000ULL)
+ : BitsToDouble(0x4330000000000000ULL),
+ MVT::f64);
+ // subtract the bias
+ SDOperand Sub = DAG.getNode(ISD::FSUB, MVT::f64, Load, Bias);
+ // final result
+ SDOperand Result;
+ // handle final rounding
+ if (DestVT == MVT::f64) {
+ // do nothing
+ Result = Sub;
+ } else {
+ // if f32 then cast to f32
+ Result = DAG.getNode(ISD::FP_ROUND, MVT::f32, Sub);
+ }
+ return Result;
+ }
+ assert(!isSigned && "Legalize cannot Expand SINT_TO_FP for i64 yet");
+ SDOperand Tmp1 = DAG.getNode(ISD::SINT_TO_FP, DestVT, Op0);
+
+ SDOperand SignSet = DAG.getSetCC(TLI.getSetCCResultTy(), Op0,
+ DAG.getConstant(0, Op0.getValueType()),
+ ISD::SETLT);
+ SDOperand Zero = getIntPtrConstant(0), Four = getIntPtrConstant(4);
+ SDOperand CstOffset = DAG.getNode(ISD::SELECT, Zero.getValueType(),
+ SignSet, Four, Zero);
+
+ // If the sign bit of the integer is set, the large number will be treated
+ // as a negative number. To counteract this, the dynamic code adds an
+ // offset depending on the data type.
+ uint64_t FF;
+ switch (Op0.getValueType()) {
+ default: assert(0 && "Unsupported integer type!");
+ case MVT::i8 : FF = 0x43800000ULL; break; // 2^8 (as a float)
+ case MVT::i16: FF = 0x47800000ULL; break; // 2^16 (as a float)
+ case MVT::i32: FF = 0x4F800000ULL; break; // 2^32 (as a float)
+ case MVT::i64: FF = 0x5F800000ULL; break; // 2^64 (as a float)
+ }
+ if (TLI.isLittleEndian()) FF <<= 32;
+ static Constant *FudgeFactor = ConstantInt::get(Type::Int64Ty, FF);
+
+ SDOperand CPIdx = DAG.getConstantPool(FudgeFactor, TLI.getPointerTy());
+ CPIdx = DAG.getNode(ISD::ADD, TLI.getPointerTy(), CPIdx, CstOffset);
+ SDOperand FudgeInReg;
+ if (DestVT == MVT::f32)
+ FudgeInReg = DAG.getLoad(MVT::f32, DAG.getEntryNode(), CPIdx, NULL, 0);
+ else {
+ assert(DestVT == MVT::f64 && "Unexpected conversion");
+ FudgeInReg = LegalizeOp(DAG.getExtLoad(ISD::EXTLOAD, MVT::f64,
+ DAG.getEntryNode(), CPIdx,
+ NULL, 0, MVT::f32));
+ }
+
+ return DAG.getNode(ISD::FADD, DestVT, Tmp1, FudgeInReg);
+}
+
+/// PromoteLegalINT_TO_FP - This function is responsible for legalizing a
+/// *INT_TO_FP operation of the specified operand when the target requests that
+/// we promote it. At this point, we know that the result and operand types are
+/// legal for the target, and that there is a legal UINT_TO_FP or SINT_TO_FP
+/// operation that takes a larger input.
+SDOperand SelectionDAGLegalize::PromoteLegalINT_TO_FP(SDOperand LegalOp,
+ MVT::ValueType DestVT,
+ bool isSigned) {
+ // First step, figure out the appropriate *INT_TO_FP operation to use.
+ MVT::ValueType NewInTy = LegalOp.getValueType();
+
+ unsigned OpToUse = 0;
+
+ // Scan for the appropriate larger type to use.
+ while (1) {
+ NewInTy = (MVT::ValueType)(NewInTy+1);
+ assert(MVT::isInteger(NewInTy) && "Ran out of possibilities!");
+
+ // If the target supports SINT_TO_FP of this type, use it.
+ switch (TLI.getOperationAction(ISD::SINT_TO_FP, NewInTy)) {
+ default: break;
+ case TargetLowering::Legal:
+ if (!TLI.isTypeLegal(NewInTy))
+ break; // Can't use this datatype.
+ // FALL THROUGH.
+ case TargetLowering::Custom:
+ OpToUse = ISD::SINT_TO_FP;
+ break;
+ }
+ if (OpToUse) break;
+ if (isSigned) continue;
+
+ // If the target supports UINT_TO_FP of this type, use it.
+ switch (TLI.getOperationAction(ISD::UINT_TO_FP, NewInTy)) {
+ default: break;
+ case TargetLowering::Legal:
+ if (!TLI.isTypeLegal(NewInTy))
+ break; // Can't use this datatype.
+ // FALL THROUGH.
+ case TargetLowering::Custom:
+ OpToUse = ISD::UINT_TO_FP;
+ break;
+ }
+ if (OpToUse) break;
+
+ // Otherwise, try a larger type.
+ }
+
+ // Okay, we found the operation and type to use. Zero extend our input to the
+ // desired type then run the operation on it.
+ return DAG.getNode(OpToUse, DestVT,
+ DAG.getNode(isSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND,
+ NewInTy, LegalOp));
+}
+
+/// PromoteLegalFP_TO_INT - This function is responsible for legalizing a
+/// FP_TO_*INT operation of the specified operand when the target requests that
+/// we promote it. At this point, we know that the result and operand types are
+/// legal for the target, and that there is a legal FP_TO_UINT or FP_TO_SINT
+/// operation that returns a larger result.
+SDOperand SelectionDAGLegalize::PromoteLegalFP_TO_INT(SDOperand LegalOp,
+ MVT::ValueType DestVT,
+ bool isSigned) {
+ // First step, figure out the appropriate FP_TO*INT operation to use.
+ MVT::ValueType NewOutTy = DestVT;
+
+ unsigned OpToUse = 0;
+
+ // Scan for the appropriate larger type to use.
+ while (1) {
+ NewOutTy = (MVT::ValueType)(NewOutTy+1);
+ assert(MVT::isInteger(NewOutTy) && "Ran out of possibilities!");
+
+ // If the target supports FP_TO_SINT returning this type, use it.
+ switch (TLI.getOperationAction(ISD::FP_TO_SINT, NewOutTy)) {
+ default: break;
+ case TargetLowering::Legal:
+ if (!TLI.isTypeLegal(NewOutTy))
+ break; // Can't use this datatype.
+ // FALL THROUGH.
+ case TargetLowering::Custom:
+ OpToUse = ISD::FP_TO_SINT;
+ break;
+ }
+ if (OpToUse) break;
+
+ // If the target supports FP_TO_UINT of this type, use it.
+ switch (TLI.getOperationAction(ISD::FP_TO_UINT, NewOutTy)) {
+ default: break;
+ case TargetLowering::Legal:
+ if (!TLI.isTypeLegal(NewOutTy))
+ break; // Can't use this datatype.
+ // FALL THROUGH.
+ case TargetLowering::Custom:
+ OpToUse = ISD::FP_TO_UINT;
+ break;
+ }
+ if (OpToUse) break;
+
+ // Otherwise, try a larger type.
+ }
+
+ // Okay, we found the operation and type to use. Truncate the result of the
+ // extended FP_TO_*INT operation to the desired size.
+ return DAG.getNode(ISD::TRUNCATE, DestVT,
+ DAG.getNode(OpToUse, NewOutTy, LegalOp));
+}
+
+/// ExpandBSWAP - Open code the operations for BSWAP of the specified operation.
+///
+SDOperand SelectionDAGLegalize::ExpandBSWAP(SDOperand Op) {
+ MVT::ValueType VT = Op.getValueType();
+ MVT::ValueType SHVT = TLI.getShiftAmountTy();
+ SDOperand Tmp1, Tmp2, Tmp3, Tmp4, Tmp5, Tmp6, Tmp7, Tmp8;
+ switch (VT) {
+ default: assert(0 && "Unhandled Expand type in BSWAP!"); abort();
+ case MVT::i16:
+ Tmp2 = DAG.getNode(ISD::SHL, VT, Op, DAG.getConstant(8, SHVT));
+ Tmp1 = DAG.getNode(ISD::SRL, VT, Op, DAG.getConstant(8, SHVT));
+ return DAG.getNode(ISD::OR, VT, Tmp1, Tmp2);
+ case MVT::i32:
+ Tmp4 = DAG.getNode(ISD::SHL, VT, Op, DAG.getConstant(24, SHVT));
+ Tmp3 = DAG.getNode(ISD::SHL, VT, Op, DAG.getConstant(8, SHVT));
+ Tmp2 = DAG.getNode(ISD::SRL, VT, Op, DAG.getConstant(8, SHVT));
+ Tmp1 = DAG.getNode(ISD::SRL, VT, Op, DAG.getConstant(24, SHVT));
+ Tmp3 = DAG.getNode(ISD::AND, VT, Tmp3, DAG.getConstant(0xFF0000, VT));
+ Tmp2 = DAG.getNode(ISD::AND, VT, Tmp2, DAG.getConstant(0xFF00, VT));
+ Tmp4 = DAG.getNode(ISD::OR, VT, Tmp4, Tmp3);
+ Tmp2 = DAG.getNode(ISD::OR, VT, Tmp2, Tmp1);
+ return DAG.getNode(ISD::OR, VT, Tmp4, Tmp2);
+ case MVT::i64:
+ Tmp8 = DAG.getNode(ISD::SHL, VT, Op, DAG.getConstant(56, SHVT));
+ Tmp7 = DAG.getNode(ISD::SHL, VT, Op, DAG.getConstant(40, SHVT));
+ Tmp6 = DAG.getNode(ISD::SHL, VT, Op, DAG.getConstant(24, SHVT));
+ Tmp5 = DAG.getNode(ISD::SHL, VT, Op, DAG.getConstant(8, SHVT));
+ Tmp4 = DAG.getNode(ISD::SRL, VT, Op, DAG.getConstant(8, SHVT));
+ Tmp3 = DAG.getNode(ISD::SRL, VT, Op, DAG.getConstant(24, SHVT));
+ Tmp2 = DAG.getNode(ISD::SRL, VT, Op, DAG.getConstant(40, SHVT));
+ Tmp1 = DAG.getNode(ISD::SRL, VT, Op, DAG.getConstant(56, SHVT));
+ Tmp7 = DAG.getNode(ISD::AND, VT, Tmp7, DAG.getConstant(255ULL<<48, VT));
+ Tmp6 = DAG.getNode(ISD::AND, VT, Tmp6, DAG.getConstant(255ULL<<40, VT));
+ Tmp5 = DAG.getNode(ISD::AND, VT, Tmp5, DAG.getConstant(255ULL<<32, VT));
+ Tmp4 = DAG.getNode(ISD::AND, VT, Tmp4, DAG.getConstant(255ULL<<24, VT));
+ Tmp3 = DAG.getNode(ISD::AND, VT, Tmp3, DAG.getConstant(255ULL<<16, VT));
+ Tmp2 = DAG.getNode(ISD::AND, VT, Tmp2, DAG.getConstant(255ULL<<8 , VT));
+ Tmp8 = DAG.getNode(ISD::OR, VT, Tmp8, Tmp7);
+ Tmp6 = DAG.getNode(ISD::OR, VT, Tmp6, Tmp5);
+ Tmp4 = DAG.getNode(ISD::OR, VT, Tmp4, Tmp3);
+ Tmp2 = DAG.getNode(ISD::OR, VT, Tmp2, Tmp1);
+ Tmp8 = DAG.getNode(ISD::OR, VT, Tmp8, Tmp6);
+ Tmp4 = DAG.getNode(ISD::OR, VT, Tmp4, Tmp2);
+ return DAG.getNode(ISD::OR, VT, Tmp8, Tmp4);
+ }
+}
+
+/// ExpandBitCount - Expand the specified bitcount instruction into operations.
+///
+SDOperand SelectionDAGLegalize::ExpandBitCount(unsigned Opc, SDOperand Op) {
+ switch (Opc) {
+ default: assert(0 && "Cannot expand this yet!");
+ case ISD::CTPOP: {
+ static const uint64_t mask[6] = {
+ 0x5555555555555555ULL, 0x3333333333333333ULL,
+ 0x0F0F0F0F0F0F0F0FULL, 0x00FF00FF00FF00FFULL,
+ 0x0000FFFF0000FFFFULL, 0x00000000FFFFFFFFULL
+ };
+ MVT::ValueType VT = Op.getValueType();
+ MVT::ValueType ShVT = TLI.getShiftAmountTy();
+ unsigned len = MVT::getSizeInBits(VT);
+ for (unsigned i = 0; (1U << i) <= (len / 2); ++i) {
+ //x = (x & mask[i][len/8]) + (x >> (1 << i) & mask[i][len/8])
+ SDOperand Tmp2 = DAG.getConstant(mask[i], VT);
+ SDOperand Tmp3 = DAG.getConstant(1ULL << i, ShVT);
+ Op = DAG.getNode(ISD::ADD, VT, DAG.getNode(ISD::AND, VT, Op, Tmp2),
+ DAG.getNode(ISD::AND, VT,
+ DAG.getNode(ISD::SRL, VT, Op, Tmp3),Tmp2));
+ }
+ return Op;
+ }
+ case ISD::CTLZ: {
+ // for now, we do this:
+ // x = x | (x >> 1);
+ // x = x | (x >> 2);
+ // ...
+ // x = x | (x >>16);
+ // x = x | (x >>32); // for 64-bit input
+ // return popcount(~x);
+ //
+ // but see also: http://www.hackersdelight.org/HDcode/nlz.cc
+ MVT::ValueType VT = Op.getValueType();
+ MVT::ValueType ShVT = TLI.getShiftAmountTy();
+ unsigned len = MVT::getSizeInBits(VT);
+ for (unsigned i = 0; (1U << i) <= (len / 2); ++i) {
+ SDOperand Tmp3 = DAG.getConstant(1ULL << i, ShVT);
+ Op = DAG.getNode(ISD::OR, VT, Op, DAG.getNode(ISD::SRL, VT, Op, Tmp3));
+ }
+ Op = DAG.getNode(ISD::XOR, VT, Op, DAG.getConstant(~0ULL, VT));
+ return DAG.getNode(ISD::CTPOP, VT, Op);
+ }
+ case ISD::CTTZ: {
+ // for now, we use: { return popcount(~x & (x - 1)); }
+ // unless the target has ctlz but not ctpop, in which case we use:
+ // { return 32 - nlz(~x & (x-1)); }
+ // see also http://www.hackersdelight.org/HDcode/ntz.cc
+ MVT::ValueType VT = Op.getValueType();
+ SDOperand Tmp2 = DAG.getConstant(~0ULL, VT);
+ SDOperand Tmp3 = DAG.getNode(ISD::AND, VT,
+ DAG.getNode(ISD::XOR, VT, Op, Tmp2),
+ DAG.getNode(ISD::SUB, VT, Op, DAG.getConstant(1, VT)));
+ // If ISD::CTLZ is legal and CTPOP isn't, then do that instead.
+ if (!TLI.isOperationLegal(ISD::CTPOP, VT) &&
+ TLI.isOperationLegal(ISD::CTLZ, VT))
+ return DAG.getNode(ISD::SUB, VT,
+ DAG.getConstant(MVT::getSizeInBits(VT), VT),
+ DAG.getNode(ISD::CTLZ, VT, Tmp3));
+ return DAG.getNode(ISD::CTPOP, VT, Tmp3);
+ }
+ }
+}
+
+/// ExpandOp - Expand the specified SDOperand into its two component pieces
+/// Lo&Hi. Note that the Op MUST be an expanded type. As a result of this, the
+/// LegalizeNodes map is filled in for any results that are not expanded, the
+/// ExpandedNodes map is filled in for any results that are expanded, and the
+/// Lo/Hi values are returned.
+void SelectionDAGLegalize::ExpandOp(SDOperand Op, SDOperand &Lo, SDOperand &Hi){
+ MVT::ValueType VT = Op.getValueType();
+ MVT::ValueType NVT = TLI.getTypeToTransformTo(VT);
+ SDNode *Node = Op.Val;
+ assert(getTypeAction(VT) == Expand && "Not an expanded type!");
+ assert(((MVT::isInteger(NVT) && NVT < VT) || MVT::isFloatingPoint(VT) ||
+ MVT::isVector(VT)) &&
+ "Cannot expand to FP value or to larger int value!");
+
+ // See if we already expanded it.
+ DenseMap<SDOperand, std::pair<SDOperand, SDOperand> >::iterator I
+ = ExpandedNodes.find(Op);
+ if (I != ExpandedNodes.end()) {
+ Lo = I->second.first;
+ Hi = I->second.second;
+ return;
+ }
+
+ switch (Node->getOpcode()) {
+ case ISD::CopyFromReg:
+ assert(0 && "CopyFromReg must be legal!");
+ default:
+#ifndef NDEBUG
+ cerr << "NODE: "; Node->dump(&DAG); cerr << "\n";
+#endif
+ assert(0 && "Do not know how to expand this operator!");
+ abort();
+ case ISD::UNDEF:
+ NVT = TLI.getTypeToExpandTo(VT);
+ Lo = DAG.getNode(ISD::UNDEF, NVT);
+ Hi = DAG.getNode(ISD::UNDEF, NVT);
+ break;
+ case ISD::Constant: {
+ uint64_t Cst = cast<ConstantSDNode>(Node)->getValue();
+ Lo = DAG.getConstant(Cst, NVT);
+ Hi = DAG.getConstant(Cst >> MVT::getSizeInBits(NVT), NVT);
+ break;
+ }
+ case ISD::ConstantFP: {
+ ConstantFPSDNode *CFP = cast<ConstantFPSDNode>(Node);
+ Lo = ExpandConstantFP(CFP, false, DAG, TLI);
+ if (getTypeAction(Lo.getValueType()) == Expand)
+ ExpandOp(Lo, Lo, Hi);
+ break;
+ }
+ case ISD::BUILD_PAIR:
+ // Return the operands.
+ Lo = Node->getOperand(0);
+ Hi = Node->getOperand(1);
+ break;
+
+ case ISD::SIGN_EXTEND_INREG:
+ ExpandOp(Node->getOperand(0), Lo, Hi);
+ // sext_inreg the low part if needed.
+ Lo = DAG.getNode(ISD::SIGN_EXTEND_INREG, NVT, Lo, Node->getOperand(1));
+
+ // The high part gets the sign extension from the lo-part. This handles
+ // things like sextinreg V:i64 from i8.
+ Hi = DAG.getNode(ISD::SRA, NVT, Lo,
+ DAG.getConstant(MVT::getSizeInBits(NVT)-1,
+ TLI.getShiftAmountTy()));
+ break;
+
+ case ISD::BSWAP: {
+ ExpandOp(Node->getOperand(0), Lo, Hi);
+ SDOperand TempLo = DAG.getNode(ISD::BSWAP, NVT, Hi);
+ Hi = DAG.getNode(ISD::BSWAP, NVT, Lo);
+ Lo = TempLo;
+ break;
+ }
+
+ case ISD::CTPOP:
+ ExpandOp(Node->getOperand(0), Lo, Hi);
+ Lo = DAG.getNode(ISD::ADD, NVT, // ctpop(HL) -> ctpop(H)+ctpop(L)
+ DAG.getNode(ISD::CTPOP, NVT, Lo),
+ DAG.getNode(ISD::CTPOP, NVT, Hi));
+ Hi = DAG.getConstant(0, NVT);
+ break;
+
+ case ISD::CTLZ: {
+ // ctlz (HL) -> ctlz(H) != 32 ? ctlz(H) : (ctlz(L)+32)
+ ExpandOp(Node->getOperand(0), Lo, Hi);
+ SDOperand BitsC = DAG.getConstant(MVT::getSizeInBits(NVT), NVT);
+ SDOperand HLZ = DAG.getNode(ISD::CTLZ, NVT, Hi);
+ SDOperand TopNotZero = DAG.getSetCC(TLI.getSetCCResultTy(), HLZ, BitsC,
+ ISD::SETNE);
+ SDOperand LowPart = DAG.getNode(ISD::CTLZ, NVT, Lo);
+ LowPart = DAG.getNode(ISD::ADD, NVT, LowPart, BitsC);
+
+ Lo = DAG.getNode(ISD::SELECT, NVT, TopNotZero, HLZ, LowPart);
+ Hi = DAG.getConstant(0, NVT);
+ break;
+ }
+
+ case ISD::CTTZ: {
+ // cttz (HL) -> cttz(L) != 32 ? cttz(L) : (cttz(H)+32)
+ ExpandOp(Node->getOperand(0), Lo, Hi);
+ SDOperand BitsC = DAG.getConstant(MVT::getSizeInBits(NVT), NVT);
+ SDOperand LTZ = DAG.getNode(ISD::CTTZ, NVT, Lo);
+ SDOperand BotNotZero = DAG.getSetCC(TLI.getSetCCResultTy(), LTZ, BitsC,
+ ISD::SETNE);
+ SDOperand HiPart = DAG.getNode(ISD::CTTZ, NVT, Hi);
+ HiPart = DAG.getNode(ISD::ADD, NVT, HiPart, BitsC);
+
+ Lo = DAG.getNode(ISD::SELECT, NVT, BotNotZero, LTZ, HiPart);
+ Hi = DAG.getConstant(0, NVT);
+ break;
+ }
+
+ case ISD::VAARG: {
+ SDOperand Ch = Node->getOperand(0); // Legalize the chain.
+ SDOperand Ptr = Node->getOperand(1); // Legalize the pointer.
+ Lo = DAG.getVAArg(NVT, Ch, Ptr, Node->getOperand(2));
+ Hi = DAG.getVAArg(NVT, Lo.getValue(1), Ptr, Node->getOperand(2));
+
+ // Remember that we legalized the chain.
+ Hi = LegalizeOp(Hi);
+ AddLegalizedOperand(Op.getValue(1), Hi.getValue(1));
+ if (!TLI.isLittleEndian())
+ std::swap(Lo, Hi);
+ break;
+ }
+
+ case ISD::LOAD: {
+ LoadSDNode *LD = cast<LoadSDNode>(Node);
+ SDOperand Ch = LD->getChain(); // Legalize the chain.
+ SDOperand Ptr = LD->getBasePtr(); // Legalize the pointer.
+ ISD::LoadExtType ExtType = LD->getExtensionType();
+ int SVOffset = LD->getSrcValueOffset();
+ unsigned Alignment = LD->getAlignment();
+ bool isVolatile = LD->isVolatile();
+
+ if (ExtType == ISD::NON_EXTLOAD) {
+ Lo = DAG.getLoad(NVT, Ch, Ptr, LD->getSrcValue(), SVOffset,
+ isVolatile, Alignment);
+ if (VT == MVT::f32 || VT == MVT::f64) {
+ // f32->i32 or f64->i64 one to one expansion.
+ // Remember that we legalized the chain.
+ AddLegalizedOperand(SDOperand(Node, 1), LegalizeOp(Lo.getValue(1)));
+ // Recursively expand the new load.
+ if (getTypeAction(NVT) == Expand)
+ ExpandOp(Lo, Lo, Hi);
+ break;
+ }
+
+ // Increment the pointer to the other half.
+ unsigned IncrementSize = MVT::getSizeInBits(Lo.getValueType())/8;
+ Ptr = DAG.getNode(ISD::ADD, Ptr.getValueType(), Ptr,
+ getIntPtrConstant(IncrementSize));
+ SVOffset += IncrementSize;
+ if (Alignment > IncrementSize)
+ Alignment = IncrementSize;
+ Hi = DAG.getLoad(NVT, Ch, Ptr, LD->getSrcValue(), SVOffset,
+ isVolatile, Alignment);
+
+ // Build a factor node to remember that this load is independent of the
+ // other one.
+ SDOperand TF = DAG.getNode(ISD::TokenFactor, MVT::Other, Lo.getValue(1),
+ Hi.getValue(1));
+
+ // Remember that we legalized the chain.
+ AddLegalizedOperand(Op.getValue(1), LegalizeOp(TF));
+ if (!TLI.isLittleEndian())
+ std::swap(Lo, Hi);
+ } else {
+ MVT::ValueType EVT = LD->getLoadedVT();
+
+ if (VT == MVT::f64 && EVT == MVT::f32) {
+ // f64 = EXTLOAD f32 should expand to LOAD, FP_EXTEND
+ SDOperand Load = DAG.getLoad(EVT, Ch, Ptr, LD->getSrcValue(),
+ SVOffset, isVolatile, Alignment);
+ // Remember that we legalized the chain.
+ AddLegalizedOperand(SDOperand(Node, 1), LegalizeOp(Load.getValue(1)));
+ ExpandOp(DAG.getNode(ISD::FP_EXTEND, VT, Load), Lo, Hi);
+ break;
+ }
+
+ if (EVT == NVT)
+ Lo = DAG.getLoad(NVT, Ch, Ptr, LD->getSrcValue(),
+ SVOffset, isVolatile, Alignment);
+ else
+ Lo = DAG.getExtLoad(ExtType, NVT, Ch, Ptr, LD->getSrcValue(),
+ SVOffset, EVT, isVolatile,
+ Alignment);
+
+ // Remember that we legalized the chain.
+ AddLegalizedOperand(SDOperand(Node, 1), LegalizeOp(Lo.getValue(1)));
+
+ if (ExtType == ISD::SEXTLOAD) {
+ // The high part is obtained by SRA'ing all but one of the bits of the
+ // lo part.
+ unsigned LoSize = MVT::getSizeInBits(Lo.getValueType());
+ Hi = DAG.getNode(ISD::SRA, NVT, Lo,
+ DAG.getConstant(LoSize-1, TLI.getShiftAmountTy()));
+ } else if (ExtType == ISD::ZEXTLOAD) {
+ // The high part is just a zero.
+ Hi = DAG.getConstant(0, NVT);
+ } else /* if (ExtType == ISD::EXTLOAD) */ {
+ // The high part is undefined.
+ Hi = DAG.getNode(ISD::UNDEF, NVT);
+ }
+ }
+ break;
+ }
+ case ISD::AND:
+ case ISD::OR:
+ case ISD::XOR: { // Simple logical operators -> two trivial pieces.
+ SDOperand LL, LH, RL, RH;
+ ExpandOp(Node->getOperand(0), LL, LH);
+ ExpandOp(Node->getOperand(1), RL, RH);
+ Lo = DAG.getNode(Node->getOpcode(), NVT, LL, RL);
+ Hi = DAG.getNode(Node->getOpcode(), NVT, LH, RH);
+ break;
+ }
+ case ISD::SELECT: {
+ SDOperand LL, LH, RL, RH;
+ ExpandOp(Node->getOperand(1), LL, LH);
+ ExpandOp(Node->getOperand(2), RL, RH);
+ if (getTypeAction(NVT) == Expand)
+ NVT = TLI.getTypeToExpandTo(NVT);
+ Lo = DAG.getNode(ISD::SELECT, NVT, Node->getOperand(0), LL, RL);
+ if (VT != MVT::f32)
+ Hi = DAG.getNode(ISD::SELECT, NVT, Node->getOperand(0), LH, RH);
+ break;
+ }
+ case ISD::SELECT_CC: {
+ SDOperand TL, TH, FL, FH;
+ ExpandOp(Node->getOperand(2), TL, TH);
+ ExpandOp(Node->getOperand(3), FL, FH);
+ if (getTypeAction(NVT) == Expand)
+ NVT = TLI.getTypeToExpandTo(NVT);
+ Lo = DAG.getNode(ISD::SELECT_CC, NVT, Node->getOperand(0),
+ Node->getOperand(1), TL, FL, Node->getOperand(4));
+ if (VT != MVT::f32)
+ Hi = DAG.getNode(ISD::SELECT_CC, NVT, Node->getOperand(0),
+ Node->getOperand(1), TH, FH, Node->getOperand(4));
+ break;
+ }
+ case ISD::ANY_EXTEND:
+ // The low part is any extension of the input (which degenerates to a copy).
+ Lo = DAG.getNode(ISD::ANY_EXTEND, NVT, Node->getOperand(0));
+ // The high part is undefined.
+ Hi = DAG.getNode(ISD::UNDEF, NVT);
+ break;
+ case ISD::SIGN_EXTEND: {
+ // The low part is just a sign extension of the input (which degenerates to
+ // a copy).
+ Lo = DAG.getNode(ISD::SIGN_EXTEND, NVT, Node->getOperand(0));
+
+ // The high part is obtained by SRA'ing all but one of the bits of the lo
+ // part.
+ unsigned LoSize = MVT::getSizeInBits(Lo.getValueType());
+ Hi = DAG.getNode(ISD::SRA, NVT, Lo,
+ DAG.getConstant(LoSize-1, TLI.getShiftAmountTy()));
+ break;
+ }
+ case ISD::ZERO_EXTEND:
+ // The low part is just a zero extension of the input (which degenerates to
+ // a copy).
+ Lo = DAG.getNode(ISD::ZERO_EXTEND, NVT, Node->getOperand(0));
+
+ // The high part is just a zero.
+ Hi = DAG.getConstant(0, NVT);
+ break;
+
+ case ISD::TRUNCATE: {
+ // The input value must be larger than this value. Expand *it*.
+ SDOperand NewLo;
+ ExpandOp(Node->getOperand(0), NewLo, Hi);
+
+ // The low part is now either the right size, or it is closer. If not the
+ // right size, make an illegal truncate so we recursively expand it.
+ if (NewLo.getValueType() != Node->getValueType(0))
+ NewLo = DAG.getNode(ISD::TRUNCATE, Node->getValueType(0), NewLo);
+ ExpandOp(NewLo, Lo, Hi);
+ break;
+ }
+
+ case ISD::BIT_CONVERT: {
+ SDOperand Tmp;
+ if (TLI.getOperationAction(ISD::BIT_CONVERT, VT) == TargetLowering::Custom){
+ // If the target wants to, allow it to lower this itself.
+ switch (getTypeAction(Node->getOperand(0).getValueType())) {
+ case Expand: assert(0 && "cannot expand FP!");
+ case Legal: Tmp = LegalizeOp(Node->getOperand(0)); break;
+ case Promote: Tmp = PromoteOp (Node->getOperand(0)); break;
+ }
+ Tmp = TLI.LowerOperation(DAG.getNode(ISD::BIT_CONVERT, VT, Tmp), DAG);
+ }
+
+ // f32 / f64 must be expanded to i32 / i64.
+ if (VT == MVT::f32 || VT == MVT::f64) {
+ Lo = DAG.getNode(ISD::BIT_CONVERT, NVT, Node->getOperand(0));
+ if (getTypeAction(NVT) == Expand)
+ ExpandOp(Lo, Lo, Hi);
+ break;
+ }
+
+ // If source operand will be expanded to the same type as VT, i.e.
+ // i64 <- f64, i32 <- f32, expand the source operand instead.
+ MVT::ValueType VT0 = Node->getOperand(0).getValueType();
+ if (getTypeAction(VT0) == Expand && TLI.getTypeToTransformTo(VT0) == VT) {
+ ExpandOp(Node->getOperand(0), Lo, Hi);
+ break;
+ }
+
+ // Turn this into a load/store pair by default.
+ if (Tmp.Val == 0)
+ Tmp = ExpandBIT_CONVERT(VT, Node->getOperand(0));
+
+ ExpandOp(Tmp, Lo, Hi);
+ break;
+ }
+
+ case ISD::READCYCLECOUNTER:
+ assert(TLI.getOperationAction(ISD::READCYCLECOUNTER, VT) ==
+ TargetLowering::Custom &&
+ "Must custom expand ReadCycleCounter");
+ Lo = TLI.LowerOperation(Op, DAG);
+ assert(Lo.Val && "Node must be custom expanded!");
+ Hi = Lo.getValue(1);
+ AddLegalizedOperand(SDOperand(Node, 1), // Remember we legalized the chain.
+ LegalizeOp(Lo.getValue(2)));
+ break;
+
+ // These operators cannot be expanded directly, emit them as calls to
+ // library functions.
+ case ISD::FP_TO_SINT: {
+ if (TLI.getOperationAction(ISD::FP_TO_SINT, VT) == TargetLowering::Custom) {
+ SDOperand Op;
+ switch (getTypeAction(Node->getOperand(0).getValueType())) {
+ case Expand: assert(0 && "cannot expand FP!");
+ case Legal: Op = LegalizeOp(Node->getOperand(0)); break;
+ case Promote: Op = PromoteOp (Node->getOperand(0)); break;
+ }
+
+ Op = TLI.LowerOperation(DAG.getNode(ISD::FP_TO_SINT, VT, Op), DAG);
+
+ // Now that the custom expander is done, expand the result, which is still
+ // VT.
+ if (Op.Val) {
+ ExpandOp(Op, Lo, Hi);
+ break;
+ }
+ }
+
+ RTLIB::Libcall LC;
+ if (Node->getOperand(0).getValueType() == MVT::f32)
+ LC = RTLIB::FPTOSINT_F32_I64;
+ else
+ LC = RTLIB::FPTOSINT_F64_I64;
+ Lo = ExpandLibCall(TLI.getLibcallName(LC), Node,
+ false/*sign irrelevant*/, Hi);
+ break;
+ }
+
+ case ISD::FP_TO_UINT: {
+ if (TLI.getOperationAction(ISD::FP_TO_UINT, VT) == TargetLowering::Custom) {
+ SDOperand Op;
+ switch (getTypeAction(Node->getOperand(0).getValueType())) {
+ case Expand: assert(0 && "cannot expand FP!");
+ case Legal: Op = LegalizeOp(Node->getOperand(0)); break;
+ case Promote: Op = PromoteOp (Node->getOperand(0)); break;
+ }
+
+ Op = TLI.LowerOperation(DAG.getNode(ISD::FP_TO_UINT, VT, Op), DAG);
+
+ // Now that the custom expander is done, expand the result.
+ if (Op.Val) {
+ ExpandOp(Op, Lo, Hi);
+ break;
+ }
+ }
+
+ RTLIB::Libcall LC;
+ if (Node->getOperand(0).getValueType() == MVT::f32)
+ LC = RTLIB::FPTOUINT_F32_I64;
+ else
+ LC = RTLIB::FPTOUINT_F64_I64;
+ Lo = ExpandLibCall(TLI.getLibcallName(LC), Node,
+ false/*sign irrelevant*/, Hi);
+ break;
+ }
+
+ case ISD::SHL: {
+ // If the target wants custom lowering, do so.
+ SDOperand ShiftAmt = LegalizeOp(Node->getOperand(1));
+ if (TLI.getOperationAction(ISD::SHL, VT) == TargetLowering::Custom) {
+ SDOperand Op = DAG.getNode(ISD::SHL, VT, Node->getOperand(0), ShiftAmt);
+ Op = TLI.LowerOperation(Op, DAG);
+ if (Op.Val) {
+ // Now that the custom expander is done, expand the result, which is
+ // still VT.
+ ExpandOp(Op, Lo, Hi);
+ break;
+ }
+ }
+
+ // If ADDC/ADDE are supported and if the shift amount is a constant 1, emit
+ // this X << 1 as X+X.
+ if (ConstantSDNode *ShAmt = dyn_cast<ConstantSDNode>(ShiftAmt)) {
+ if (ShAmt->getValue() == 1 && TLI.isOperationLegal(ISD::ADDC, NVT) &&
+ TLI.isOperationLegal(ISD::ADDE, NVT)) {
+ SDOperand LoOps[2], HiOps[3];
+ ExpandOp(Node->getOperand(0), LoOps[0], HiOps[0]);
+ SDVTList VTList = DAG.getVTList(LoOps[0].getValueType(), MVT::Flag);
+ LoOps[1] = LoOps[0];
+ Lo = DAG.getNode(ISD::ADDC, VTList, LoOps, 2);
+
+ HiOps[1] = HiOps[0];
+ HiOps[2] = Lo.getValue(1);
+ Hi = DAG.getNode(ISD::ADDE, VTList, HiOps, 3);
+ break;
+ }
+ }
+
+ // If we can emit an efficient shift operation, do so now.
+ if (ExpandShift(ISD::SHL, Node->getOperand(0), ShiftAmt, Lo, Hi))
+ break;
+
+ // If this target supports SHL_PARTS, use it.
+ TargetLowering::LegalizeAction Action =
+ TLI.getOperationAction(ISD::SHL_PARTS, NVT);
+ if ((Action == TargetLowering::Legal && TLI.isTypeLegal(NVT)) ||
+ Action == TargetLowering::Custom) {
+ ExpandShiftParts(ISD::SHL_PARTS, Node->getOperand(0), ShiftAmt, Lo, Hi);
+ break;
+ }
+
+ // Otherwise, emit a libcall.
+ Lo = ExpandLibCall(TLI.getLibcallName(RTLIB::SHL_I64), Node,
+ false/*left shift=unsigned*/, Hi);
+ break;
+ }
+
+ case ISD::SRA: {
+ // If the target wants custom lowering, do so.
+ SDOperand ShiftAmt = LegalizeOp(Node->getOperand(1));
+ if (TLI.getOperationAction(ISD::SRA, VT) == TargetLowering::Custom) {
+ SDOperand Op = DAG.getNode(ISD::SRA, VT, Node->getOperand(0), ShiftAmt);
+ Op = TLI.LowerOperation(Op, DAG);
+ if (Op.Val) {
+ // Now that the custom expander is done, expand the result, which is
+ // still VT.
+ ExpandOp(Op, Lo, Hi);
+ break;
+ }
+ }
+
+ // If we can emit an efficient shift operation, do so now.
+ if (ExpandShift(ISD::SRA, Node->getOperand(0), ShiftAmt, Lo, Hi))
+ break;
+
+ // If this target supports SRA_PARTS, use it.
+ TargetLowering::LegalizeAction Action =
+ TLI.getOperationAction(ISD::SRA_PARTS, NVT);
+ if ((Action == TargetLowering::Legal && TLI.isTypeLegal(NVT)) ||
+ Action == TargetLowering::Custom) {
+ ExpandShiftParts(ISD::SRA_PARTS, Node->getOperand(0), ShiftAmt, Lo, Hi);
+ break;
+ }
+
+ // Otherwise, emit a libcall.
+ Lo = ExpandLibCall(TLI.getLibcallName(RTLIB::SRA_I64), Node,
+ true/*ashr is signed*/, Hi);
+ break;
+ }
+
+ case ISD::SRL: {
+ // If the target wants custom lowering, do so.
+ SDOperand ShiftAmt = LegalizeOp(Node->getOperand(1));
+ if (TLI.getOperationAction(ISD::SRL, VT) == TargetLowering::Custom) {
+ SDOperand Op = DAG.getNode(ISD::SRL, VT, Node->getOperand(0), ShiftAmt);
+ Op = TLI.LowerOperation(Op, DAG);
+ if (Op.Val) {
+ // Now that the custom expander is done, expand the result, which is
+ // still VT.
+ ExpandOp(Op, Lo, Hi);
+ break;
+ }
+ }
+
+ // If we can emit an efficient shift operation, do so now.
+ if (ExpandShift(ISD::SRL, Node->getOperand(0), ShiftAmt, Lo, Hi))
+ break;
+
+ // If this target supports SRL_PARTS, use it.
+ TargetLowering::LegalizeAction Action =
+ TLI.getOperationAction(ISD::SRL_PARTS, NVT);
+ if ((Action == TargetLowering::Legal && TLI.isTypeLegal(NVT)) ||
+ Action == TargetLowering::Custom) {
+ ExpandShiftParts(ISD::SRL_PARTS, Node->getOperand(0), ShiftAmt, Lo, Hi);
+ break;
+ }
+
+ // Otherwise, emit a libcall.
+ Lo = ExpandLibCall(TLI.getLibcallName(RTLIB::SRL_I64), Node,
+ false/*lshr is unsigned*/, Hi);
+ break;
+ }
+
+ case ISD::ADD:
+ case ISD::SUB: {
+ // If the target wants to custom expand this, let them.
+ if (TLI.getOperationAction(Node->getOpcode(), VT) ==
+ TargetLowering::Custom) {
+ Op = TLI.LowerOperation(Op, DAG);
+ if (Op.Val) {
+ ExpandOp(Op, Lo, Hi);
+ break;
+ }
+ }
+
+ // Expand the subcomponents.
+ SDOperand LHSL, LHSH, RHSL, RHSH;
+ ExpandOp(Node->getOperand(0), LHSL, LHSH);
+ ExpandOp(Node->getOperand(1), RHSL, RHSH);
+ SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Flag);
+ SDOperand LoOps[2], HiOps[3];
+ LoOps[0] = LHSL;
+ LoOps[1] = RHSL;
+ HiOps[0] = LHSH;
+ HiOps[1] = RHSH;
+ if (Node->getOpcode() == ISD::ADD) {
+ Lo = DAG.getNode(ISD::ADDC, VTList, LoOps, 2);
+ HiOps[2] = Lo.getValue(1);
+ Hi = DAG.getNode(ISD::ADDE, VTList, HiOps, 3);
+ } else {
+ Lo = DAG.getNode(ISD::SUBC, VTList, LoOps, 2);
+ HiOps[2] = Lo.getValue(1);
+ Hi = DAG.getNode(ISD::SUBE, VTList, HiOps, 3);
+ }
+ break;
+ }
+
+ case ISD::ADDC:
+ case ISD::SUBC: {
+ // Expand the subcomponents.
+ SDOperand LHSL, LHSH, RHSL, RHSH;
+ ExpandOp(Node->getOperand(0), LHSL, LHSH);
+ ExpandOp(Node->getOperand(1), RHSL, RHSH);
+ SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Flag);
+ SDOperand LoOps[2] = { LHSL, RHSL };
+ SDOperand HiOps[3] = { LHSH, RHSH };
+
+ if (Node->getOpcode() == ISD::ADDC) {
+ Lo = DAG.getNode(ISD::ADDC, VTList, LoOps, 2);
+ HiOps[2] = Lo.getValue(1);
+ Hi = DAG.getNode(ISD::ADDE, VTList, HiOps, 3);
+ } else {
+ Lo = DAG.getNode(ISD::SUBC, VTList, LoOps, 2);
+ HiOps[2] = Lo.getValue(1);
+ Hi = DAG.getNode(ISD::SUBE, VTList, HiOps, 3);
+ }
+ // Remember that we legalized the flag.
+ AddLegalizedOperand(Op.getValue(1), LegalizeOp(Hi.getValue(1)));
+ break;
+ }
+ case ISD::ADDE:
+ case ISD::SUBE: {
+ // Expand the subcomponents.
+ SDOperand LHSL, LHSH, RHSL, RHSH;
+ ExpandOp(Node->getOperand(0), LHSL, LHSH);
+ ExpandOp(Node->getOperand(1), RHSL, RHSH);
+ SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Flag);
+ SDOperand LoOps[3] = { LHSL, RHSL, Node->getOperand(2) };
+ SDOperand HiOps[3] = { LHSH, RHSH };
+
+ Lo = DAG.getNode(Node->getOpcode(), VTList, LoOps, 3);
+ HiOps[2] = Lo.getValue(1);
+ Hi = DAG.getNode(Node->getOpcode(), VTList, HiOps, 3);
+
+ // Remember that we legalized the flag.
+ AddLegalizedOperand(Op.getValue(1), LegalizeOp(Hi.getValue(1)));
+ break;
+ }
+ case ISD::MUL: {
+ // If the target wants to custom expand this, let them.
+ if (TLI.getOperationAction(ISD::MUL, VT) == TargetLowering::Custom) {
+ SDOperand New = TLI.LowerOperation(Op, DAG);
+ if (New.Val) {
+ ExpandOp(New, Lo, Hi);
+ break;
+ }
+ }
+
+ bool HasMULHS = TLI.isOperationLegal(ISD::MULHS, NVT);
+ bool HasMULHU = TLI.isOperationLegal(ISD::MULHU, NVT);
+ if (HasMULHS || HasMULHU) {
+ SDOperand LL, LH, RL, RH;
+ ExpandOp(Node->getOperand(0), LL, LH);
+ ExpandOp(Node->getOperand(1), RL, RH);
+ unsigned SH = MVT::getSizeInBits(RH.getValueType())-1;
+ // FIXME: Move this to the dag combiner.
+ // MULHS implicitly sign extends its inputs. Check to see if ExpandOp
+ // extended the sign bit of the low half through the upper half, and if so
+ // emit a MULHS instead of the alternate sequence that is valid for any
+ // i64 x i64 multiply.
+ if (HasMULHS &&
+ // is RH an extension of the sign bit of RL?
+ RH.getOpcode() == ISD::SRA && RH.getOperand(0) == RL &&
+ RH.getOperand(1).getOpcode() == ISD::Constant &&
+ cast<ConstantSDNode>(RH.getOperand(1))->getValue() == SH &&
+ // is LH an extension of the sign bit of LL?
+ LH.getOpcode() == ISD::SRA && LH.getOperand(0) == LL &&
+ LH.getOperand(1).getOpcode() == ISD::Constant &&
+ cast<ConstantSDNode>(LH.getOperand(1))->getValue() == SH) {
+ // Low part:
+ Lo = DAG.getNode(ISD::MUL, NVT, LL, RL);
+ // High part:
+ Hi = DAG.getNode(ISD::MULHS, NVT, LL, RL);
+ break;
+ } else if (HasMULHU) {
+ // Low part:
+ Lo = DAG.getNode(ISD::MUL, NVT, LL, RL);
+
+ // High part:
+ Hi = DAG.getNode(ISD::MULHU, NVT, LL, RL);
+ RH = DAG.getNode(ISD::MUL, NVT, LL, RH);
+ LH = DAG.getNode(ISD::MUL, NVT, LH, RL);
+ Hi = DAG.getNode(ISD::ADD, NVT, Hi, RH);
+ Hi = DAG.getNode(ISD::ADD, NVT, Hi, LH);
+ break;
+ }
+ }
+
+ Lo = ExpandLibCall(TLI.getLibcallName(RTLIB::MUL_I64), Node,
+ false/*sign irrelevant*/, Hi);
+ break;
+ }
+ case ISD::SDIV:
+ Lo = ExpandLibCall(TLI.getLibcallName(RTLIB::SDIV_I64), Node, true, Hi);
+ break;
+ case ISD::UDIV:
+ Lo = ExpandLibCall(TLI.getLibcallName(RTLIB::UDIV_I64), Node, true, Hi);
+ break;
+ case ISD::SREM:
+ Lo = ExpandLibCall(TLI.getLibcallName(RTLIB::SREM_I64), Node, true, Hi);
+ break;
+ case ISD::UREM:
+ Lo = ExpandLibCall(TLI.getLibcallName(RTLIB::UREM_I64), Node, true, Hi);
+ break;
+
+ case ISD::FADD:
+ Lo = ExpandLibCall(TLI.getLibcallName((VT == MVT::f32)
+ ? RTLIB::ADD_F32 : RTLIB::ADD_F64),
+ Node, false, Hi);
+ break;
+ case ISD::FSUB:
+ Lo = ExpandLibCall(TLI.getLibcallName((VT == MVT::f32)
+ ? RTLIB::SUB_F32 : RTLIB::SUB_F64),
+ Node, false, Hi);
+ break;
+ case ISD::FMUL:
+ Lo = ExpandLibCall(TLI.getLibcallName((VT == MVT::f32)
+ ? RTLIB::MUL_F32 : RTLIB::MUL_F64),
+ Node, false, Hi);
+ break;
+ case ISD::FDIV:
+ Lo = ExpandLibCall(TLI.getLibcallName((VT == MVT::f32)
+ ? RTLIB::DIV_F32 : RTLIB::DIV_F64),
+ Node, false, Hi);
+ break;
+ case ISD::FP_EXTEND:
+ Lo = ExpandLibCall(TLI.getLibcallName(RTLIB::FPEXT_F32_F64), Node, true,Hi);
+ break;
+ case ISD::FP_ROUND:
+ Lo = ExpandLibCall(TLI.getLibcallName(RTLIB::FPROUND_F64_F32),Node,true,Hi);
+ break;
+ case ISD::FSQRT:
+ case ISD::FSIN:
+ case ISD::FCOS: {
+ RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
+ switch(Node->getOpcode()) {
+ case ISD::FSQRT:
+ LC = (VT == MVT::f32) ? RTLIB::SQRT_F32 : RTLIB::SQRT_F64;
+ break;
+ case ISD::FSIN:
+ LC = (VT == MVT::f32) ? RTLIB::SIN_F32 : RTLIB::SIN_F64;
+ break;
+ case ISD::FCOS:
+ LC = (VT == MVT::f32) ? RTLIB::COS_F32 : RTLIB::COS_F64;
+ break;
+ default: assert(0 && "Unreachable!");
+ }
+ Lo = ExpandLibCall(TLI.getLibcallName(LC), Node, false, Hi);
+ break;
+ }
+ case ISD::FABS: {
+ SDOperand Mask = (VT == MVT::f64)
+ ? DAG.getConstantFP(BitsToDouble(~(1ULL << 63)), VT)
+ : DAG.getConstantFP(BitsToFloat(~(1U << 31)), VT);
+ Mask = DAG.getNode(ISD::BIT_CONVERT, NVT, Mask);
+ Lo = DAG.getNode(ISD::BIT_CONVERT, NVT, Node->getOperand(0));
+ Lo = DAG.getNode(ISD::AND, NVT, Lo, Mask);
+ if (getTypeAction(NVT) == Expand)
+ ExpandOp(Lo, Lo, Hi);
+ break;
+ }
+ case ISD::FNEG: {
+ SDOperand Mask = (VT == MVT::f64)
+ ? DAG.getConstantFP(BitsToDouble(1ULL << 63), VT)
+ : DAG.getConstantFP(BitsToFloat(1U << 31), VT);
+ Mask = DAG.getNode(ISD::BIT_CONVERT, NVT, Mask);
+ Lo = DAG.getNode(ISD::BIT_CONVERT, NVT, Node->getOperand(0));
+ Lo = DAG.getNode(ISD::XOR, NVT, Lo, Mask);
+ if (getTypeAction(NVT) == Expand)
+ ExpandOp(Lo, Lo, Hi);
+ break;
+ }
+ case ISD::FCOPYSIGN: {
+ Lo = ExpandFCOPYSIGNToBitwiseOps(Node, NVT, DAG, TLI);
+ if (getTypeAction(NVT) == Expand)
+ ExpandOp(Lo, Lo, Hi);
+ break;
+ }
+ case ISD::SINT_TO_FP:
+ case ISD::UINT_TO_FP: {
+ bool isSigned = Node->getOpcode() == ISD::SINT_TO_FP;
+ MVT::ValueType SrcVT = Node->getOperand(0).getValueType();
+ RTLIB::Libcall LC;
+ if (Node->getOperand(0).getValueType() == MVT::i64) {
+ if (VT == MVT::f32)
+ LC = isSigned ? RTLIB::SINTTOFP_I64_F32 : RTLIB::UINTTOFP_I64_F32;
+ else
+ LC = isSigned ? RTLIB::SINTTOFP_I64_F64 : RTLIB::UINTTOFP_I64_F64;
+ } else {
+ if (VT == MVT::f32)
+ LC = isSigned ? RTLIB::SINTTOFP_I32_F32 : RTLIB::UINTTOFP_I32_F32;
+ else
+ LC = isSigned ? RTLIB::SINTTOFP_I32_F64 : RTLIB::UINTTOFP_I32_F64;
+ }
+
+ // Promote the operand if needed.
+ if (getTypeAction(SrcVT) == Promote) {
+ SDOperand Tmp = PromoteOp(Node->getOperand(0));
+ Tmp = isSigned
+ ? DAG.getNode(ISD::SIGN_EXTEND_INREG, Tmp.getValueType(), Tmp,
+ DAG.getValueType(SrcVT))
+ : DAG.getZeroExtendInReg(Tmp, SrcVT);
+ Node = DAG.UpdateNodeOperands(Op, Tmp).Val;
+ }
+
+ const char *LibCall = TLI.getLibcallName(LC);
+ if (LibCall)
+ Lo = ExpandLibCall(TLI.getLibcallName(LC), Node, isSigned, Hi);
+ else {
+ Lo = ExpandIntToFP(Node->getOpcode() == ISD::SINT_TO_FP, VT,
+ Node->getOperand(0));
+ if (getTypeAction(Lo.getValueType()) == Expand)
+ ExpandOp(Lo, Lo, Hi);
+ }
+ break;
+ }
+ }
+
+ // Make sure the resultant values have been legalized themselves, unless this
+ // is a type that requires multi-step expansion.
+ if (getTypeAction(NVT) != Expand && NVT != MVT::isVoid) {
+ Lo = LegalizeOp(Lo);
+ if (Hi.Val)
+ // Don't legalize the high part if it is expanded to a single node.
+ Hi = LegalizeOp(Hi);
+ }
+
+ // Remember in a map if the values will be reused later.
+ bool isNew = ExpandedNodes.insert(std::make_pair(Op, std::make_pair(Lo, Hi)));
+ assert(isNew && "Value already expanded?!?");
+}
+
+/// SplitVectorOp - Given an operand of vector type, break it down into
+/// two smaller values, still of vector type.
+void SelectionDAGLegalize::SplitVectorOp(SDOperand Op, SDOperand &Lo,
+ SDOperand &Hi) {
+ assert(MVT::isVector(Op.getValueType()) && "Cannot split non-vector type!");
+ SDNode *Node = Op.Val;
+ unsigned NumElements = MVT::getVectorNumElements(Node->getValueType(0));
+ assert(NumElements > 1 && "Cannot split a single element vector!");
+ unsigned NewNumElts = NumElements/2;
+ MVT::ValueType NewEltVT = MVT::getVectorElementType(Node->getValueType(0));
+ MVT::ValueType NewVT = MVT::getVectorType(NewEltVT, NewNumElts);
+
+ // See if we already split it.
+ std::map<SDOperand, std::pair<SDOperand, SDOperand> >::iterator I
+ = SplitNodes.find(Op);
+ if (I != SplitNodes.end()) {
+ Lo = I->second.first;
+ Hi = I->second.second;
+ return;
+ }
+
+ switch (Node->getOpcode()) {
+ default:
+#ifndef NDEBUG
+ Node->dump(&DAG);
+#endif
+ assert(0 && "Unhandled operation in SplitVectorOp!");
+ case ISD::BUILD_PAIR:
+ Lo = Node->getOperand(0);
+ Hi = Node->getOperand(1);
+ break;
+ case ISD::BUILD_VECTOR: {
+ SmallVector<SDOperand, 8> LoOps(Node->op_begin(),
+ Node->op_begin()+NewNumElts);
+ Lo = DAG.getNode(ISD::BUILD_VECTOR, NewVT, &LoOps[0], LoOps.size());
+
+ SmallVector<SDOperand, 8> HiOps(Node->op_begin()+NewNumElts,
+ Node->op_end());
+ Hi = DAG.getNode(ISD::BUILD_VECTOR, NewVT, &HiOps[0], HiOps.size());
+ break;
+ }
+ case ISD::CONCAT_VECTORS: {
+ unsigned NewNumSubvectors = Node->getNumOperands() / 2;
+ if (NewNumSubvectors == 1) {
+ Lo = Node->getOperand(0);
+ Hi = Node->getOperand(1);
+ } else {
+ SmallVector<SDOperand, 8> LoOps(Node->op_begin(),
+ Node->op_begin()+NewNumSubvectors);
+ Lo = DAG.getNode(ISD::CONCAT_VECTORS, NewVT, &LoOps[0], LoOps.size());
+
+ SmallVector<SDOperand, 8> HiOps(Node->op_begin()+NewNumSubvectors,
+ Node->op_end());
+ Hi = DAG.getNode(ISD::CONCAT_VECTORS, NewVT, &HiOps[0], HiOps.size());
+ }
+ break;
+ }
+ case ISD::ADD:
+ case ISD::SUB:
+ case ISD::MUL:
+ case ISD::FADD:
+ case ISD::FSUB:
+ case ISD::FMUL:
+ case ISD::SDIV:
+ case ISD::UDIV:
+ case ISD::FDIV:
+ case ISD::AND:
+ case ISD::OR:
+ case ISD::XOR: {
+ SDOperand LL, LH, RL, RH;
+ SplitVectorOp(Node->getOperand(0), LL, LH);
+ SplitVectorOp(Node->getOperand(1), RL, RH);
+
+ Lo = DAG.getNode(Node->getOpcode(), NewVT, LL, RL);
+ Hi = DAG.getNode(Node->getOpcode(), NewVT, LH, RH);
+ break;
+ }
+ case ISD::LOAD: {
+ LoadSDNode *LD = cast<LoadSDNode>(Node);
+ SDOperand Ch = LD->getChain();
+ SDOperand Ptr = LD->getBasePtr();
+ const Value *SV = LD->getSrcValue();
+ int SVOffset = LD->getSrcValueOffset();
+ unsigned Alignment = LD->getAlignment();
+ bool isVolatile = LD->isVolatile();
+
+ Lo = DAG.getLoad(NewVT, Ch, Ptr, SV, SVOffset, isVolatile, Alignment);
+ unsigned IncrementSize = NewNumElts * MVT::getSizeInBits(NewEltVT)/8;
+ Ptr = DAG.getNode(ISD::ADD, Ptr.getValueType(), Ptr,
+ getIntPtrConstant(IncrementSize));
+ SVOffset += IncrementSize;
+ if (Alignment > IncrementSize)
+ Alignment = IncrementSize;
+ Hi = DAG.getLoad(NewVT, Ch, Ptr, SV, SVOffset, isVolatile, Alignment);
+
+ // Build a factor node to remember that this load is independent of the
+ // other one.
+ SDOperand TF = DAG.getNode(ISD::TokenFactor, MVT::Other, Lo.getValue(1),
+ Hi.getValue(1));
+
+ // Remember that we legalized the chain.
+ AddLegalizedOperand(Op.getValue(1), LegalizeOp(TF));
+ break;
+ }
+ case ISD::BIT_CONVERT: {
+ // We know the result is a vector. The input may be either a vector or a
+ // scalar value.
+ SDOperand InOp = Node->getOperand(0);
+ if (!MVT::isVector(InOp.getValueType()) ||
+ MVT::getVectorNumElements(InOp.getValueType()) == 1) {
+ // The input is a scalar or single-element vector.
+ // Lower to a store/load so that it can be split.
+ // FIXME: this could be improved probably.
+ SDOperand Ptr = CreateStackTemporary(InOp.getValueType());
+
+ SDOperand St = DAG.getStore(DAG.getEntryNode(),
+ InOp, Ptr, NULL, 0);
+ InOp = DAG.getLoad(Op.getValueType(), St, Ptr, NULL, 0);
+ }
+ // Split the vector and convert each of the pieces now.
+ SplitVectorOp(InOp, Lo, Hi);
+ Lo = DAG.getNode(ISD::BIT_CONVERT, NewVT, Lo);
+ Hi = DAG.getNode(ISD::BIT_CONVERT, NewVT, Hi);
+ break;
+ }
+ }
+
+ // Remember in a map if the values will be reused later.
+ bool isNew =
+ SplitNodes.insert(std::make_pair(Op, std::make_pair(Lo, Hi))).second;
+ assert(isNew && "Value already split?!?");
+}
+
+
+/// ScalarizeVectorOp - Given an operand of single-element vector type
+/// (e.g. v1f32), convert it into the equivalent operation that returns a
+/// scalar (e.g. f32) value.
+SDOperand SelectionDAGLegalize::ScalarizeVectorOp(SDOperand Op) {
+ assert(MVT::isVector(Op.getValueType()) &&
+ "Bad ScalarizeVectorOp invocation!");
+ SDNode *Node = Op.Val;
+ MVT::ValueType NewVT = MVT::getVectorElementType(Op.getValueType());
+ assert(MVT::getVectorNumElements(Op.getValueType()) == 1);
+
+ // See if we already scalarized it.
+ std::map<SDOperand, SDOperand>::iterator I = ScalarizedNodes.find(Op);
+ if (I != ScalarizedNodes.end()) return I->second;
+
+ SDOperand Result;
+ switch (Node->getOpcode()) {
+ default:
+#ifndef NDEBUG
+ Node->dump(&DAG); cerr << "\n";
+#endif
+ assert(0 && "Unknown vector operation in ScalarizeVectorOp!");
+ case ISD::ADD:
+ case ISD::FADD:
+ case ISD::SUB:
+ case ISD::FSUB:
+ case ISD::MUL:
+ case ISD::FMUL:
+ case ISD::SDIV:
+ case ISD::UDIV:
+ case ISD::FDIV:
+ case ISD::SREM:
+ case ISD::UREM:
+ case ISD::FREM:
+ case ISD::AND:
+ case ISD::OR:
+ case ISD::XOR:
+ Result = DAG.getNode(Node->getOpcode(),
+ NewVT,
+ ScalarizeVectorOp(Node->getOperand(0)),
+ ScalarizeVectorOp(Node->getOperand(1)));
+ break;
+ case ISD::FNEG:
+ case ISD::FABS:
+ case ISD::FSQRT:
+ case ISD::FSIN:
+ case ISD::FCOS:
+ Result = DAG.getNode(Node->getOpcode(),
+ NewVT,
+ ScalarizeVectorOp(Node->getOperand(0)));
+ break;
+ case ISD::LOAD: {
+ LoadSDNode *LD = cast<LoadSDNode>(Node);
+ SDOperand Ch = LegalizeOp(LD->getChain()); // Legalize the chain.
+ SDOperand Ptr = LegalizeOp(LD->getBasePtr()); // Legalize the pointer.
+
+ const Value *SV = LD->getSrcValue();
+ int SVOffset = LD->getSrcValueOffset();
+ Result = DAG.getLoad(NewVT, Ch, Ptr, SV, SVOffset,
+ LD->isVolatile(), LD->getAlignment());
+
+ // Remember that we legalized the chain.
+ AddLegalizedOperand(Op.getValue(1), LegalizeOp(Result.getValue(1)));
+ break;
+ }
+ case ISD::BUILD_VECTOR:
+ Result = Node->getOperand(0);
+ break;
+ case ISD::INSERT_VECTOR_ELT:
+ // Returning the inserted scalar element.
+ Result = Node->getOperand(1);
+ break;
+ case ISD::CONCAT_VECTORS:
+ assert(Node->getOperand(0).getValueType() == NewVT &&
+ "Concat of non-legal vectors not yet supported!");
+ Result = Node->getOperand(0);
+ break;
+ case ISD::VECTOR_SHUFFLE: {
+ // Figure out if the scalar is the LHS or RHS and return it.
+ SDOperand EltNum = Node->getOperand(2).getOperand(0);
+ if (cast<ConstantSDNode>(EltNum)->getValue())
+ Result = ScalarizeVectorOp(Node->getOperand(1));
+ else
+ Result = ScalarizeVectorOp(Node->getOperand(0));
+ break;
+ }
+ case ISD::EXTRACT_SUBVECTOR:
+ Result = Node->getOperand(0);
+ assert(Result.getValueType() == NewVT);
+ break;
+ case ISD::BIT_CONVERT:
+ Result = DAG.getNode(ISD::BIT_CONVERT, NewVT, Op.getOperand(0));
+ break;
+ case ISD::SELECT:
+ Result = DAG.getNode(ISD::SELECT, NewVT, Op.getOperand(0),
+ ScalarizeVectorOp(Op.getOperand(1)),
+ ScalarizeVectorOp(Op.getOperand(2)));
+ break;
+ }
+
+ if (TLI.isTypeLegal(NewVT))
+ Result = LegalizeOp(Result);
+ bool isNew = ScalarizedNodes.insert(std::make_pair(Op, Result)).second;
+ assert(isNew && "Value already scalarized?");
+ return Result;
+}
+
+
+// SelectionDAG::Legalize - This is the entry point for the file.
+//
+void SelectionDAG::Legalize() {
+ if (ViewLegalizeDAGs) viewGraph();
+
+ /// run - This is the main entry point to this class.
+ ///
+ SelectionDAGLegalize(*this).LegalizeDAG();
+}
+
diff --git a/lib/CodeGen/SelectionDAG/Makefile b/lib/CodeGen/SelectionDAG/Makefile
new file mode 100644
index 0000000..6c50288
--- /dev/null
+++ b/lib/CodeGen/SelectionDAG/Makefile
@@ -0,0 +1,15 @@
+##===- lib/CodeGen/SelectionDAG/Makefile -------------------*- Makefile -*-===##
+#
+# The LLVM Compiler Infrastructure
+#
+# This file was developed by the LLVM research group and is distributed under
+# the University of Illinois Open Source License. See LICENSE.TXT for details.
+#
+##===----------------------------------------------------------------------===##
+LEVEL = ../../..
+LIBRARYNAME = LLVMSelectionDAG
+PARALLEL_DIRS =
+BUILD_ARCHIVE = 1
+DONT_BUILD_RELINKED = 1
+
+include $(LEVEL)/Makefile.common
diff --git a/lib/CodeGen/SelectionDAG/ScheduleDAG.cpp b/lib/CodeGen/SelectionDAG/ScheduleDAG.cpp
new file mode 100644
index 0000000..06b2329
--- /dev/null
+++ b/lib/CodeGen/SelectionDAG/ScheduleDAG.cpp
@@ -0,0 +1,725 @@
+//===---- ScheduleDAG.cpp - Implement the ScheduleDAG class ---------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file was developed by James M. Laskey and is distributed under the
+// University of Illinois Open Source License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This implements a simple two pass scheduler. The first pass attempts to push
+// backward any lengthy instructions and critical paths. The second pass packs
+// instructions into semi-optimal time slots.
+//
+//===----------------------------------------------------------------------===//
+
+#define DEBUG_TYPE "pre-RA-sched"
+#include "llvm/Type.h"
+#include "llvm/CodeGen/ScheduleDAG.h"
+#include "llvm/CodeGen/MachineConstantPool.h"
+#include "llvm/CodeGen/MachineFunction.h"
+#include "llvm/CodeGen/SSARegMap.h"
+#include "llvm/Target/TargetData.h"
+#include "llvm/Target/TargetMachine.h"
+#include "llvm/Target/TargetInstrInfo.h"
+#include "llvm/Target/TargetLowering.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/MathExtras.h"
+using namespace llvm;
+
+/// BuildSchedUnits - Build SUnits from the selection dag that we are input.
+/// This SUnit graph is similar to the SelectionDAG, but represents flagged
+/// together nodes with a single SUnit.
+void ScheduleDAG::BuildSchedUnits() {
+ // Reserve entries in the vector for each of the SUnits we are creating. This
+ // ensure that reallocation of the vector won't happen, so SUnit*'s won't get
+ // invalidated.
+ SUnits.reserve(std::distance(DAG.allnodes_begin(), DAG.allnodes_end()));
+
+ const InstrItineraryData &InstrItins = TM.getInstrItineraryData();
+
+ for (SelectionDAG::allnodes_iterator NI = DAG.allnodes_begin(),
+ E = DAG.allnodes_end(); NI != E; ++NI) {
+ if (isPassiveNode(NI)) // Leaf node, e.g. a TargetImmediate.
+ continue;
+
+ // If this node has already been processed, stop now.
+ if (SUnitMap[NI]) continue;
+
+ SUnit *NodeSUnit = NewSUnit(NI);
+
+ // See if anything is flagged to this node, if so, add them to flagged
+ // nodes. Nodes can have at most one flag input and one flag output. Flags
+ // are required the be the last operand and result of a node.
+
+ // Scan up, adding flagged preds to FlaggedNodes.
+ SDNode *N = NI;
+ if (N->getNumOperands() &&
+ N->getOperand(N->getNumOperands()-1).getValueType() == MVT::Flag) {
+ do {
+ N = N->getOperand(N->getNumOperands()-1).Val;
+ NodeSUnit->FlaggedNodes.push_back(N);
+ SUnitMap[N] = NodeSUnit;
+ } while (N->getNumOperands() &&
+ N->getOperand(N->getNumOperands()-1).getValueType()== MVT::Flag);
+ std::reverse(NodeSUnit->FlaggedNodes.begin(),
+ NodeSUnit->FlaggedNodes.end());
+ }
+
+ // Scan down, adding this node and any flagged succs to FlaggedNodes if they
+ // have a user of the flag operand.
+ N = NI;
+ while (N->getValueType(N->getNumValues()-1) == MVT::Flag) {
+ SDOperand FlagVal(N, N->getNumValues()-1);
+
+ // There are either zero or one users of the Flag result.
+ bool HasFlagUse = false;
+ for (SDNode::use_iterator UI = N->use_begin(), E = N->use_end();
+ UI != E; ++UI)
+ if (FlagVal.isOperand(*UI)) {
+ HasFlagUse = true;
+ NodeSUnit->FlaggedNodes.push_back(N);
+ SUnitMap[N] = NodeSUnit;
+ N = *UI;
+ break;
+ }
+ if (!HasFlagUse) break;
+ }
+
+ // Now all flagged nodes are in FlaggedNodes and N is the bottom-most node.
+ // Update the SUnit
+ NodeSUnit->Node = N;
+ SUnitMap[N] = NodeSUnit;
+
+ // Compute the latency for the node. We use the sum of the latencies for
+ // all nodes flagged together into this SUnit.
+ if (InstrItins.isEmpty()) {
+ // No latency information.
+ NodeSUnit->Latency = 1;
+ } else {
+ NodeSUnit->Latency = 0;
+ if (N->isTargetOpcode()) {
+ unsigned SchedClass = TII->getSchedClass(N->getTargetOpcode());
+ InstrStage *S = InstrItins.begin(SchedClass);
+ InstrStage *E = InstrItins.end(SchedClass);
+ for (; S != E; ++S)
+ NodeSUnit->Latency += S->Cycles;
+ }
+ for (unsigned i = 0, e = NodeSUnit->FlaggedNodes.size(); i != e; ++i) {
+ SDNode *FNode = NodeSUnit->FlaggedNodes[i];
+ if (FNode->isTargetOpcode()) {
+ unsigned SchedClass = TII->getSchedClass(FNode->getTargetOpcode());
+ InstrStage *S = InstrItins.begin(SchedClass);
+ InstrStage *E = InstrItins.end(SchedClass);
+ for (; S != E; ++S)
+ NodeSUnit->Latency += S->Cycles;
+ }
+ }
+ }
+ }
+
+ // Pass 2: add the preds, succs, etc.
+ for (unsigned su = 0, e = SUnits.size(); su != e; ++su) {
+ SUnit *SU = &SUnits[su];
+ SDNode *MainNode = SU->Node;
+
+ if (MainNode->isTargetOpcode()) {
+ unsigned Opc = MainNode->getTargetOpcode();
+ for (unsigned i = 0, ee = TII->getNumOperands(Opc); i != ee; ++i) {
+ if (TII->getOperandConstraint(Opc, i, TOI::TIED_TO) != -1) {
+ SU->isTwoAddress = true;
+ break;
+ }
+ }
+ if (TII->isCommutableInstr(Opc))
+ SU->isCommutable = true;
+ }
+
+ // Find all predecessors and successors of the group.
+ // Temporarily add N to make code simpler.
+ SU->FlaggedNodes.push_back(MainNode);
+
+ for (unsigned n = 0, e = SU->FlaggedNodes.size(); n != e; ++n) {
+ SDNode *N = SU->FlaggedNodes[n];
+
+ for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) {
+ SDNode *OpN = N->getOperand(i).Val;
+ if (isPassiveNode(OpN)) continue; // Not scheduled.
+ SUnit *OpSU = SUnitMap[OpN];
+ assert(OpSU && "Node has no SUnit!");
+ if (OpSU == SU) continue; // In the same group.
+
+ MVT::ValueType OpVT = N->getOperand(i).getValueType();
+ assert(OpVT != MVT::Flag && "Flagged nodes should be in same sunit!");
+ bool isChain = OpVT == MVT::Other;
+
+ if (SU->addPred(OpSU, isChain)) {
+ if (!isChain) {
+ SU->NumPreds++;
+ SU->NumPredsLeft++;
+ } else {
+ SU->NumChainPredsLeft++;
+ }
+ }
+ if (OpSU->addSucc(SU, isChain)) {
+ if (!isChain) {
+ OpSU->NumSuccs++;
+ OpSU->NumSuccsLeft++;
+ } else {
+ OpSU->NumChainSuccsLeft++;
+ }
+ }
+ }
+ }
+
+ // Remove MainNode from FlaggedNodes again.
+ SU->FlaggedNodes.pop_back();
+ }
+
+ return;
+}
+
+void ScheduleDAG::CalculateDepths() {
+ std::vector<std::pair<SUnit*, unsigned> > WorkList;
+ for (unsigned i = 0, e = SUnits.size(); i != e; ++i)
+ if (SUnits[i].Preds.size() == 0/* && &SUnits[i] != Entry*/)
+ WorkList.push_back(std::make_pair(&SUnits[i], 0U));
+
+ while (!WorkList.empty()) {
+ SUnit *SU = WorkList.back().first;
+ unsigned Depth = WorkList.back().second;
+ WorkList.pop_back();
+ if (SU->Depth == 0 || Depth > SU->Depth) {
+ SU->Depth = Depth;
+ for (SUnit::succ_iterator I = SU->Succs.begin(), E = SU->Succs.end();
+ I != E; ++I)
+ WorkList.push_back(std::make_pair(I->first, Depth+1));
+ }
+ }
+}
+
+void ScheduleDAG::CalculateHeights() {
+ std::vector<std::pair<SUnit*, unsigned> > WorkList;
+ SUnit *Root = SUnitMap[DAG.getRoot().Val];
+ WorkList.push_back(std::make_pair(Root, 0U));
+
+ while (!WorkList.empty()) {
+ SUnit *SU = WorkList.back().first;
+ unsigned Height = WorkList.back().second;
+ WorkList.pop_back();
+ if (SU->Height == 0 || Height > SU->Height) {
+ SU->Height = Height;
+ for (SUnit::pred_iterator I = SU->Preds.begin(), E = SU->Preds.end();
+ I != E; ++I)
+ WorkList.push_back(std::make_pair(I->first, Height+1));
+ }
+ }
+}
+
+/// CountResults - The results of target nodes have register or immediate
+/// operands first, then an optional chain, and optional flag operands (which do
+/// not go into the machine instrs.)
+unsigned ScheduleDAG::CountResults(SDNode *Node) {
+ unsigned N = Node->getNumValues();
+ while (N && Node->getValueType(N - 1) == MVT::Flag)
+ --N;
+ if (N && Node->getValueType(N - 1) == MVT::Other)
+ --N; // Skip over chain result.
+ return N;
+}
+
+/// CountOperands The inputs to target nodes have any actual inputs first,
+/// followed by an optional chain operand, then flag operands. Compute the
+/// number of actual operands that will go into the machine instr.
+unsigned ScheduleDAG::CountOperands(SDNode *Node) {
+ unsigned N = Node->getNumOperands();
+ while (N && Node->getOperand(N - 1).getValueType() == MVT::Flag)
+ --N;
+ if (N && Node->getOperand(N - 1).getValueType() == MVT::Other)
+ --N; // Ignore chain if it exists.
+ return N;
+}
+
+static const TargetRegisterClass *getInstrOperandRegClass(
+ const MRegisterInfo *MRI,
+ const TargetInstrInfo *TII,
+ const TargetInstrDescriptor *II,
+ unsigned Op) {
+ if (Op >= II->numOperands) {
+ assert((II->Flags & M_VARIABLE_OPS)&& "Invalid operand # of instruction");
+ return NULL;
+ }
+ const TargetOperandInfo &toi = II->OpInfo[Op];
+ return (toi.Flags & M_LOOK_UP_PTR_REG_CLASS)
+ ? TII->getPointerRegClass() : MRI->getRegClass(toi.RegClass);
+}
+
+static void CreateVirtualRegisters(SDNode *Node,
+ unsigned NumResults,
+ const MRegisterInfo *MRI,
+ MachineInstr *MI,
+ SSARegMap *RegMap,
+ const TargetInstrInfo *TII,
+ const TargetInstrDescriptor &II,
+ DenseMap<SDOperand, unsigned> &VRBaseMap) {
+ for (unsigned i = 0; i < NumResults; ++i) {
+ // If the specific node value is only used by a CopyToReg and the dest reg
+ // is a vreg, use the CopyToReg'd destination register instead of creating
+ // a new vreg.
+ unsigned VRBase = 0;
+ for (SDNode::use_iterator UI = Node->use_begin(), E = Node->use_end();
+ UI != E; ++UI) {
+ SDNode *Use = *UI;
+ if (Use->getOpcode() == ISD::CopyToReg &&
+ Use->getOperand(2).Val == Node &&
+ Use->getOperand(2).ResNo == i) {
+ unsigned Reg = cast<RegisterSDNode>(Use->getOperand(1))->getReg();
+ if (MRegisterInfo::isVirtualRegister(Reg)) {
+ VRBase = Reg;
+ MI->addRegOperand(Reg, true);
+ break;
+ }
+ }
+ }
+
+ if (VRBase == 0) {
+ // Create the result registers for this node and add the result regs to
+ // the machine instruction.
+ const TargetRegisterClass *RC = getInstrOperandRegClass(MRI, TII, &II, i);
+ assert(RC && "Isn't a register operand!");
+ VRBase = RegMap->createVirtualRegister(RC);
+ MI->addRegOperand(VRBase, true);
+ }
+
+ bool isNew = VRBaseMap.insert(std::make_pair(SDOperand(Node,i), VRBase));
+ assert(isNew && "Node emitted out of order - early");
+ }
+}
+
+/// getVR - Return the virtual register corresponding to the specified result
+/// of the specified node.
+static unsigned getVR(SDOperand Op, DenseMap<SDOperand, unsigned> &VRBaseMap) {
+ DenseMap<SDOperand, unsigned>::iterator I = VRBaseMap.find(Op);
+ assert(I != VRBaseMap.end() && "Node emitted out of order - late");
+ return I->second;
+}
+
+
+/// AddOperand - Add the specified operand to the specified machine instr. II
+/// specifies the instruction information for the node, and IIOpNum is the
+/// operand number (in the II) that we are adding. IIOpNum and II are used for
+/// assertions only.
+void ScheduleDAG::AddOperand(MachineInstr *MI, SDOperand Op,
+ unsigned IIOpNum,
+ const TargetInstrDescriptor *II,
+ DenseMap<SDOperand, unsigned> &VRBaseMap) {
+ if (Op.isTargetOpcode()) {
+ // Note that this case is redundant with the final else block, but we
+ // include it because it is the most common and it makes the logic
+ // simpler here.
+ assert(Op.getValueType() != MVT::Other &&
+ Op.getValueType() != MVT::Flag &&
+ "Chain and flag operands should occur at end of operand list!");
+
+ // Get/emit the operand.
+ unsigned VReg = getVR(Op, VRBaseMap);
+ const TargetInstrDescriptor *TID = MI->getInstrDescriptor();
+ bool isOptDef = (IIOpNum < TID->numOperands)
+ ? (TID->OpInfo[IIOpNum].Flags & M_OPTIONAL_DEF_OPERAND) : false;
+ MI->addRegOperand(VReg, isOptDef);
+
+ // Verify that it is right.
+ assert(MRegisterInfo::isVirtualRegister(VReg) && "Not a vreg?");
+ if (II) {
+ const TargetRegisterClass *RC =
+ getInstrOperandRegClass(MRI, TII, II, IIOpNum);
+ assert(RC && "Don't have operand info for this instruction!");
+ const TargetRegisterClass *VRC = RegMap->getRegClass(VReg);
+ if (VRC != RC) {
+ cerr << "Register class of operand and regclass of use don't agree!\n";
+#ifndef NDEBUG
+ cerr << "Operand = " << IIOpNum << "\n";
+ cerr << "Op->Val = "; Op.Val->dump(&DAG); cerr << "\n";
+ cerr << "MI = "; MI->print(cerr);
+ cerr << "VReg = " << VReg << "\n";
+ cerr << "VReg RegClass size = " << VRC->getSize()
+ << ", align = " << VRC->getAlignment() << "\n";
+ cerr << "Expected RegClass size = " << RC->getSize()
+ << ", align = " << RC->getAlignment() << "\n";
+#endif
+ cerr << "Fatal error, aborting.\n";
+ abort();
+ }
+ }
+ } else if (ConstantSDNode *C =
+ dyn_cast<ConstantSDNode>(Op)) {
+ MI->addImmOperand(C->getValue());
+ } else if (RegisterSDNode *R =
+ dyn_cast<RegisterSDNode>(Op)) {
+ MI->addRegOperand(R->getReg(), false);
+ } else if (GlobalAddressSDNode *TGA =
+ dyn_cast<GlobalAddressSDNode>(Op)) {
+ MI->addGlobalAddressOperand(TGA->getGlobal(), TGA->getOffset());
+ } else if (BasicBlockSDNode *BB =
+ dyn_cast<BasicBlockSDNode>(Op)) {
+ MI->addMachineBasicBlockOperand(BB->getBasicBlock());
+ } else if (FrameIndexSDNode *FI =
+ dyn_cast<FrameIndexSDNode>(Op)) {
+ MI->addFrameIndexOperand(FI->getIndex());
+ } else if (JumpTableSDNode *JT =
+ dyn_cast<JumpTableSDNode>(Op)) {
+ MI->addJumpTableIndexOperand(JT->getIndex());
+ } else if (ConstantPoolSDNode *CP =
+ dyn_cast<ConstantPoolSDNode>(Op)) {
+ int Offset = CP->getOffset();
+ unsigned Align = CP->getAlignment();
+ const Type *Type = CP->getType();
+ // MachineConstantPool wants an explicit alignment.
+ if (Align == 0) {
+ Align = TM.getTargetData()->getPreferredTypeAlignmentShift(Type);
+ if (Align == 0) {
+ // Alignment of vector types. FIXME!
+ Align = TM.getTargetData()->getTypeSize(Type);
+ Align = Log2_64(Align);
+ }
+ }
+
+ unsigned Idx;
+ if (CP->isMachineConstantPoolEntry())
+ Idx = ConstPool->getConstantPoolIndex(CP->getMachineCPVal(), Align);
+ else
+ Idx = ConstPool->getConstantPoolIndex(CP->getConstVal(), Align);
+ MI->addConstantPoolIndexOperand(Idx, Offset);
+ } else if (ExternalSymbolSDNode *ES =
+ dyn_cast<ExternalSymbolSDNode>(Op)) {
+ MI->addExternalSymbolOperand(ES->getSymbol());
+ } else {
+ assert(Op.getValueType() != MVT::Other &&
+ Op.getValueType() != MVT::Flag &&
+ "Chain and flag operands should occur at end of operand list!");
+ unsigned VReg = getVR(Op, VRBaseMap);
+ MI->addRegOperand(VReg, false);
+
+ // Verify that it is right.
+ assert(MRegisterInfo::isVirtualRegister(VReg) && "Not a vreg?");
+ if (II) {
+ const TargetRegisterClass *RC =
+ getInstrOperandRegClass(MRI, TII, II, IIOpNum);
+ assert(RC && "Don't have operand info for this instruction!");
+ assert(RegMap->getRegClass(VReg) == RC &&
+ "Register class of operand and regclass of use don't agree!");
+ }
+ }
+
+}
+
+// Returns the Register Class of a physical register
+static const TargetRegisterClass *getPhysicalRegisterRegClass(
+ const MRegisterInfo *MRI,
+ MVT::ValueType VT,
+ unsigned reg) {
+ assert(MRegisterInfo::isPhysicalRegister(reg) &&
+ "reg must be a physical register");
+ // Pick the register class of the right type that contains this physreg.
+ for (MRegisterInfo::regclass_iterator I = MRI->regclass_begin(),
+ E = MRI->regclass_end(); I != E; ++I)
+ if ((*I)->hasType(VT) && (*I)->contains(reg))
+ return *I;
+ assert(false && "Couldn't find the register class");
+ return 0;
+}
+
+/// EmitNode - Generate machine code for an node and needed dependencies.
+///
+void ScheduleDAG::EmitNode(SDNode *Node,
+ DenseMap<SDOperand, unsigned> &VRBaseMap) {
+ // If machine instruction
+ if (Node->isTargetOpcode()) {
+ unsigned Opc = Node->getTargetOpcode();
+ const TargetInstrDescriptor &II = TII->get(Opc);
+
+ unsigned NumResults = CountResults(Node);
+ unsigned NodeOperands = CountOperands(Node);
+ unsigned NumMIOperands = NodeOperands + NumResults;
+#ifndef NDEBUG
+ assert((unsigned(II.numOperands) == NumMIOperands ||
+ (II.Flags & M_VARIABLE_OPS)) &&
+ "#operands for dag node doesn't match .td file!");
+#endif
+
+ // Create the new machine instruction.
+ MachineInstr *MI = new MachineInstr(II);
+
+ // Add result register values for things that are defined by this
+ // instruction.
+ if (NumResults)
+ CreateVirtualRegisters(Node, NumResults, MRI, MI, RegMap,
+ TII, II, VRBaseMap);
+
+ // Emit all of the actual operands of this instruction, adding them to the
+ // instruction as appropriate.
+ for (unsigned i = 0; i != NodeOperands; ++i)
+ AddOperand(MI, Node->getOperand(i), i+NumResults, &II, VRBaseMap);
+
+ // Commute node if it has been determined to be profitable.
+ if (CommuteSet.count(Node)) {
+ MachineInstr *NewMI = TII->commuteInstruction(MI);
+ if (NewMI == 0)
+ DOUT << "Sched: COMMUTING FAILED!\n";
+ else {
+ DOUT << "Sched: COMMUTED TO: " << *NewMI;
+ if (MI != NewMI) {
+ delete MI;
+ MI = NewMI;
+ }
+ }
+ }
+
+ // Now that we have emitted all operands, emit this instruction itself.
+ if ((II.Flags & M_USES_CUSTOM_DAG_SCHED_INSERTION) == 0) {
+ BB->insert(BB->end(), MI);
+ } else {
+ // Insert this instruction into the end of the basic block, potentially
+ // taking some custom action.
+ BB = DAG.getTargetLoweringInfo().InsertAtEndOfBasicBlock(MI, BB);
+ }
+ } else {
+ switch (Node->getOpcode()) {
+ default:
+#ifndef NDEBUG
+ Node->dump(&DAG);
+#endif
+ assert(0 && "This target-independent node should have been selected!");
+ case ISD::EntryToken: // fall thru
+ case ISD::TokenFactor:
+ case ISD::LABEL:
+ break;
+ case ISD::CopyToReg: {
+ unsigned InReg;
+ if (RegisterSDNode *R = dyn_cast<RegisterSDNode>(Node->getOperand(2)))
+ InReg = R->getReg();
+ else
+ InReg = getVR(Node->getOperand(2), VRBaseMap);
+ unsigned DestReg = cast<RegisterSDNode>(Node->getOperand(1))->getReg();
+ if (InReg != DestReg) {// Coalesced away the copy?
+ const TargetRegisterClass *TRC = 0;
+ // Get the target register class
+ if (MRegisterInfo::isVirtualRegister(InReg))
+ TRC = RegMap->getRegClass(InReg);
+ else
+ TRC = getPhysicalRegisterRegClass(MRI,
+ Node->getOperand(2).getValueType(),
+ InReg);
+ MRI->copyRegToReg(*BB, BB->end(), DestReg, InReg, TRC);
+ }
+ break;
+ }
+ case ISD::CopyFromReg: {
+ unsigned VRBase = 0;
+ unsigned SrcReg = cast<RegisterSDNode>(Node->getOperand(1))->getReg();
+ if (MRegisterInfo::isVirtualRegister(SrcReg)) {
+ // Just use the input register directly!
+ bool isNew = VRBaseMap.insert(std::make_pair(SDOperand(Node,0),SrcReg));
+ assert(isNew && "Node emitted out of order - early");
+ break;
+ }
+
+ // If the node is only used by a CopyToReg and the dest reg is a vreg, use
+ // the CopyToReg'd destination register instead of creating a new vreg.
+ for (SDNode::use_iterator UI = Node->use_begin(), E = Node->use_end();
+ UI != E; ++UI) {
+ SDNode *Use = *UI;
+ if (Use->getOpcode() == ISD::CopyToReg &&
+ Use->getOperand(2).Val == Node) {
+ unsigned DestReg = cast<RegisterSDNode>(Use->getOperand(1))->getReg();
+ if (MRegisterInfo::isVirtualRegister(DestReg)) {
+ VRBase = DestReg;
+ break;
+ }
+ }
+ }
+
+ // Figure out the register class to create for the destreg.
+ const TargetRegisterClass *TRC = 0;
+ if (VRBase) {
+ TRC = RegMap->getRegClass(VRBase);
+ } else {
+ TRC = getPhysicalRegisterRegClass(MRI, Node->getValueType(0), SrcReg);
+
+ // Create the reg, emit the copy.
+ VRBase = RegMap->createVirtualRegister(TRC);
+ }
+ MRI->copyRegToReg(*BB, BB->end(), VRBase, SrcReg, TRC);
+
+ bool isNew = VRBaseMap.insert(std::make_pair(SDOperand(Node,0), VRBase));
+ assert(isNew && "Node emitted out of order - early");
+ break;
+ }
+ case ISD::INLINEASM: {
+ unsigned NumOps = Node->getNumOperands();
+ if (Node->getOperand(NumOps-1).getValueType() == MVT::Flag)
+ --NumOps; // Ignore the flag operand.
+
+ // Create the inline asm machine instruction.
+ MachineInstr *MI =
+ new MachineInstr(BB, TII->get(TargetInstrInfo::INLINEASM));
+
+ // Add the asm string as an external symbol operand.
+ const char *AsmStr =
+ cast<ExternalSymbolSDNode>(Node->getOperand(1))->getSymbol();
+ MI->addExternalSymbolOperand(AsmStr);
+
+ // Add all of the operand registers to the instruction.
+ for (unsigned i = 2; i != NumOps;) {
+ unsigned Flags = cast<ConstantSDNode>(Node->getOperand(i))->getValue();
+ unsigned NumVals = Flags >> 3;
+
+ MI->addImmOperand(Flags);
+ ++i; // Skip the ID value.
+
+ switch (Flags & 7) {
+ default: assert(0 && "Bad flags!");
+ case 1: // Use of register.
+ for (; NumVals; --NumVals, ++i) {
+ unsigned Reg = cast<RegisterSDNode>(Node->getOperand(i))->getReg();
+ MI->addRegOperand(Reg, false);
+ }
+ break;
+ case 2: // Def of register.
+ for (; NumVals; --NumVals, ++i) {
+ unsigned Reg = cast<RegisterSDNode>(Node->getOperand(i))->getReg();
+ MI->addRegOperand(Reg, true);
+ }
+ break;
+ case 3: { // Immediate.
+ assert(NumVals == 1 && "Unknown immediate value!");
+ if (ConstantSDNode *CS=dyn_cast<ConstantSDNode>(Node->getOperand(i))){
+ MI->addImmOperand(CS->getValue());
+ } else {
+ GlobalAddressSDNode *GA =
+ cast<GlobalAddressSDNode>(Node->getOperand(i));
+ MI->addGlobalAddressOperand(GA->getGlobal(), GA->getOffset());
+ }
+ ++i;
+ break;
+ }
+ case 4: // Addressing mode.
+ // The addressing mode has been selected, just add all of the
+ // operands to the machine instruction.
+ for (; NumVals; --NumVals, ++i)
+ AddOperand(MI, Node->getOperand(i), 0, 0, VRBaseMap);
+ break;
+ }
+ }
+ break;
+ }
+ }
+ }
+}
+
+void ScheduleDAG::EmitNoop() {
+ TII->insertNoop(*BB, BB->end());
+}
+
+/// EmitSchedule - Emit the machine code in scheduled order.
+void ScheduleDAG::EmitSchedule() {
+ // If this is the first basic block in the function, and if it has live ins
+ // that need to be copied into vregs, emit the copies into the top of the
+ // block before emitting the code for the block.
+ MachineFunction &MF = DAG.getMachineFunction();
+ if (&MF.front() == BB && MF.livein_begin() != MF.livein_end()) {
+ for (MachineFunction::livein_iterator LI = MF.livein_begin(),
+ E = MF.livein_end(); LI != E; ++LI)
+ if (LI->second)
+ MRI->copyRegToReg(*MF.begin(), MF.begin()->end(), LI->second,
+ LI->first, RegMap->getRegClass(LI->second));
+ }
+
+
+ // Finally, emit the code for all of the scheduled instructions.
+ DenseMap<SDOperand, unsigned> VRBaseMap;
+ for (unsigned i = 0, e = Sequence.size(); i != e; i++) {
+ if (SUnit *SU = Sequence[i]) {
+ for (unsigned j = 0, ee = SU->FlaggedNodes.size(); j != ee; j++)
+ EmitNode(SU->FlaggedNodes[j], VRBaseMap);
+ EmitNode(SU->Node, VRBaseMap);
+ } else {
+ // Null SUnit* is a noop.
+ EmitNoop();
+ }
+ }
+}
+
+/// dump - dump the schedule.
+void ScheduleDAG::dumpSchedule() const {
+ for (unsigned i = 0, e = Sequence.size(); i != e; i++) {
+ if (SUnit *SU = Sequence[i])
+ SU->dump(&DAG);
+ else
+ cerr << "**** NOOP ****\n";
+ }
+}
+
+
+/// Run - perform scheduling.
+///
+MachineBasicBlock *ScheduleDAG::Run() {
+ TII = TM.getInstrInfo();
+ MRI = TM.getRegisterInfo();
+ RegMap = BB->getParent()->getSSARegMap();
+ ConstPool = BB->getParent()->getConstantPool();
+
+ Schedule();
+ return BB;
+}
+
+/// SUnit - Scheduling unit. It's an wrapper around either a single SDNode or
+/// a group of nodes flagged together.
+void SUnit::dump(const SelectionDAG *G) const {
+ cerr << "SU(" << NodeNum << "): ";
+ Node->dump(G);
+ cerr << "\n";
+ if (FlaggedNodes.size() != 0) {
+ for (unsigned i = 0, e = FlaggedNodes.size(); i != e; i++) {
+ cerr << " ";
+ FlaggedNodes[i]->dump(G);
+ cerr << "\n";
+ }
+ }
+}
+
+void SUnit::dumpAll(const SelectionDAG *G) const {
+ dump(G);
+
+ cerr << " # preds left : " << NumPredsLeft << "\n";
+ cerr << " # succs left : " << NumSuccsLeft << "\n";
+ cerr << " # chain preds left : " << NumChainPredsLeft << "\n";
+ cerr << " # chain succs left : " << NumChainSuccsLeft << "\n";
+ cerr << " Latency : " << Latency << "\n";
+ cerr << " Depth : " << Depth << "\n";
+ cerr << " Height : " << Height << "\n";
+
+ if (Preds.size() != 0) {
+ cerr << " Predecessors:\n";
+ for (SUnit::const_succ_iterator I = Preds.begin(), E = Preds.end();
+ I != E; ++I) {
+ if (I->second)
+ cerr << " ch #";
+ else
+ cerr << " val #";
+ cerr << I->first << " - SU(" << I->first->NodeNum << ")\n";
+ }
+ }
+ if (Succs.size() != 0) {
+ cerr << " Successors:\n";
+ for (SUnit::const_succ_iterator I = Succs.begin(), E = Succs.end();
+ I != E; ++I) {
+ if (I->second)
+ cerr << " ch #";
+ else
+ cerr << " val #";
+ cerr << I->first << " - SU(" << I->first->NodeNum << ")\n";
+ }
+ }
+ cerr << "\n";
+}
diff --git a/lib/CodeGen/SelectionDAG/ScheduleDAGList.cpp b/lib/CodeGen/SelectionDAG/ScheduleDAGList.cpp
new file mode 100644
index 0000000..9e4e46f
--- /dev/null
+++ b/lib/CodeGen/SelectionDAG/ScheduleDAGList.cpp
@@ -0,0 +1,531 @@
+//===---- ScheduleDAGList.cpp - Implement a list scheduler for isel DAG ---===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file was developed by Evan Cheng and is distributed under the
+// University of Illinois Open Source License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This implements a top-down list scheduler, using standard algorithms.
+// The basic approach uses a priority queue of available nodes to schedule.
+// One at a time, nodes are taken from the priority queue (thus in priority
+// order), checked for legality to schedule, and emitted if legal.
+//
+// Nodes may not be legal to schedule either due to structural hazards (e.g.
+// pipeline or resource constraints) or because an input to the instruction has
+// not completed execution.
+//
+//===----------------------------------------------------------------------===//
+
+#define DEBUG_TYPE "pre-RA-sched"
+#include "llvm/CodeGen/ScheduleDAG.h"
+#include "llvm/CodeGen/SchedulerRegistry.h"
+#include "llvm/CodeGen/SelectionDAGISel.h"
+#include "llvm/CodeGen/SSARegMap.h"
+#include "llvm/Target/MRegisterInfo.h"
+#include "llvm/Target/TargetData.h"
+#include "llvm/Target/TargetMachine.h"
+#include "llvm/Target/TargetInstrInfo.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/Compiler.h"
+#include "llvm/ADT/Statistic.h"
+#include <climits>
+#include <queue>
+using namespace llvm;
+
+STATISTIC(NumNoops , "Number of noops inserted");
+STATISTIC(NumStalls, "Number of pipeline stalls");
+
+static RegisterScheduler
+ tdListDAGScheduler("list-td", " Top-down list scheduler",
+ createTDListDAGScheduler);
+
+namespace {
+//===----------------------------------------------------------------------===//
+/// ScheduleDAGList - The actual list scheduler implementation. This supports
+/// top-down scheduling.
+///
+class VISIBILITY_HIDDEN ScheduleDAGList : public ScheduleDAG {
+private:
+ /// AvailableQueue - The priority queue to use for the available SUnits.
+ ///
+ SchedulingPriorityQueue *AvailableQueue;
+
+ /// PendingQueue - This contains all of the instructions whose operands have
+ /// been issued, but their results are not ready yet (due to the latency of
+ /// the operation). Once the operands becomes available, the instruction is
+ /// added to the AvailableQueue. This keeps track of each SUnit and the
+ /// number of cycles left to execute before the operation is available.
+ std::vector<std::pair<unsigned, SUnit*> > PendingQueue;
+
+ /// HazardRec - The hazard recognizer to use.
+ HazardRecognizer *HazardRec;
+
+public:
+ ScheduleDAGList(SelectionDAG &dag, MachineBasicBlock *bb,
+ const TargetMachine &tm,
+ SchedulingPriorityQueue *availqueue,
+ HazardRecognizer *HR)
+ : ScheduleDAG(dag, bb, tm),
+ AvailableQueue(availqueue), HazardRec(HR) {
+ }
+
+ ~ScheduleDAGList() {
+ delete HazardRec;
+ delete AvailableQueue;
+ }
+
+ void Schedule();
+
+private:
+ void ReleaseSucc(SUnit *SuccSU, bool isChain);
+ void ScheduleNodeTopDown(SUnit *SU, unsigned CurCycle);
+ void ListScheduleTopDown();
+};
+} // end anonymous namespace
+
+HazardRecognizer::~HazardRecognizer() {}
+
+
+/// Schedule - Schedule the DAG using list scheduling.
+void ScheduleDAGList::Schedule() {
+ DOUT << "********** List Scheduling **********\n";
+
+ // Build scheduling units.
+ BuildSchedUnits();
+
+ AvailableQueue->initNodes(SUnitMap, SUnits);
+
+ ListScheduleTopDown();
+
+ AvailableQueue->releaseState();
+
+ DOUT << "*** Final schedule ***\n";
+ DEBUG(dumpSchedule());
+ DOUT << "\n";
+
+ // Emit in scheduled order
+ EmitSchedule();
+}
+
+//===----------------------------------------------------------------------===//
+// Top-Down Scheduling
+//===----------------------------------------------------------------------===//
+
+/// ReleaseSucc - Decrement the NumPredsLeft count of a successor. Add it to
+/// the PendingQueue if the count reaches zero.
+void ScheduleDAGList::ReleaseSucc(SUnit *SuccSU, bool isChain) {
+ if (!isChain)
+ SuccSU->NumPredsLeft--;
+ else
+ SuccSU->NumChainPredsLeft--;
+
+ assert(SuccSU->NumPredsLeft >= 0 && SuccSU->NumChainPredsLeft >= 0 &&
+ "List scheduling internal error");
+
+ if ((SuccSU->NumPredsLeft + SuccSU->NumChainPredsLeft) == 0) {
+ // Compute how many cycles it will be before this actually becomes
+ // available. This is the max of the start time of all predecessors plus
+ // their latencies.
+ unsigned AvailableCycle = 0;
+ for (SUnit::pred_iterator I = SuccSU->Preds.begin(),
+ E = SuccSU->Preds.end(); I != E; ++I) {
+ // If this is a token edge, we don't need to wait for the latency of the
+ // preceeding instruction (e.g. a long-latency load) unless there is also
+ // some other data dependence.
+ SUnit &Pred = *I->first;
+ unsigned PredDoneCycle = Pred.Cycle;
+ if (!I->second)
+ PredDoneCycle += Pred.Latency;
+ else if (Pred.Latency)
+ PredDoneCycle += 1;
+
+ AvailableCycle = std::max(AvailableCycle, PredDoneCycle);
+ }
+
+ PendingQueue.push_back(std::make_pair(AvailableCycle, SuccSU));
+ }
+}
+
+/// ScheduleNodeTopDown - Add the node to the schedule. Decrement the pending
+/// count of its successors. If a successor pending count is zero, add it to
+/// the Available queue.
+void ScheduleDAGList::ScheduleNodeTopDown(SUnit *SU, unsigned CurCycle) {
+ DOUT << "*** Scheduling [" << CurCycle << "]: ";
+ DEBUG(SU->dump(&DAG));
+
+ Sequence.push_back(SU);
+ SU->Cycle = CurCycle;
+
+ // Bottom up: release successors.
+ for (SUnit::succ_iterator I = SU->Succs.begin(), E = SU->Succs.end();
+ I != E; ++I)
+ ReleaseSucc(I->first, I->second);
+}
+
+/// ListScheduleTopDown - The main loop of list scheduling for top-down
+/// schedulers.
+void ScheduleDAGList::ListScheduleTopDown() {
+ unsigned CurCycle = 0;
+ SUnit *Entry = SUnitMap[DAG.getEntryNode().Val];
+
+ // All leaves to Available queue.
+ for (unsigned i = 0, e = SUnits.size(); i != e; ++i) {
+ // It is available if it has no predecessors.
+ if (SUnits[i].Preds.size() == 0 && &SUnits[i] != Entry) {
+ AvailableQueue->push(&SUnits[i]);
+ SUnits[i].isAvailable = SUnits[i].isPending = true;
+ }
+ }
+
+ // Emit the entry node first.
+ ScheduleNodeTopDown(Entry, CurCycle);
+ HazardRec->EmitInstruction(Entry->Node);
+
+ // While Available queue is not empty, grab the node with the highest
+ // priority. If it is not ready put it back. Schedule the node.
+ std::vector<SUnit*> NotReady;
+ while (!AvailableQueue->empty() || !PendingQueue.empty()) {
+ // Check to see if any of the pending instructions are ready to issue. If
+ // so, add them to the available queue.
+ for (unsigned i = 0, e = PendingQueue.size(); i != e; ++i) {
+ if (PendingQueue[i].first == CurCycle) {
+ AvailableQueue->push(PendingQueue[i].second);
+ PendingQueue[i].second->isAvailable = true;
+ PendingQueue[i] = PendingQueue.back();
+ PendingQueue.pop_back();
+ --i; --e;
+ } else {
+ assert(PendingQueue[i].first > CurCycle && "Negative latency?");
+ }
+ }
+
+ // If there are no instructions available, don't try to issue anything, and
+ // don't advance the hazard recognizer.
+ if (AvailableQueue->empty()) {
+ ++CurCycle;
+ continue;
+ }
+
+ SUnit *FoundSUnit = 0;
+ SDNode *FoundNode = 0;
+
+ bool HasNoopHazards = false;
+ while (!AvailableQueue->empty()) {
+ SUnit *CurSUnit = AvailableQueue->pop();
+
+ // Get the node represented by this SUnit.
+ FoundNode = CurSUnit->Node;
+
+ // If this is a pseudo op, like copyfromreg, look to see if there is a
+ // real target node flagged to it. If so, use the target node.
+ for (unsigned i = 0, e = CurSUnit->FlaggedNodes.size();
+ FoundNode->getOpcode() < ISD::BUILTIN_OP_END && i != e; ++i)
+ FoundNode = CurSUnit->FlaggedNodes[i];
+
+ HazardRecognizer::HazardType HT = HazardRec->getHazardType(FoundNode);
+ if (HT == HazardRecognizer::NoHazard) {
+ FoundSUnit = CurSUnit;
+ break;
+ }
+
+ // Remember if this is a noop hazard.
+ HasNoopHazards |= HT == HazardRecognizer::NoopHazard;
+
+ NotReady.push_back(CurSUnit);
+ }
+
+ // Add the nodes that aren't ready back onto the available list.
+ if (!NotReady.empty()) {
+ AvailableQueue->push_all(NotReady);
+ NotReady.clear();
+ }
+
+ // If we found a node to schedule, do it now.
+ if (FoundSUnit) {
+ ScheduleNodeTopDown(FoundSUnit, CurCycle);
+ HazardRec->EmitInstruction(FoundNode);
+ FoundSUnit->isScheduled = true;
+ AvailableQueue->ScheduledNode(FoundSUnit);
+
+ // If this is a pseudo-op node, we don't want to increment the current
+ // cycle.
+ if (FoundSUnit->Latency) // Don't increment CurCycle for pseudo-ops!
+ ++CurCycle;
+ } else if (!HasNoopHazards) {
+ // Otherwise, we have a pipeline stall, but no other problem, just advance
+ // the current cycle and try again.
+ DOUT << "*** Advancing cycle, no work to do\n";
+ HazardRec->AdvanceCycle();
+ ++NumStalls;
+ ++CurCycle;
+ } else {
+ // Otherwise, we have no instructions to issue and we have instructions
+ // that will fault if we don't do this right. This is the case for
+ // processors without pipeline interlocks and other cases.
+ DOUT << "*** Emitting noop\n";
+ HazardRec->EmitNoop();
+ Sequence.push_back(0); // NULL SUnit* -> noop
+ ++NumNoops;
+ ++CurCycle;
+ }
+ }
+
+#ifndef NDEBUG
+ // Verify that all SUnits were scheduled.
+ bool AnyNotSched = false;
+ for (unsigned i = 0, e = SUnits.size(); i != e; ++i) {
+ if (SUnits[i].NumPredsLeft != 0 || SUnits[i].NumChainPredsLeft != 0) {
+ if (!AnyNotSched)
+ cerr << "*** List scheduling failed! ***\n";
+ SUnits[i].dump(&DAG);
+ cerr << "has not been scheduled!\n";
+ AnyNotSched = true;
+ }
+ }
+ assert(!AnyNotSched);
+#endif
+}
+
+//===----------------------------------------------------------------------===//
+// LatencyPriorityQueue Implementation
+//===----------------------------------------------------------------------===//
+//
+// This is a SchedulingPriorityQueue that schedules using latency information to
+// reduce the length of the critical path through the basic block.
+//
+namespace {
+ class LatencyPriorityQueue;
+
+ /// Sorting functions for the Available queue.
+ struct latency_sort : public std::binary_function<SUnit*, SUnit*, bool> {
+ LatencyPriorityQueue *PQ;
+ latency_sort(LatencyPriorityQueue *pq) : PQ(pq) {}
+ latency_sort(const latency_sort &RHS) : PQ(RHS.PQ) {}
+
+ bool operator()(const SUnit* left, const SUnit* right) const;
+ };
+} // end anonymous namespace
+
+namespace {
+ class LatencyPriorityQueue : public SchedulingPriorityQueue {
+ // SUnits - The SUnits for the current graph.
+ std::vector<SUnit> *SUnits;
+
+ // Latencies - The latency (max of latency from this node to the bb exit)
+ // for each node.
+ std::vector<int> Latencies;
+
+ /// NumNodesSolelyBlocking - This vector contains, for every node in the
+ /// Queue, the number of nodes that the node is the sole unscheduled
+ /// predecessor for. This is used as a tie-breaker heuristic for better
+ /// mobility.
+ std::vector<unsigned> NumNodesSolelyBlocking;
+
+ std::priority_queue<SUnit*, std::vector<SUnit*>, latency_sort> Queue;
+public:
+ LatencyPriorityQueue() : Queue(latency_sort(this)) {
+ }
+
+ void initNodes(DenseMap<SDNode*, SUnit*> &sumap,
+ std::vector<SUnit> &sunits) {
+ SUnits = &sunits;
+ // Calculate node priorities.
+ CalculatePriorities();
+ }
+ void releaseState() {
+ SUnits = 0;
+ Latencies.clear();
+ }
+
+ unsigned getLatency(unsigned NodeNum) const {
+ assert(NodeNum < Latencies.size());
+ return Latencies[NodeNum];
+ }
+
+ unsigned getNumSolelyBlockNodes(unsigned NodeNum) const {
+ assert(NodeNum < NumNodesSolelyBlocking.size());
+ return NumNodesSolelyBlocking[NodeNum];
+ }
+
+ bool empty() const { return Queue.empty(); }
+
+ virtual void push(SUnit *U) {
+ push_impl(U);
+ }
+ void push_impl(SUnit *U);
+
+ void push_all(const std::vector<SUnit *> &Nodes) {
+ for (unsigned i = 0, e = Nodes.size(); i != e; ++i)
+ push_impl(Nodes[i]);
+ }
+
+ SUnit *pop() {
+ if (empty()) return NULL;
+ SUnit *V = Queue.top();
+ Queue.pop();
+ return V;
+ }
+
+ // ScheduledNode - As nodes are scheduled, we look to see if there are any
+ // successor nodes that have a single unscheduled predecessor. If so, that
+ // single predecessor has a higher priority, since scheduling it will make
+ // the node available.
+ void ScheduledNode(SUnit *Node);
+
+private:
+ void CalculatePriorities();
+ int CalcLatency(const SUnit &SU);
+ void AdjustPriorityOfUnscheduledPreds(SUnit *SU);
+ SUnit *getSingleUnscheduledPred(SUnit *SU);
+
+ /// RemoveFromPriorityQueue - This is a really inefficient way to remove a
+ /// node from a priority queue. We should roll our own heap to make this
+ /// better or something.
+ void RemoveFromPriorityQueue(SUnit *SU) {
+ std::vector<SUnit*> Temp;
+
+ assert(!Queue.empty() && "Not in queue!");
+ while (Queue.top() != SU) {
+ Temp.push_back(Queue.top());
+ Queue.pop();
+ assert(!Queue.empty() && "Not in queue!");
+ }
+
+ // Remove the node from the PQ.
+ Queue.pop();
+
+ // Add all the other nodes back.
+ for (unsigned i = 0, e = Temp.size(); i != e; ++i)
+ Queue.push(Temp[i]);
+ }
+ };
+}
+
+bool latency_sort::operator()(const SUnit *LHS, const SUnit *RHS) const {
+ unsigned LHSNum = LHS->NodeNum;
+ unsigned RHSNum = RHS->NodeNum;
+
+ // The most important heuristic is scheduling the critical path.
+ unsigned LHSLatency = PQ->getLatency(LHSNum);
+ unsigned RHSLatency = PQ->getLatency(RHSNum);
+ if (LHSLatency < RHSLatency) return true;
+ if (LHSLatency > RHSLatency) return false;
+
+ // After that, if two nodes have identical latencies, look to see if one will
+ // unblock more other nodes than the other.
+ unsigned LHSBlocked = PQ->getNumSolelyBlockNodes(LHSNum);
+ unsigned RHSBlocked = PQ->getNumSolelyBlockNodes(RHSNum);
+ if (LHSBlocked < RHSBlocked) return true;
+ if (LHSBlocked > RHSBlocked) return false;
+
+ // Finally, just to provide a stable ordering, use the node number as a
+ // deciding factor.
+ return LHSNum < RHSNum;
+}
+
+
+/// CalcNodePriority - Calculate the maximal path from the node to the exit.
+///
+int LatencyPriorityQueue::CalcLatency(const SUnit &SU) {
+ int &Latency = Latencies[SU.NodeNum];
+ if (Latency != -1)
+ return Latency;
+
+ int MaxSuccLatency = 0;
+ for (SUnit::const_succ_iterator I = SU.Succs.begin(), E = SU.Succs.end();
+ I != E; ++I)
+ MaxSuccLatency = std::max(MaxSuccLatency, CalcLatency(*I->first));
+
+ return Latency = MaxSuccLatency + SU.Latency;
+}
+
+/// CalculatePriorities - Calculate priorities of all scheduling units.
+void LatencyPriorityQueue::CalculatePriorities() {
+ Latencies.assign(SUnits->size(), -1);
+ NumNodesSolelyBlocking.assign(SUnits->size(), 0);
+
+ for (unsigned i = 0, e = SUnits->size(); i != e; ++i)
+ CalcLatency((*SUnits)[i]);
+}
+
+/// getSingleUnscheduledPred - If there is exactly one unscheduled predecessor
+/// of SU, return it, otherwise return null.
+SUnit *LatencyPriorityQueue::getSingleUnscheduledPred(SUnit *SU) {
+ SUnit *OnlyAvailablePred = 0;
+ for (SUnit::const_pred_iterator I = SU->Preds.begin(), E = SU->Preds.end();
+ I != E; ++I) {
+ SUnit &Pred = *I->first;
+ if (!Pred.isScheduled) {
+ // We found an available, but not scheduled, predecessor. If it's the
+ // only one we have found, keep track of it... otherwise give up.
+ if (OnlyAvailablePred && OnlyAvailablePred != &Pred)
+ return 0;
+ OnlyAvailablePred = &Pred;
+ }
+ }
+
+ return OnlyAvailablePred;
+}
+
+void LatencyPriorityQueue::push_impl(SUnit *SU) {
+ // Look at all of the successors of this node. Count the number of nodes that
+ // this node is the sole unscheduled node for.
+ unsigned NumNodesBlocking = 0;
+ for (SUnit::const_succ_iterator I = SU->Succs.begin(), E = SU->Succs.end();
+ I != E; ++I)
+ if (getSingleUnscheduledPred(I->first) == SU)
+ ++NumNodesBlocking;
+ NumNodesSolelyBlocking[SU->NodeNum] = NumNodesBlocking;
+
+ Queue.push(SU);
+}
+
+
+// ScheduledNode - As nodes are scheduled, we look to see if there are any
+// successor nodes that have a single unscheduled predecessor. If so, that
+// single predecessor has a higher priority, since scheduling it will make
+// the node available.
+void LatencyPriorityQueue::ScheduledNode(SUnit *SU) {
+ for (SUnit::const_succ_iterator I = SU->Succs.begin(), E = SU->Succs.end();
+ I != E; ++I)
+ AdjustPriorityOfUnscheduledPreds(I->first);
+}
+
+/// AdjustPriorityOfUnscheduledPreds - One of the predecessors of SU was just
+/// scheduled. If SU is not itself available, then there is at least one
+/// predecessor node that has not been scheduled yet. If SU has exactly ONE
+/// unscheduled predecessor, we want to increase its priority: it getting
+/// scheduled will make this node available, so it is better than some other
+/// node of the same priority that will not make a node available.
+void LatencyPriorityQueue::AdjustPriorityOfUnscheduledPreds(SUnit *SU) {
+ if (SU->isPending) return; // All preds scheduled.
+
+ SUnit *OnlyAvailablePred = getSingleUnscheduledPred(SU);
+ if (OnlyAvailablePred == 0 || !OnlyAvailablePred->isAvailable) return;
+
+ // Okay, we found a single predecessor that is available, but not scheduled.
+ // Since it is available, it must be in the priority queue. First remove it.
+ RemoveFromPriorityQueue(OnlyAvailablePred);
+
+ // Reinsert the node into the priority queue, which recomputes its
+ // NumNodesSolelyBlocking value.
+ push(OnlyAvailablePred);
+}
+
+
+//===----------------------------------------------------------------------===//
+// Public Constructor Functions
+//===----------------------------------------------------------------------===//
+
+/// createTDListDAGScheduler - This creates a top-down list scheduler with a
+/// new hazard recognizer. This scheduler takes ownership of the hazard
+/// recognizer and deletes it when done.
+ScheduleDAG* llvm::createTDListDAGScheduler(SelectionDAGISel *IS,
+ SelectionDAG *DAG,
+ MachineBasicBlock *BB) {
+ return new ScheduleDAGList(*DAG, BB, DAG->getTarget(),
+ new LatencyPriorityQueue(),
+ IS->CreateTargetHazardRecognizer());
+}
diff --git a/lib/CodeGen/SelectionDAG/ScheduleDAGRRList.cpp b/lib/CodeGen/SelectionDAG/ScheduleDAGRRList.cpp
new file mode 100644
index 0000000..f95be7d
--- /dev/null
+++ b/lib/CodeGen/SelectionDAG/ScheduleDAGRRList.cpp
@@ -0,0 +1,944 @@
+//===----- ScheduleDAGList.cpp - Reg pressure reduction list scheduler ----===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file was developed by Evan Cheng and is distributed under the
+// University of Illinois Open Source License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This implements bottom-up and top-down register pressure reduction list
+// schedulers, using standard algorithms. The basic approach uses a priority
+// queue of available nodes to schedule. One at a time, nodes are taken from
+// the priority queue (thus in priority order), checked for legality to
+// schedule, and emitted if legal.
+//
+//===----------------------------------------------------------------------===//
+
+#define DEBUG_TYPE "pre-RA-sched"
+#include "llvm/CodeGen/ScheduleDAG.h"
+#include "llvm/CodeGen/SchedulerRegistry.h"
+#include "llvm/CodeGen/SSARegMap.h"
+#include "llvm/Target/MRegisterInfo.h"
+#include "llvm/Target/TargetData.h"
+#include "llvm/Target/TargetMachine.h"
+#include "llvm/Target/TargetInstrInfo.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/Compiler.h"
+#include "llvm/ADT/Statistic.h"
+#include <climits>
+#include <queue>
+#include "llvm/Support/CommandLine.h"
+using namespace llvm;
+
+static RegisterScheduler
+ burrListDAGScheduler("list-burr",
+ " Bottom-up register reduction list scheduling",
+ createBURRListDAGScheduler);
+static RegisterScheduler
+ tdrListrDAGScheduler("list-tdrr",
+ " Top-down register reduction list scheduling",
+ createTDRRListDAGScheduler);
+
+namespace {
+//===----------------------------------------------------------------------===//
+/// ScheduleDAGRRList - The actual register reduction list scheduler
+/// implementation. This supports both top-down and bottom-up scheduling.
+///
+
+class VISIBILITY_HIDDEN ScheduleDAGRRList : public ScheduleDAG {
+private:
+ /// isBottomUp - This is true if the scheduling problem is bottom-up, false if
+ /// it is top-down.
+ bool isBottomUp;
+
+ /// AvailableQueue - The priority queue to use for the available SUnits.
+ ///
+ SchedulingPriorityQueue *AvailableQueue;
+
+public:
+ ScheduleDAGRRList(SelectionDAG &dag, MachineBasicBlock *bb,
+ const TargetMachine &tm, bool isbottomup,
+ SchedulingPriorityQueue *availqueue)
+ : ScheduleDAG(dag, bb, tm), isBottomUp(isbottomup),
+ AvailableQueue(availqueue) {
+ }
+
+ ~ScheduleDAGRRList() {
+ delete AvailableQueue;
+ }
+
+ void Schedule();
+
+private:
+ void ReleasePred(SUnit *PredSU, bool isChain, unsigned CurCycle);
+ void ReleaseSucc(SUnit *SuccSU, bool isChain, unsigned CurCycle);
+ void ScheduleNodeBottomUp(SUnit *SU, unsigned CurCycle);
+ void ScheduleNodeTopDown(SUnit *SU, unsigned CurCycle);
+ void ListScheduleTopDown();
+ void ListScheduleBottomUp();
+ void CommuteNodesToReducePressure();
+};
+} // end anonymous namespace
+
+
+/// Schedule - Schedule the DAG using list scheduling.
+void ScheduleDAGRRList::Schedule() {
+ DOUT << "********** List Scheduling **********\n";
+
+ // Build scheduling units.
+ BuildSchedUnits();
+
+ DEBUG(for (unsigned su = 0, e = SUnits.size(); su != e; ++su)
+ SUnits[su].dumpAll(&DAG));
+ CalculateDepths();
+ CalculateHeights();
+
+ AvailableQueue->initNodes(SUnitMap, SUnits);
+
+ // Execute the actual scheduling loop Top-Down or Bottom-Up as appropriate.
+ if (isBottomUp)
+ ListScheduleBottomUp();
+ else
+ ListScheduleTopDown();
+
+ AvailableQueue->releaseState();
+
+ CommuteNodesToReducePressure();
+
+ DOUT << "*** Final schedule ***\n";
+ DEBUG(dumpSchedule());
+ DOUT << "\n";
+
+ // Emit in scheduled order
+ EmitSchedule();
+}
+
+/// CommuteNodesToReducePressure - If a node is two-address and commutable, and
+/// it is not the last use of its first operand, add it to the CommuteSet if
+/// possible. It will be commuted when it is translated to a MI.
+void ScheduleDAGRRList::CommuteNodesToReducePressure() {
+ SmallPtrSet<SUnit*, 4> OperandSeen;
+ for (unsigned i = Sequence.size()-1; i != 0; --i) { // Ignore first node.
+ SUnit *SU = Sequence[i];
+ if (!SU) continue;
+ if (SU->isCommutable) {
+ unsigned Opc = SU->Node->getTargetOpcode();
+ unsigned NumRes = CountResults(SU->Node);
+ unsigned NumOps = CountOperands(SU->Node);
+ for (unsigned j = 0; j != NumOps; ++j) {
+ if (TII->getOperandConstraint(Opc, j+NumRes, TOI::TIED_TO) == -1)
+ continue;
+
+ SDNode *OpN = SU->Node->getOperand(j).Val;
+ SUnit *OpSU = SUnitMap[OpN];
+ if (OpSU && OperandSeen.count(OpSU) == 1) {
+ // Ok, so SU is not the last use of OpSU, but SU is two-address so
+ // it will clobber OpSU. Try to commute SU if no other source operands
+ // are live below.
+ bool DoCommute = true;
+ for (unsigned k = 0; k < NumOps; ++k) {
+ if (k != j) {
+ OpN = SU->Node->getOperand(k).Val;
+ OpSU = SUnitMap[OpN];
+ if (OpSU && OperandSeen.count(OpSU) == 1) {
+ DoCommute = false;
+ break;
+ }
+ }
+ }
+ if (DoCommute)
+ CommuteSet.insert(SU->Node);
+ }
+
+ // Only look at the first use&def node for now.
+ break;
+ }
+ }
+
+ for (SUnit::pred_iterator I = SU->Preds.begin(), E = SU->Preds.end();
+ I != E; ++I) {
+ if (!I->second)
+ OperandSeen.insert(I->first);
+ }
+ }
+}
+
+//===----------------------------------------------------------------------===//
+// Bottom-Up Scheduling
+//===----------------------------------------------------------------------===//
+
+/// ReleasePred - Decrement the NumSuccsLeft count of a predecessor. Add it to
+/// the Available queue is the count reaches zero. Also update its cycle bound.
+void ScheduleDAGRRList::ReleasePred(SUnit *PredSU, bool isChain,
+ unsigned CurCycle) {
+ // FIXME: the distance between two nodes is not always == the predecessor's
+ // latency. For example, the reader can very well read the register written
+ // by the predecessor later than the issue cycle. It also depends on the
+ // interrupt model (drain vs. freeze).
+ PredSU->CycleBound = std::max(PredSU->CycleBound, CurCycle + PredSU->Latency);
+
+ if (!isChain)
+ PredSU->NumSuccsLeft--;
+ else
+ PredSU->NumChainSuccsLeft--;
+
+#ifndef NDEBUG
+ if (PredSU->NumSuccsLeft < 0 || PredSU->NumChainSuccsLeft < 0) {
+ cerr << "*** List scheduling failed! ***\n";
+ PredSU->dump(&DAG);
+ cerr << " has been released too many times!\n";
+ assert(0);
+ }
+#endif
+
+ if ((PredSU->NumSuccsLeft + PredSU->NumChainSuccsLeft) == 0) {
+ // EntryToken has to go last! Special case it here.
+ if (PredSU->Node->getOpcode() != ISD::EntryToken) {
+ PredSU->isAvailable = true;
+ AvailableQueue->push(PredSU);
+ }
+ }
+}
+
+/// ScheduleNodeBottomUp - Add the node to the schedule. Decrement the pending
+/// count of its predecessors. If a predecessor pending count is zero, add it to
+/// the Available queue.
+void ScheduleDAGRRList::ScheduleNodeBottomUp(SUnit *SU, unsigned CurCycle) {
+ DOUT << "*** Scheduling [" << CurCycle << "]: ";
+ DEBUG(SU->dump(&DAG));
+ SU->Cycle = CurCycle;
+
+ AvailableQueue->ScheduledNode(SU);
+ Sequence.push_back(SU);
+
+ // Bottom up: release predecessors
+ for (SUnit::pred_iterator I = SU->Preds.begin(), E = SU->Preds.end();
+ I != E; ++I)
+ ReleasePred(I->first, I->second, CurCycle);
+ SU->isScheduled = true;
+}
+
+/// isReady - True if node's lower cycle bound is less or equal to the current
+/// scheduling cycle. Always true if all nodes have uniform latency 1.
+static inline bool isReady(SUnit *SU, unsigned CurCycle) {
+ return SU->CycleBound <= CurCycle;
+}
+
+/// ListScheduleBottomUp - The main loop of list scheduling for bottom-up
+/// schedulers.
+void ScheduleDAGRRList::ListScheduleBottomUp() {
+ unsigned CurCycle = 0;
+ // Add root to Available queue.
+ AvailableQueue->push(SUnitMap[DAG.getRoot().Val]);
+
+ // While Available queue is not empty, grab the node with the highest
+ // priority. If it is not ready put it back. Schedule the node.
+ std::vector<SUnit*> NotReady;
+ while (!AvailableQueue->empty()) {
+ SUnit *CurNode = AvailableQueue->pop();
+ while (CurNode && !isReady(CurNode, CurCycle)) {
+ NotReady.push_back(CurNode);
+ CurNode = AvailableQueue->pop();
+ }
+
+ // Add the nodes that aren't ready back onto the available list.
+ AvailableQueue->push_all(NotReady);
+ NotReady.clear();
+
+ if (CurNode != NULL)
+ ScheduleNodeBottomUp(CurNode, CurCycle);
+ CurCycle++;
+ }
+
+ // Add entry node last
+ if (DAG.getEntryNode().Val != DAG.getRoot().Val) {
+ SUnit *Entry = SUnitMap[DAG.getEntryNode().Val];
+ Sequence.push_back(Entry);
+ }
+
+ // Reverse the order if it is bottom up.
+ std::reverse(Sequence.begin(), Sequence.end());
+
+
+#ifndef NDEBUG
+ // Verify that all SUnits were scheduled.
+ bool AnyNotSched = false;
+ for (unsigned i = 0, e = SUnits.size(); i != e; ++i) {
+ if (SUnits[i].NumSuccsLeft != 0 || SUnits[i].NumChainSuccsLeft != 0) {
+ if (!AnyNotSched)
+ cerr << "*** List scheduling failed! ***\n";
+ SUnits[i].dump(&DAG);
+ cerr << "has not been scheduled!\n";
+ AnyNotSched = true;
+ }
+ }
+ assert(!AnyNotSched);
+#endif
+}
+
+//===----------------------------------------------------------------------===//
+// Top-Down Scheduling
+//===----------------------------------------------------------------------===//
+
+/// ReleaseSucc - Decrement the NumPredsLeft count of a successor. Add it to
+/// the PendingQueue if the count reaches zero.
+void ScheduleDAGRRList::ReleaseSucc(SUnit *SuccSU, bool isChain,
+ unsigned CurCycle) {
+ // FIXME: the distance between two nodes is not always == the predecessor's
+ // latency. For example, the reader can very well read the register written
+ // by the predecessor later than the issue cycle. It also depends on the
+ // interrupt model (drain vs. freeze).
+ SuccSU->CycleBound = std::max(SuccSU->CycleBound, CurCycle + SuccSU->Latency);
+
+ if (!isChain)
+ SuccSU->NumPredsLeft--;
+ else
+ SuccSU->NumChainPredsLeft--;
+
+#ifndef NDEBUG
+ if (SuccSU->NumPredsLeft < 0 || SuccSU->NumChainPredsLeft < 0) {
+ cerr << "*** List scheduling failed! ***\n";
+ SuccSU->dump(&DAG);
+ cerr << " has been released too many times!\n";
+ assert(0);
+ }
+#endif
+
+ if ((SuccSU->NumPredsLeft + SuccSU->NumChainPredsLeft) == 0) {
+ SuccSU->isAvailable = true;
+ AvailableQueue->push(SuccSU);
+ }
+}
+
+
+/// ScheduleNodeTopDown - Add the node to the schedule. Decrement the pending
+/// count of its successors. If a successor pending count is zero, add it to
+/// the Available queue.
+void ScheduleDAGRRList::ScheduleNodeTopDown(SUnit *SU, unsigned CurCycle) {
+ DOUT << "*** Scheduling [" << CurCycle << "]: ";
+ DEBUG(SU->dump(&DAG));
+ SU->Cycle = CurCycle;
+
+ AvailableQueue->ScheduledNode(SU);
+ Sequence.push_back(SU);
+
+ // Top down: release successors
+ for (SUnit::succ_iterator I = SU->Succs.begin(), E = SU->Succs.end();
+ I != E; ++I)
+ ReleaseSucc(I->first, I->second, CurCycle);
+ SU->isScheduled = true;
+}
+
+void ScheduleDAGRRList::ListScheduleTopDown() {
+ unsigned CurCycle = 0;
+ SUnit *Entry = SUnitMap[DAG.getEntryNode().Val];
+
+ // All leaves to Available queue.
+ for (unsigned i = 0, e = SUnits.size(); i != e; ++i) {
+ // It is available if it has no predecessors.
+ if (SUnits[i].Preds.size() == 0 && &SUnits[i] != Entry) {
+ AvailableQueue->push(&SUnits[i]);
+ SUnits[i].isAvailable = true;
+ }
+ }
+
+ // Emit the entry node first.
+ ScheduleNodeTopDown(Entry, CurCycle);
+ CurCycle++;
+
+ // While Available queue is not empty, grab the node with the highest
+ // priority. If it is not ready put it back. Schedule the node.
+ std::vector<SUnit*> NotReady;
+ while (!AvailableQueue->empty()) {
+ SUnit *CurNode = AvailableQueue->pop();
+ while (CurNode && !isReady(CurNode, CurCycle)) {
+ NotReady.push_back(CurNode);
+ CurNode = AvailableQueue->pop();
+ }
+
+ // Add the nodes that aren't ready back onto the available list.
+ AvailableQueue->push_all(NotReady);
+ NotReady.clear();
+
+ if (CurNode != NULL)
+ ScheduleNodeTopDown(CurNode, CurCycle);
+ CurCycle++;
+ }
+
+
+#ifndef NDEBUG
+ // Verify that all SUnits were scheduled.
+ bool AnyNotSched = false;
+ for (unsigned i = 0, e = SUnits.size(); i != e; ++i) {
+ if (!SUnits[i].isScheduled) {
+ if (!AnyNotSched)
+ cerr << "*** List scheduling failed! ***\n";
+ SUnits[i].dump(&DAG);
+ cerr << "has not been scheduled!\n";
+ AnyNotSched = true;
+ }
+ }
+ assert(!AnyNotSched);
+#endif
+}
+
+
+
+//===----------------------------------------------------------------------===//
+// RegReductionPriorityQueue Implementation
+//===----------------------------------------------------------------------===//
+//
+// This is a SchedulingPriorityQueue that schedules using Sethi Ullman numbers
+// to reduce register pressure.
+//
+namespace {
+ template<class SF>
+ class RegReductionPriorityQueue;
+
+ /// Sorting functions for the Available queue.
+ struct bu_ls_rr_sort : public std::binary_function<SUnit*, SUnit*, bool> {
+ RegReductionPriorityQueue<bu_ls_rr_sort> *SPQ;
+ bu_ls_rr_sort(RegReductionPriorityQueue<bu_ls_rr_sort> *spq) : SPQ(spq) {}
+ bu_ls_rr_sort(const bu_ls_rr_sort &RHS) : SPQ(RHS.SPQ) {}
+
+ bool operator()(const SUnit* left, const SUnit* right) const;
+ };
+
+ struct td_ls_rr_sort : public std::binary_function<SUnit*, SUnit*, bool> {
+ RegReductionPriorityQueue<td_ls_rr_sort> *SPQ;
+ td_ls_rr_sort(RegReductionPriorityQueue<td_ls_rr_sort> *spq) : SPQ(spq) {}
+ td_ls_rr_sort(const td_ls_rr_sort &RHS) : SPQ(RHS.SPQ) {}
+
+ bool operator()(const SUnit* left, const SUnit* right) const;
+ };
+} // end anonymous namespace
+
+static inline bool isCopyFromLiveIn(const SUnit *SU) {
+ SDNode *N = SU->Node;
+ return N->getOpcode() == ISD::CopyFromReg &&
+ N->getOperand(N->getNumOperands()-1).getValueType() != MVT::Flag;
+}
+
+namespace {
+ template<class SF>
+ class VISIBILITY_HIDDEN RegReductionPriorityQueue
+ : public SchedulingPriorityQueue {
+ std::priority_queue<SUnit*, std::vector<SUnit*>, SF> Queue;
+
+ public:
+ RegReductionPriorityQueue() :
+ Queue(SF(this)) {}
+
+ virtual void initNodes(DenseMap<SDNode*, SUnit*> &sumap,
+ std::vector<SUnit> &sunits) {}
+ virtual void releaseState() {}
+
+ virtual unsigned getNodePriority(const SUnit *SU) const {
+ return 0;
+ }
+
+ bool empty() const { return Queue.empty(); }
+
+ void push(SUnit *U) {
+ Queue.push(U);
+ }
+ void push_all(const std::vector<SUnit *> &Nodes) {
+ for (unsigned i = 0, e = Nodes.size(); i != e; ++i)
+ Queue.push(Nodes[i]);
+ }
+
+ SUnit *pop() {
+ if (empty()) return NULL;
+ SUnit *V = Queue.top();
+ Queue.pop();
+ return V;
+ }
+
+ virtual bool isDUOperand(const SUnit *SU1, const SUnit *SU2) {
+ return false;
+ }
+ };
+
+ template<class SF>
+ class VISIBILITY_HIDDEN BURegReductionPriorityQueue
+ : public RegReductionPriorityQueue<SF> {
+ // SUnitMap SDNode to SUnit mapping (n -> 1).
+ DenseMap<SDNode*, SUnit*> *SUnitMap;
+
+ // SUnits - The SUnits for the current graph.
+ const std::vector<SUnit> *SUnits;
+
+ // SethiUllmanNumbers - The SethiUllman number for each node.
+ std::vector<unsigned> SethiUllmanNumbers;
+
+ const TargetInstrInfo *TII;
+ public:
+ BURegReductionPriorityQueue(const TargetInstrInfo *tii)
+ : TII(tii) {}
+
+ void initNodes(DenseMap<SDNode*, SUnit*> &sumap,
+ std::vector<SUnit> &sunits) {
+ SUnitMap = &sumap;
+ SUnits = &sunits;
+ // Add pseudo dependency edges for two-address nodes.
+ AddPseudoTwoAddrDeps();
+ // Calculate node priorities.
+ CalculateSethiUllmanNumbers();
+ }
+
+ void releaseState() {
+ SUnits = 0;
+ SethiUllmanNumbers.clear();
+ }
+
+ unsigned getNodePriority(const SUnit *SU) const {
+ assert(SU->NodeNum < SethiUllmanNumbers.size());
+ unsigned Opc = SU->Node->getOpcode();
+ if (Opc == ISD::CopyFromReg && !isCopyFromLiveIn(SU))
+ // CopyFromReg should be close to its def because it restricts
+ // allocation choices. But if it is a livein then perhaps we want it
+ // closer to its uses so it can be coalesced.
+ return 0xffff;
+ else if (Opc == ISD::TokenFactor || Opc == ISD::CopyToReg)
+ // CopyToReg should be close to its uses to facilitate coalescing and
+ // avoid spilling.
+ return 0;
+ else if (SU->NumSuccs == 0)
+ // If SU does not have a use, i.e. it doesn't produce a value that would
+ // be consumed (e.g. store), then it terminates a chain of computation.
+ // Give it a large SethiUllman number so it will be scheduled right
+ // before its predecessors that it doesn't lengthen their live ranges.
+ return 0xffff;
+ else if (SU->NumPreds == 0)
+ // If SU does not have a def, schedule it close to its uses because it
+ // does not lengthen any live ranges.
+ return 0;
+ else
+ return SethiUllmanNumbers[SU->NodeNum];
+ }
+
+ bool isDUOperand(const SUnit *SU1, const SUnit *SU2) {
+ unsigned Opc = SU1->Node->getTargetOpcode();
+ unsigned NumRes = ScheduleDAG::CountResults(SU1->Node);
+ unsigned NumOps = ScheduleDAG::CountOperands(SU1->Node);
+ for (unsigned i = 0; i != NumOps; ++i) {
+ if (TII->getOperandConstraint(Opc, i+NumRes, TOI::TIED_TO) == -1)
+ continue;
+ if (SU1->Node->getOperand(i).isOperand(SU2->Node))
+ return true;
+ }
+ return false;
+ }
+ private:
+ bool canClobber(SUnit *SU, SUnit *Op);
+ void AddPseudoTwoAddrDeps();
+ void CalculateSethiUllmanNumbers();
+ unsigned CalcNodeSethiUllmanNumber(const SUnit *SU);
+ };
+
+
+ template<class SF>
+ class TDRegReductionPriorityQueue : public RegReductionPriorityQueue<SF> {
+ // SUnitMap SDNode to SUnit mapping (n -> 1).
+ DenseMap<SDNode*, SUnit*> *SUnitMap;
+
+ // SUnits - The SUnits for the current graph.
+ const std::vector<SUnit> *SUnits;
+
+ // SethiUllmanNumbers - The SethiUllman number for each node.
+ std::vector<unsigned> SethiUllmanNumbers;
+
+ public:
+ TDRegReductionPriorityQueue() {}
+
+ void initNodes(DenseMap<SDNode*, SUnit*> &sumap,
+ std::vector<SUnit> &sunits) {
+ SUnitMap = &sumap;
+ SUnits = &sunits;
+ // Calculate node priorities.
+ CalculateSethiUllmanNumbers();
+ }
+
+ void releaseState() {
+ SUnits = 0;
+ SethiUllmanNumbers.clear();
+ }
+
+ unsigned getNodePriority(const SUnit *SU) const {
+ assert(SU->NodeNum < SethiUllmanNumbers.size());
+ return SethiUllmanNumbers[SU->NodeNum];
+ }
+
+ private:
+ void CalculateSethiUllmanNumbers();
+ unsigned CalcNodeSethiUllmanNumber(const SUnit *SU);
+ };
+}
+
+/// closestSucc - Returns the scheduled cycle of the successor which is
+/// closet to the current cycle.
+static unsigned closestSucc(const SUnit *SU) {
+ unsigned MaxCycle = 0;
+ for (SUnit::const_succ_iterator I = SU->Succs.begin(), E = SU->Succs.end();
+ I != E; ++I) {
+ unsigned Cycle = I->first->Cycle;
+ // If there are bunch of CopyToRegs stacked up, they should be considered
+ // to be at the same position.
+ if (I->first->Node->getOpcode() == ISD::CopyToReg)
+ Cycle = closestSucc(I->first)+1;
+ if (Cycle > MaxCycle)
+ MaxCycle = Cycle;
+ }
+ return MaxCycle;
+}
+
+/// calcMaxScratches - Returns an cost estimate of the worse case requirement
+/// for scratch registers. Live-in operands and live-out results don't count
+/// since they are "fixed".
+static unsigned calcMaxScratches(const SUnit *SU) {
+ unsigned Scratches = 0;
+ for (SUnit::const_pred_iterator I = SU->Preds.begin(), E = SU->Preds.end();
+ I != E; ++I) {
+ if (I->second) continue; // ignore chain preds
+ if (I->first->Node->getOpcode() != ISD::CopyFromReg)
+ Scratches++;
+ }
+ for (SUnit::const_succ_iterator I = SU->Succs.begin(), E = SU->Succs.end();
+ I != E; ++I) {
+ if (I->second) continue; // ignore chain succs
+ if (I->first->Node->getOpcode() != ISD::CopyToReg)
+ Scratches += 10;
+ }
+ return Scratches;
+}
+
+// Bottom up
+bool bu_ls_rr_sort::operator()(const SUnit *left, const SUnit *right) const {
+ // There used to be a special tie breaker here that looked for
+ // two-address instructions and preferred the instruction with a
+ // def&use operand. The special case triggered diagnostics when
+ // _GLIBCXX_DEBUG was enabled because it broke the strict weak
+ // ordering that priority_queue requires. It didn't help much anyway
+ // because AddPseudoTwoAddrDeps already covers many of the cases
+ // where it would have applied. In addition, it's counter-intuitive
+ // that a tie breaker would be the first thing attempted. There's a
+ // "real" tie breaker below that is the operation of last resort.
+ // The fact that the "special tie breaker" would trigger when there
+ // wasn't otherwise a tie is what broke the strict weak ordering
+ // constraint.
+
+ unsigned LPriority = SPQ->getNodePriority(left);
+ unsigned RPriority = SPQ->getNodePriority(right);
+ if (LPriority > RPriority)
+ return true;
+ else if (LPriority == RPriority) {
+ // Try schedule def + use closer when Sethi-Ullman numbers are the same.
+ // e.g.
+ // t1 = op t2, c1
+ // t3 = op t4, c2
+ //
+ // and the following instructions are both ready.
+ // t2 = op c3
+ // t4 = op c4
+ //
+ // Then schedule t2 = op first.
+ // i.e.
+ // t4 = op c4
+ // t2 = op c3
+ // t1 = op t2, c1
+ // t3 = op t4, c2
+ //
+ // This creates more short live intervals.
+ unsigned LDist = closestSucc(left);
+ unsigned RDist = closestSucc(right);
+ if (LDist < RDist)
+ return true;
+ else if (LDist == RDist) {
+ // Intuitively, it's good to push down instructions whose results are
+ // liveout so their long live ranges won't conflict with other values
+ // which are needed inside the BB. Further prioritize liveout instructions
+ // by the number of operands which are calculated within the BB.
+ unsigned LScratch = calcMaxScratches(left);
+ unsigned RScratch = calcMaxScratches(right);
+ if (LScratch > RScratch)
+ return true;
+ else if (LScratch == RScratch)
+ if (left->Height > right->Height)
+ return true;
+ else if (left->Height == right->Height)
+ if (left->Depth < right->Depth)
+ return true;
+ else if (left->Depth == right->Depth)
+ if (left->CycleBound > right->CycleBound)
+ return true;
+ }
+ }
+ return false;
+}
+
+// FIXME: This is probably too slow!
+static void isReachable(SUnit *SU, SUnit *TargetSU,
+ SmallPtrSet<SUnit*, 32> &Visited, bool &Reached) {
+ if (Reached) return;
+ if (SU == TargetSU) {
+ Reached = true;
+ return;
+ }
+ if (!Visited.insert(SU)) return;
+
+ for (SUnit::pred_iterator I = SU->Preds.begin(), E = SU->Preds.end(); I != E;
+ ++I)
+ isReachable(I->first, TargetSU, Visited, Reached);
+}
+
+static bool isReachable(SUnit *SU, SUnit *TargetSU) {
+ SmallPtrSet<SUnit*, 32> Visited;
+ bool Reached = false;
+ isReachable(SU, TargetSU, Visited, Reached);
+ return Reached;
+}
+
+template<class SF>
+bool BURegReductionPriorityQueue<SF>::canClobber(SUnit *SU, SUnit *Op) {
+ if (SU->isTwoAddress) {
+ unsigned Opc = SU->Node->getTargetOpcode();
+ unsigned NumRes = ScheduleDAG::CountResults(SU->Node);
+ unsigned NumOps = ScheduleDAG::CountOperands(SU->Node);
+ for (unsigned i = 0; i != NumOps; ++i) {
+ if (TII->getOperandConstraint(Opc, i+NumRes, TOI::TIED_TO) != -1) {
+ SDNode *DU = SU->Node->getOperand(i).Val;
+ if (Op == (*SUnitMap)[DU])
+ return true;
+ }
+ }
+ }
+ return false;
+}
+
+
+/// AddPseudoTwoAddrDeps - If two nodes share an operand and one of them uses
+/// it as a def&use operand. Add a pseudo control edge from it to the other
+/// node (if it won't create a cycle) so the two-address one will be scheduled
+/// first (lower in the schedule).
+template<class SF>
+void BURegReductionPriorityQueue<SF>::AddPseudoTwoAddrDeps() {
+ for (unsigned i = 0, e = SUnits->size(); i != e; ++i) {
+ SUnit *SU = (SUnit *)&((*SUnits)[i]);
+ if (!SU->isTwoAddress)
+ continue;
+
+ SDNode *Node = SU->Node;
+ if (!Node->isTargetOpcode())
+ continue;
+
+ unsigned Opc = Node->getTargetOpcode();
+ unsigned NumRes = ScheduleDAG::CountResults(Node);
+ unsigned NumOps = ScheduleDAG::CountOperands(Node);
+ for (unsigned j = 0; j != NumOps; ++j) {
+ if (TII->getOperandConstraint(Opc, j+NumRes, TOI::TIED_TO) != -1) {
+ SDNode *DU = SU->Node->getOperand(j).Val;
+ SUnit *DUSU = (*SUnitMap)[DU];
+ if (!DUSU) continue;
+ for (SUnit::succ_iterator I = DUSU->Succs.begin(),E = DUSU->Succs.end();
+ I != E; ++I) {
+ if (I->second) continue;
+ SUnit *SuccSU = I->first;
+ if (SuccSU != SU &&
+ (!canClobber(SuccSU, DUSU) ||
+ (!SU->isCommutable && SuccSU->isCommutable))){
+ if (SuccSU->Depth == SU->Depth && !isReachable(SuccSU, SU)) {
+ DOUT << "Adding an edge from SU # " << SU->NodeNum
+ << " to SU #" << SuccSU->NodeNum << "\n";
+ if (SU->addPred(SuccSU, true))
+ SU->NumChainPredsLeft++;
+ if (SuccSU->addSucc(SU, true))
+ SuccSU->NumChainSuccsLeft++;
+ }
+ }
+ }
+ }
+ }
+ }
+}
+
+/// CalcNodeSethiUllmanNumber - Priority is the Sethi Ullman number.
+/// Smaller number is the higher priority.
+template<class SF>
+unsigned BURegReductionPriorityQueue<SF>::
+CalcNodeSethiUllmanNumber(const SUnit *SU) {
+ unsigned &SethiUllmanNumber = SethiUllmanNumbers[SU->NodeNum];
+ if (SethiUllmanNumber != 0)
+ return SethiUllmanNumber;
+
+ unsigned Extra = 0;
+ for (SUnit::const_pred_iterator I = SU->Preds.begin(), E = SU->Preds.end();
+ I != E; ++I) {
+ if (I->second) continue; // ignore chain preds
+ SUnit *PredSU = I->first;
+ unsigned PredSethiUllman = CalcNodeSethiUllmanNumber(PredSU);
+ if (PredSethiUllman > SethiUllmanNumber) {
+ SethiUllmanNumber = PredSethiUllman;
+ Extra = 0;
+ } else if (PredSethiUllman == SethiUllmanNumber && !I->second)
+ Extra++;
+ }
+
+ SethiUllmanNumber += Extra;
+
+ if (SethiUllmanNumber == 0)
+ SethiUllmanNumber = 1;
+
+ return SethiUllmanNumber;
+}
+
+/// CalculateSethiUllmanNumbers - Calculate Sethi-Ullman numbers of all
+/// scheduling units.
+template<class SF>
+void BURegReductionPriorityQueue<SF>::CalculateSethiUllmanNumbers() {
+ SethiUllmanNumbers.assign(SUnits->size(), 0);
+
+ for (unsigned i = 0, e = SUnits->size(); i != e; ++i)
+ CalcNodeSethiUllmanNumber(&(*SUnits)[i]);
+}
+
+static unsigned SumOfUnscheduledPredsOfSuccs(const SUnit *SU) {
+ unsigned Sum = 0;
+ for (SUnit::const_succ_iterator I = SU->Succs.begin(), E = SU->Succs.end();
+ I != E; ++I) {
+ SUnit *SuccSU = I->first;
+ for (SUnit::const_pred_iterator II = SuccSU->Preds.begin(),
+ EE = SuccSU->Preds.end(); II != EE; ++II) {
+ SUnit *PredSU = II->first;
+ if (!PredSU->isScheduled)
+ Sum++;
+ }
+ }
+
+ return Sum;
+}
+
+
+// Top down
+bool td_ls_rr_sort::operator()(const SUnit *left, const SUnit *right) const {
+ unsigned LPriority = SPQ->getNodePriority(left);
+ unsigned RPriority = SPQ->getNodePriority(right);
+ bool LIsTarget = left->Node->isTargetOpcode();
+ bool RIsTarget = right->Node->isTargetOpcode();
+ bool LIsFloater = LIsTarget && left->NumPreds == 0;
+ bool RIsFloater = RIsTarget && right->NumPreds == 0;
+ unsigned LBonus = (SumOfUnscheduledPredsOfSuccs(left) == 1) ? 2 : 0;
+ unsigned RBonus = (SumOfUnscheduledPredsOfSuccs(right) == 1) ? 2 : 0;
+
+ if (left->NumSuccs == 0 && right->NumSuccs != 0)
+ return false;
+ else if (left->NumSuccs != 0 && right->NumSuccs == 0)
+ return true;
+
+ // Special tie breaker: if two nodes share a operand, the one that use it
+ // as a def&use operand is preferred.
+ if (LIsTarget && RIsTarget) {
+ if (left->isTwoAddress && !right->isTwoAddress) {
+ SDNode *DUNode = left->Node->getOperand(0).Val;
+ if (DUNode->isOperand(right->Node))
+ RBonus += 2;
+ }
+ if (!left->isTwoAddress && right->isTwoAddress) {
+ SDNode *DUNode = right->Node->getOperand(0).Val;
+ if (DUNode->isOperand(left->Node))
+ LBonus += 2;
+ }
+ }
+ if (LIsFloater)
+ LBonus -= 2;
+ if (RIsFloater)
+ RBonus -= 2;
+ if (left->NumSuccs == 1)
+ LBonus += 2;
+ if (right->NumSuccs == 1)
+ RBonus += 2;
+
+ if (LPriority+LBonus < RPriority+RBonus)
+ return true;
+ else if (LPriority == RPriority)
+ if (left->Depth < right->Depth)
+ return true;
+ else if (left->Depth == right->Depth)
+ if (left->NumSuccsLeft > right->NumSuccsLeft)
+ return true;
+ else if (left->NumSuccsLeft == right->NumSuccsLeft)
+ if (left->CycleBound > right->CycleBound)
+ return true;
+ return false;
+}
+
+/// CalcNodeSethiUllmanNumber - Priority is the Sethi Ullman number.
+/// Smaller number is the higher priority.
+template<class SF>
+unsigned TDRegReductionPriorityQueue<SF>::
+CalcNodeSethiUllmanNumber(const SUnit *SU) {
+ unsigned &SethiUllmanNumber = SethiUllmanNumbers[SU->NodeNum];
+ if (SethiUllmanNumber != 0)
+ return SethiUllmanNumber;
+
+ unsigned Opc = SU->Node->getOpcode();
+ if (Opc == ISD::TokenFactor || Opc == ISD::CopyToReg)
+ SethiUllmanNumber = 0xffff;
+ else if (SU->NumSuccsLeft == 0)
+ // If SU does not have a use, i.e. it doesn't produce a value that would
+ // be consumed (e.g. store), then it terminates a chain of computation.
+ // Give it a small SethiUllman number so it will be scheduled right before
+ // its predecessors that it doesn't lengthen their live ranges.
+ SethiUllmanNumber = 0;
+ else if (SU->NumPredsLeft == 0 &&
+ (Opc != ISD::CopyFromReg || isCopyFromLiveIn(SU)))
+ SethiUllmanNumber = 0xffff;
+ else {
+ int Extra = 0;
+ for (SUnit::const_pred_iterator I = SU->Preds.begin(), E = SU->Preds.end();
+ I != E; ++I) {
+ if (I->second) continue; // ignore chain preds
+ SUnit *PredSU = I->first;
+ unsigned PredSethiUllman = CalcNodeSethiUllmanNumber(PredSU);
+ if (PredSethiUllman > SethiUllmanNumber) {
+ SethiUllmanNumber = PredSethiUllman;
+ Extra = 0;
+ } else if (PredSethiUllman == SethiUllmanNumber && !I->second)
+ Extra++;
+ }
+
+ SethiUllmanNumber += Extra;
+ }
+
+ return SethiUllmanNumber;
+}
+
+/// CalculateSethiUllmanNumbers - Calculate Sethi-Ullman numbers of all
+/// scheduling units.
+template<class SF>
+void TDRegReductionPriorityQueue<SF>::CalculateSethiUllmanNumbers() {
+ SethiUllmanNumbers.assign(SUnits->size(), 0);
+
+ for (unsigned i = 0, e = SUnits->size(); i != e; ++i)
+ CalcNodeSethiUllmanNumber(&(*SUnits)[i]);
+}
+
+//===----------------------------------------------------------------------===//
+// Public Constructor Functions
+//===----------------------------------------------------------------------===//
+
+llvm::ScheduleDAG* llvm::createBURRListDAGScheduler(SelectionDAGISel *IS,
+ SelectionDAG *DAG,
+ MachineBasicBlock *BB) {
+ const TargetInstrInfo *TII = DAG->getTarget().getInstrInfo();
+ return new ScheduleDAGRRList(*DAG, BB, DAG->getTarget(), true,
+ new BURegReductionPriorityQueue<bu_ls_rr_sort>(TII));
+}
+
+llvm::ScheduleDAG* llvm::createTDRRListDAGScheduler(SelectionDAGISel *IS,
+ SelectionDAG *DAG,
+ MachineBasicBlock *BB) {
+ return new ScheduleDAGRRList(*DAG, BB, DAG->getTarget(), false,
+ new TDRegReductionPriorityQueue<td_ls_rr_sort>());
+}
+
diff --git a/lib/CodeGen/SelectionDAG/ScheduleDAGSimple.cpp b/lib/CodeGen/SelectionDAG/ScheduleDAGSimple.cpp
new file mode 100644
index 0000000..62854f7
--- /dev/null
+++ b/lib/CodeGen/SelectionDAG/ScheduleDAGSimple.cpp
@@ -0,0 +1,1156 @@
+//===-- ScheduleDAGSimple.cpp - Implement a trivial DAG scheduler ---------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file was developed by James M. Laskey and is distributed under the
+// University of Illinois Open Source License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This implements a simple two pass scheduler. The first pass attempts to push
+// backward any lengthy instructions and critical paths. The second pass packs
+// instructions into semi-optimal time slots.
+//
+//===----------------------------------------------------------------------===//
+
+#define DEBUG_TYPE "pre-RA-sched"
+#include "llvm/CodeGen/MachineFunction.h"
+#include "llvm/CodeGen/ScheduleDAG.h"
+#include "llvm/CodeGen/SchedulerRegistry.h"
+#include "llvm/CodeGen/SelectionDAG.h"
+#include "llvm/CodeGen/SSARegMap.h"
+#include "llvm/Target/TargetData.h"
+#include "llvm/Target/TargetMachine.h"
+#include "llvm/Target/TargetInstrInfo.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/Compiler.h"
+#include <algorithm>
+using namespace llvm;
+
+namespace {
+
+static RegisterScheduler
+ bfsDAGScheduler("none", " No scheduling: breadth first sequencing",
+ createBFS_DAGScheduler);
+static RegisterScheduler
+ simpleDAGScheduler("simple",
+ " Simple two pass scheduling: minimize critical path "
+ "and maximize processor utilization",
+ createSimpleDAGScheduler);
+static RegisterScheduler
+ noitinDAGScheduler("simple-noitin",
+ " Simple two pass scheduling: Same as simple "
+ "except using generic latency",
+ createNoItinsDAGScheduler);
+
+class NodeInfo;
+typedef NodeInfo *NodeInfoPtr;
+typedef std::vector<NodeInfoPtr> NIVector;
+typedef std::vector<NodeInfoPtr>::iterator NIIterator;
+
+//===--------------------------------------------------------------------===//
+///
+/// Node group - This struct is used to manage flagged node groups.
+///
+class NodeGroup {
+public:
+ NodeGroup *Next;
+private:
+ NIVector Members; // Group member nodes
+ NodeInfo *Dominator; // Node with highest latency
+ unsigned Latency; // Total latency of the group
+ int Pending; // Number of visits pending before
+ // adding to order
+
+public:
+ // Ctor.
+ NodeGroup() : Next(NULL), Dominator(NULL), Pending(0) {}
+
+ // Accessors
+ inline void setDominator(NodeInfo *D) { Dominator = D; }
+ inline NodeInfo *getTop() { return Members.front(); }
+ inline NodeInfo *getBottom() { return Members.back(); }
+ inline NodeInfo *getDominator() { return Dominator; }
+ inline void setLatency(unsigned L) { Latency = L; }
+ inline unsigned getLatency() { return Latency; }
+ inline int getPending() const { return Pending; }
+ inline void setPending(int P) { Pending = P; }
+ inline int addPending(int I) { return Pending += I; }
+
+ // Pass thru
+ inline bool group_empty() { return Members.empty(); }
+ inline NIIterator group_begin() { return Members.begin(); }
+ inline NIIterator group_end() { return Members.end(); }
+ inline void group_push_back(const NodeInfoPtr &NI) {
+ Members.push_back(NI);
+ }
+ inline NIIterator group_insert(NIIterator Pos, const NodeInfoPtr &NI) {
+ return Members.insert(Pos, NI);
+ }
+ inline void group_insert(NIIterator Pos, NIIterator First,
+ NIIterator Last) {
+ Members.insert(Pos, First, Last);
+ }
+
+ static void Add(NodeInfo *D, NodeInfo *U);
+};
+
+//===--------------------------------------------------------------------===//
+///
+/// NodeInfo - This struct tracks information used to schedule the a node.
+///
+class NodeInfo {
+private:
+ int Pending; // Number of visits pending before
+ // adding to order
+public:
+ SDNode *Node; // DAG node
+ InstrStage *StageBegin; // First stage in itinerary
+ InstrStage *StageEnd; // Last+1 stage in itinerary
+ unsigned Latency; // Total cycles to complete instr
+ bool IsCall : 1; // Is function call
+ bool IsLoad : 1; // Is memory load
+ bool IsStore : 1; // Is memory store
+ unsigned Slot; // Node's time slot
+ NodeGroup *Group; // Grouping information
+#ifndef NDEBUG
+ unsigned Preorder; // Index before scheduling
+#endif
+
+ // Ctor.
+ NodeInfo(SDNode *N = NULL)
+ : Pending(0)
+ , Node(N)
+ , StageBegin(NULL)
+ , StageEnd(NULL)
+ , Latency(0)
+ , IsCall(false)
+ , Slot(0)
+ , Group(NULL)
+#ifndef NDEBUG
+ , Preorder(0)
+#endif
+ {}
+
+ // Accessors
+ inline bool isInGroup() const {
+ assert(!Group || !Group->group_empty() && "Group with no members");
+ return Group != NULL;
+ }
+ inline bool isGroupDominator() const {
+ return isInGroup() && Group->getDominator() == this;
+ }
+ inline int getPending() const {
+ return Group ? Group->getPending() : Pending;
+ }
+ inline void setPending(int P) {
+ if (Group) Group->setPending(P);
+ else Pending = P;
+ }
+ inline int addPending(int I) {
+ if (Group) return Group->addPending(I);
+ else return Pending += I;
+ }
+};
+
+//===--------------------------------------------------------------------===//
+///
+/// NodeGroupIterator - Iterates over all the nodes indicated by the node
+/// info. If the node is in a group then iterate over the members of the
+/// group, otherwise just the node info.
+///
+class NodeGroupIterator {
+private:
+ NodeInfo *NI; // Node info
+ NIIterator NGI; // Node group iterator
+ NIIterator NGE; // Node group iterator end
+
+public:
+ // Ctor.
+ NodeGroupIterator(NodeInfo *N) : NI(N) {
+ // If the node is in a group then set up the group iterator. Otherwise
+ // the group iterators will trip first time out.
+ if (N->isInGroup()) {
+ // get Group
+ NodeGroup *Group = NI->Group;
+ NGI = Group->group_begin();
+ NGE = Group->group_end();
+ // Prevent this node from being used (will be in members list
+ NI = NULL;
+ }
+ }
+
+ /// next - Return the next node info, otherwise NULL.
+ ///
+ NodeInfo *next() {
+ // If members list
+ if (NGI != NGE) return *NGI++;
+ // Use node as the result (may be NULL)
+ NodeInfo *Result = NI;
+ // Only use once
+ NI = NULL;
+ // Return node or NULL
+ return Result;
+ }
+};
+//===--------------------------------------------------------------------===//
+
+
+//===--------------------------------------------------------------------===//
+///
+/// NodeGroupOpIterator - Iterates over all the operands of a node. If the
+/// node is a member of a group, this iterates over all the operands of all
+/// the members of the group.
+///
+class NodeGroupOpIterator {
+private:
+ NodeInfo *NI; // Node containing operands
+ NodeGroupIterator GI; // Node group iterator
+ SDNode::op_iterator OI; // Operand iterator
+ SDNode::op_iterator OE; // Operand iterator end
+
+ /// CheckNode - Test if node has more operands. If not get the next node
+ /// skipping over nodes that have no operands.
+ void CheckNode() {
+ // Only if operands are exhausted first
+ while (OI == OE) {
+ // Get next node info
+ NodeInfo *NI = GI.next();
+ // Exit if nodes are exhausted
+ if (!NI) return;
+ // Get node itself
+ SDNode *Node = NI->Node;
+ // Set up the operand iterators
+ OI = Node->op_begin();
+ OE = Node->op_end();
+ }
+ }
+
+public:
+ // Ctor.
+ NodeGroupOpIterator(NodeInfo *N)
+ : NI(N), GI(N), OI(SDNode::op_iterator()), OE(SDNode::op_iterator()) {}
+
+ /// isEnd - Returns true when not more operands are available.
+ ///
+ inline bool isEnd() { CheckNode(); return OI == OE; }
+
+ /// next - Returns the next available operand.
+ ///
+ inline SDOperand next() {
+ assert(OI != OE &&
+ "Not checking for end of NodeGroupOpIterator correctly");
+ return *OI++;
+ }
+};
+
+
+//===----------------------------------------------------------------------===//
+///
+/// BitsIterator - Provides iteration through individual bits in a bit vector.
+///
+template<class T>
+class BitsIterator {
+private:
+ T Bits; // Bits left to iterate through
+
+public:
+ /// Ctor.
+ BitsIterator(T Initial) : Bits(Initial) {}
+
+ /// Next - Returns the next bit set or zero if exhausted.
+ inline T Next() {
+ // Get the rightmost bit set
+ T Result = Bits & -Bits;
+ // Remove from rest
+ Bits &= ~Result;
+ // Return single bit or zero
+ return Result;
+ }
+};
+
+//===----------------------------------------------------------------------===//
+
+
+//===----------------------------------------------------------------------===//
+///
+/// ResourceTally - Manages the use of resources over time intervals. Each
+/// item (slot) in the tally vector represents the resources used at a given
+/// moment. A bit set to 1 indicates that a resource is in use, otherwise
+/// available. An assumption is made that the tally is large enough to schedule
+/// all current instructions (asserts otherwise.)
+///
+template<class T>
+class ResourceTally {
+private:
+ std::vector<T> Tally; // Resources used per slot
+ typedef typename std::vector<T>::iterator Iter;
+ // Tally iterator
+
+ /// SlotsAvailable - Returns true if all units are available.
+ ///
+ bool SlotsAvailable(Iter Begin, unsigned N, unsigned ResourceSet,
+ unsigned &Resource) {
+ assert(N && "Must check availability with N != 0");
+ // Determine end of interval
+ Iter End = Begin + N;
+ assert(End <= Tally.end() && "Tally is not large enough for schedule");
+
+ // Iterate thru each resource
+ BitsIterator<T> Resources(ResourceSet & ~*Begin);
+ while (unsigned Res = Resources.Next()) {
+ // Check if resource is available for next N slots
+ Iter Interval = End;
+ do {
+ Interval--;
+ if (*Interval & Res) break;
+ } while (Interval != Begin);
+
+ // If available for N
+ if (Interval == Begin) {
+ // Success
+ Resource = Res;
+ return true;
+ }
+ }
+
+ // No luck
+ Resource = 0;
+ return false;
+ }
+
+ /// RetrySlot - Finds a good candidate slot to retry search.
+ Iter RetrySlot(Iter Begin, unsigned N, unsigned ResourceSet) {
+ assert(N && "Must check availability with N != 0");
+ // Determine end of interval
+ Iter End = Begin + N;
+ assert(End <= Tally.end() && "Tally is not large enough for schedule");
+
+ while (Begin != End--) {
+ // Clear units in use
+ ResourceSet &= ~*End;
+ // If no units left then we should go no further
+ if (!ResourceSet) return End + 1;
+ }
+ // Made it all the way through
+ return Begin;
+ }
+
+ /// FindAndReserveStages - Return true if the stages can be completed. If
+ /// so mark as busy.
+ bool FindAndReserveStages(Iter Begin,
+ InstrStage *Stage, InstrStage *StageEnd) {
+ // If at last stage then we're done
+ if (Stage == StageEnd) return true;
+ // Get number of cycles for current stage
+ unsigned N = Stage->Cycles;
+ // Check to see if N slots are available, if not fail
+ unsigned Resource;
+ if (!SlotsAvailable(Begin, N, Stage->Units, Resource)) return false;
+ // Check to see if remaining stages are available, if not fail
+ if (!FindAndReserveStages(Begin + N, Stage + 1, StageEnd)) return false;
+ // Reserve resource
+ Reserve(Begin, N, Resource);
+ // Success
+ return true;
+ }
+
+ /// Reserve - Mark busy (set) the specified N slots.
+ void Reserve(Iter Begin, unsigned N, unsigned Resource) {
+ // Determine end of interval
+ Iter End = Begin + N;
+ assert(End <= Tally.end() && "Tally is not large enough for schedule");
+
+ // Set resource bit in each slot
+ for (; Begin < End; Begin++)
+ *Begin |= Resource;
+ }
+
+ /// FindSlots - Starting from Begin, locate consecutive slots where all stages
+ /// can be completed. Returns the address of first slot.
+ Iter FindSlots(Iter Begin, InstrStage *StageBegin, InstrStage *StageEnd) {
+ // Track position
+ Iter Cursor = Begin;
+
+ // Try all possible slots forward
+ while (true) {
+ // Try at cursor, if successful return position.
+ if (FindAndReserveStages(Cursor, StageBegin, StageEnd)) return Cursor;
+ // Locate a better position
+ Cursor = RetrySlot(Cursor + 1, StageBegin->Cycles, StageBegin->Units);
+ }
+ }
+
+public:
+ /// Initialize - Resize and zero the tally to the specified number of time
+ /// slots.
+ inline void Initialize(unsigned N) {
+ Tally.assign(N, 0); // Initialize tally to all zeros.
+ }
+
+ // FindAndReserve - Locate an ideal slot for the specified stages and mark
+ // as busy.
+ unsigned FindAndReserve(unsigned Slot, InstrStage *StageBegin,
+ InstrStage *StageEnd) {
+ // Where to begin
+ Iter Begin = Tally.begin() + Slot;
+ // Find a free slot
+ Iter Where = FindSlots(Begin, StageBegin, StageEnd);
+ // Distance is slot number
+ unsigned Final = Where - Tally.begin();
+ return Final;
+ }
+
+};
+
+//===----------------------------------------------------------------------===//
+///
+/// ScheduleDAGSimple - Simple two pass scheduler.
+///
+class VISIBILITY_HIDDEN ScheduleDAGSimple : public ScheduleDAG {
+private:
+ bool NoSched; // Just do a BFS schedule, nothing fancy
+ bool NoItins; // Don't use itineraries?
+ ResourceTally<unsigned> Tally; // Resource usage tally
+ unsigned NSlots; // Total latency
+ static const unsigned NotFound = ~0U; // Search marker
+
+ unsigned NodeCount; // Number of nodes in DAG
+ std::map<SDNode *, NodeInfo *> Map; // Map nodes to info
+ bool HasGroups; // True if there are any groups
+ NodeInfo *Info; // Info for nodes being scheduled
+ NIVector Ordering; // Emit ordering of nodes
+ NodeGroup *HeadNG, *TailNG; // Keep track of allocated NodeGroups
+
+public:
+
+ // Ctor.
+ ScheduleDAGSimple(bool noSched, bool noItins, SelectionDAG &dag,
+ MachineBasicBlock *bb, const TargetMachine &tm)
+ : ScheduleDAG(dag, bb, tm), NoSched(noSched), NoItins(noItins), NSlots(0),
+ NodeCount(0), HasGroups(false), Info(NULL), HeadNG(NULL), TailNG(NULL) {
+ assert(&TII && "Target doesn't provide instr info?");
+ assert(&MRI && "Target doesn't provide register info?");
+ }
+
+ virtual ~ScheduleDAGSimple() {
+ if (Info)
+ delete[] Info;
+
+ NodeGroup *NG = HeadNG;
+ while (NG) {
+ NodeGroup *NextSU = NG->Next;
+ delete NG;
+ NG = NextSU;
+ }
+ }
+
+ void Schedule();
+
+ /// getNI - Returns the node info for the specified node.
+ ///
+ NodeInfo *getNI(SDNode *Node) { return Map[Node]; }
+
+private:
+ static bool isDefiner(NodeInfo *A, NodeInfo *B);
+ void IncludeNode(NodeInfo *NI);
+ void VisitAll();
+ void GatherSchedulingInfo();
+ void FakeGroupDominators();
+ bool isStrongDependency(NodeInfo *A, NodeInfo *B);
+ bool isWeakDependency(NodeInfo *A, NodeInfo *B);
+ void ScheduleBackward();
+ void ScheduleForward();
+
+ void AddToGroup(NodeInfo *D, NodeInfo *U);
+ /// PrepareNodeInfo - Set up the basic minimum node info for scheduling.
+ ///
+ void PrepareNodeInfo();
+
+ /// IdentifyGroups - Put flagged nodes into groups.
+ ///
+ void IdentifyGroups();
+
+ /// print - Print ordering to specified output stream.
+ ///
+ void print(std::ostream &O) const;
+ void print(std::ostream *O) const { if (O) print(*O); }
+
+ void dump(const char *tag) const;
+
+ virtual void dump() const;
+
+ /// EmitAll - Emit all nodes in schedule sorted order.
+ ///
+ void EmitAll();
+
+ /// printNI - Print node info.
+ ///
+ void printNI(std::ostream &O, NodeInfo *NI) const;
+ void printNI(std::ostream *O, NodeInfo *NI) const { if (O) printNI(*O, NI); }
+
+ /// printChanges - Hilight changes in order caused by scheduling.
+ ///
+ void printChanges(unsigned Index) const;
+};
+
+//===----------------------------------------------------------------------===//
+/// Special case itineraries.
+///
+enum {
+ CallLatency = 40, // To push calls back in time
+
+ RSInteger = 0xC0000000, // Two integer units
+ RSFloat = 0x30000000, // Two float units
+ RSLoadStore = 0x0C000000, // Two load store units
+ RSBranch = 0x02000000 // One branch unit
+};
+static InstrStage LoadStage = { 5, RSLoadStore };
+static InstrStage StoreStage = { 2, RSLoadStore };
+static InstrStage IntStage = { 2, RSInteger };
+static InstrStage FloatStage = { 3, RSFloat };
+//===----------------------------------------------------------------------===//
+
+} // namespace
+
+//===----------------------------------------------------------------------===//
+
+/// PrepareNodeInfo - Set up the basic minimum node info for scheduling.
+///
+void ScheduleDAGSimple::PrepareNodeInfo() {
+ // Allocate node information
+ Info = new NodeInfo[NodeCount];
+
+ unsigned i = 0;
+ for (SelectionDAG::allnodes_iterator I = DAG.allnodes_begin(),
+ E = DAG.allnodes_end(); I != E; ++I, ++i) {
+ // Fast reference to node schedule info
+ NodeInfo* NI = &Info[i];
+ // Set up map
+ Map[I] = NI;
+ // Set node
+ NI->Node = I;
+ // Set pending visit count
+ NI->setPending(I->use_size());
+ }
+}
+
+/// IdentifyGroups - Put flagged nodes into groups.
+///
+void ScheduleDAGSimple::IdentifyGroups() {
+ for (unsigned i = 0, N = NodeCount; i < N; i++) {
+ NodeInfo* NI = &Info[i];
+ SDNode *Node = NI->Node;
+
+ // For each operand (in reverse to only look at flags)
+ for (unsigned N = Node->getNumOperands(); 0 < N--;) {
+ // Get operand
+ SDOperand Op = Node->getOperand(N);
+ // No more flags to walk
+ if (Op.getValueType() != MVT::Flag) break;
+ // Add to node group
+ AddToGroup(getNI(Op.Val), NI);
+ // Let everyone else know
+ HasGroups = true;
+ }
+ }
+}
+
+/// CountInternalUses - Returns the number of edges between the two nodes.
+///
+static unsigned CountInternalUses(NodeInfo *D, NodeInfo *U) {
+ unsigned N = 0;
+ for (unsigned M = U->Node->getNumOperands(); 0 < M--;) {
+ SDOperand Op = U->Node->getOperand(M);
+ if (Op.Val == D->Node) N++;
+ }
+
+ return N;
+}
+
+//===----------------------------------------------------------------------===//
+/// Add - Adds a definer and user pair to a node group.
+///
+void ScheduleDAGSimple::AddToGroup(NodeInfo *D, NodeInfo *U) {
+ // Get current groups
+ NodeGroup *DGroup = D->Group;
+ NodeGroup *UGroup = U->Group;
+ // If both are members of groups
+ if (DGroup && UGroup) {
+ // There may have been another edge connecting
+ if (DGroup == UGroup) return;
+ // Add the pending users count
+ DGroup->addPending(UGroup->getPending());
+ // For each member of the users group
+ NodeGroupIterator UNGI(U);
+ while (NodeInfo *UNI = UNGI.next() ) {
+ // Change the group
+ UNI->Group = DGroup;
+ // For each member of the definers group
+ NodeGroupIterator DNGI(D);
+ while (NodeInfo *DNI = DNGI.next() ) {
+ // Remove internal edges
+ DGroup->addPending(-CountInternalUses(DNI, UNI));
+ }
+ }
+ // Merge the two lists
+ DGroup->group_insert(DGroup->group_end(),
+ UGroup->group_begin(), UGroup->group_end());
+ } else if (DGroup) {
+ // Make user member of definers group
+ U->Group = DGroup;
+ // Add users uses to definers group pending
+ DGroup->addPending(U->Node->use_size());
+ // For each member of the definers group
+ NodeGroupIterator DNGI(D);
+ while (NodeInfo *DNI = DNGI.next() ) {
+ // Remove internal edges
+ DGroup->addPending(-CountInternalUses(DNI, U));
+ }
+ DGroup->group_push_back(U);
+ } else if (UGroup) {
+ // Make definer member of users group
+ D->Group = UGroup;
+ // Add definers uses to users group pending
+ UGroup->addPending(D->Node->use_size());
+ // For each member of the users group
+ NodeGroupIterator UNGI(U);
+ while (NodeInfo *UNI = UNGI.next() ) {
+ // Remove internal edges
+ UGroup->addPending(-CountInternalUses(D, UNI));
+ }
+ UGroup->group_insert(UGroup->group_begin(), D);
+ } else {
+ D->Group = U->Group = DGroup = new NodeGroup();
+ DGroup->addPending(D->Node->use_size() + U->Node->use_size() -
+ CountInternalUses(D, U));
+ DGroup->group_push_back(D);
+ DGroup->group_push_back(U);
+
+ if (HeadNG == NULL)
+ HeadNG = DGroup;
+ if (TailNG != NULL)
+ TailNG->Next = DGroup;
+ TailNG = DGroup;
+ }
+}
+
+
+/// print - Print ordering to specified output stream.
+///
+void ScheduleDAGSimple::print(std::ostream &O) const {
+#ifndef NDEBUG
+ O << "Ordering\n";
+ for (unsigned i = 0, N = Ordering.size(); i < N; i++) {
+ NodeInfo *NI = Ordering[i];
+ printNI(O, NI);
+ O << "\n";
+ if (NI->isGroupDominator()) {
+ NodeGroup *Group = NI->Group;
+ for (NIIterator NII = Group->group_begin(), E = Group->group_end();
+ NII != E; NII++) {
+ O << " ";
+ printNI(O, *NII);
+ O << "\n";
+ }
+ }
+ }
+#endif
+}
+
+void ScheduleDAGSimple::dump(const char *tag) const {
+ cerr << tag; dump();
+}
+
+void ScheduleDAGSimple::dump() const {
+ print(cerr);
+}
+
+
+/// EmitAll - Emit all nodes in schedule sorted order.
+///
+void ScheduleDAGSimple::EmitAll() {
+ // If this is the first basic block in the function, and if it has live ins
+ // that need to be copied into vregs, emit the copies into the top of the
+ // block before emitting the code for the block.
+ MachineFunction &MF = DAG.getMachineFunction();
+ if (&MF.front() == BB && MF.livein_begin() != MF.livein_end()) {
+ for (MachineFunction::livein_iterator LI = MF.livein_begin(),
+ E = MF.livein_end(); LI != E; ++LI)
+ if (LI->second)
+ MRI->copyRegToReg(*MF.begin(), MF.begin()->end(), LI->second,
+ LI->first, RegMap->getRegClass(LI->second));
+ }
+
+ DenseMap<SDOperand, unsigned> VRBaseMap;
+
+ // For each node in the ordering
+ for (unsigned i = 0, N = Ordering.size(); i < N; i++) {
+ // Get the scheduling info
+ NodeInfo *NI = Ordering[i];
+ if (NI->isInGroup()) {
+ NodeGroupIterator NGI(Ordering[i]);
+ while (NodeInfo *NI = NGI.next()) EmitNode(NI->Node, VRBaseMap);
+ } else {
+ EmitNode(NI->Node, VRBaseMap);
+ }
+ }
+}
+
+/// isFlagDefiner - Returns true if the node defines a flag result.
+static bool isFlagDefiner(SDNode *A) {
+ unsigned N = A->getNumValues();
+ return N && A->getValueType(N - 1) == MVT::Flag;
+}
+
+/// isFlagUser - Returns true if the node uses a flag result.
+///
+static bool isFlagUser(SDNode *A) {
+ unsigned N = A->getNumOperands();
+ return N && A->getOperand(N - 1).getValueType() == MVT::Flag;
+}
+
+/// printNI - Print node info.
+///
+void ScheduleDAGSimple::printNI(std::ostream &O, NodeInfo *NI) const {
+#ifndef NDEBUG
+ SDNode *Node = NI->Node;
+ O << " "
+ << std::hex << Node << std::dec
+ << ", Lat=" << NI->Latency
+ << ", Slot=" << NI->Slot
+ << ", ARITY=(" << Node->getNumOperands() << ","
+ << Node->getNumValues() << ")"
+ << " " << Node->getOperationName(&DAG);
+ if (isFlagDefiner(Node)) O << "<#";
+ if (isFlagUser(Node)) O << ">#";
+#endif
+}
+
+/// printChanges - Hilight changes in order caused by scheduling.
+///
+void ScheduleDAGSimple::printChanges(unsigned Index) const {
+#ifndef NDEBUG
+ // Get the ordered node count
+ unsigned N = Ordering.size();
+ // Determine if any changes
+ unsigned i = 0;
+ for (; i < N; i++) {
+ NodeInfo *NI = Ordering[i];
+ if (NI->Preorder != i) break;
+ }
+
+ if (i < N) {
+ cerr << Index << ". New Ordering\n";
+
+ for (i = 0; i < N; i++) {
+ NodeInfo *NI = Ordering[i];
+ cerr << " " << NI->Preorder << ". ";
+ printNI(cerr, NI);
+ cerr << "\n";
+ if (NI->isGroupDominator()) {
+ NodeGroup *Group = NI->Group;
+ for (NIIterator NII = Group->group_begin(), E = Group->group_end();
+ NII != E; NII++) {
+ cerr << " ";
+ printNI(cerr, *NII);
+ cerr << "\n";
+ }
+ }
+ }
+ } else {
+ cerr << Index << ". No Changes\n";
+ }
+#endif
+}
+
+//===----------------------------------------------------------------------===//
+/// isDefiner - Return true if node A is a definer for B.
+///
+bool ScheduleDAGSimple::isDefiner(NodeInfo *A, NodeInfo *B) {
+ // While there are A nodes
+ NodeGroupIterator NII(A);
+ while (NodeInfo *NI = NII.next()) {
+ // Extract node
+ SDNode *Node = NI->Node;
+ // While there operands in nodes of B
+ NodeGroupOpIterator NGOI(B);
+ while (!NGOI.isEnd()) {
+ SDOperand Op = NGOI.next();
+ // If node from A defines a node in B
+ if (Node == Op.Val) return true;
+ }
+ }
+ return false;
+}
+
+/// IncludeNode - Add node to NodeInfo vector.
+///
+void ScheduleDAGSimple::IncludeNode(NodeInfo *NI) {
+ // Get node
+ SDNode *Node = NI->Node;
+ // Ignore entry node
+ if (Node->getOpcode() == ISD::EntryToken) return;
+ // Check current count for node
+ int Count = NI->getPending();
+ // If the node is already in list
+ if (Count < 0) return;
+ // Decrement count to indicate a visit
+ Count--;
+ // If count has gone to zero then add node to list
+ if (!Count) {
+ // Add node
+ if (NI->isInGroup()) {
+ Ordering.push_back(NI->Group->getDominator());
+ } else {
+ Ordering.push_back(NI);
+ }
+ // indicate node has been added
+ Count--;
+ }
+ // Mark as visited with new count
+ NI->setPending(Count);
+}
+
+/// GatherSchedulingInfo - Get latency and resource information about each node.
+///
+void ScheduleDAGSimple::GatherSchedulingInfo() {
+ // Get instruction itineraries for the target
+ const InstrItineraryData &InstrItins = TM.getInstrItineraryData();
+
+ // For each node
+ for (unsigned i = 0, N = NodeCount; i < N; i++) {
+ // Get node info
+ NodeInfo* NI = &Info[i];
+ SDNode *Node = NI->Node;
+
+ // If there are itineraries and it is a machine instruction
+ if (InstrItins.isEmpty() || NoItins) {
+ // If machine opcode
+ if (Node->isTargetOpcode()) {
+ // Get return type to guess which processing unit
+ MVT::ValueType VT = Node->getValueType(0);
+ // Get machine opcode
+ MachineOpCode TOpc = Node->getTargetOpcode();
+ NI->IsCall = TII->isCall(TOpc);
+ NI->IsLoad = TII->isLoad(TOpc);
+ NI->IsStore = TII->isStore(TOpc);
+
+ if (TII->isLoad(TOpc)) NI->StageBegin = &LoadStage;
+ else if (TII->isStore(TOpc)) NI->StageBegin = &StoreStage;
+ else if (MVT::isInteger(VT)) NI->StageBegin = &IntStage;
+ else if (MVT::isFloatingPoint(VT)) NI->StageBegin = &FloatStage;
+ if (NI->StageBegin) NI->StageEnd = NI->StageBegin + 1;
+ }
+ } else if (Node->isTargetOpcode()) {
+ // get machine opcode
+ MachineOpCode TOpc = Node->getTargetOpcode();
+ // Check to see if it is a call
+ NI->IsCall = TII->isCall(TOpc);
+ // Get itinerary stages for instruction
+ unsigned II = TII->getSchedClass(TOpc);
+ NI->StageBegin = InstrItins.begin(II);
+ NI->StageEnd = InstrItins.end(II);
+ }
+
+ // One slot for the instruction itself
+ NI->Latency = 1;
+
+ // Add long latency for a call to push it back in time
+ if (NI->IsCall) NI->Latency += CallLatency;
+
+ // Sum up all the latencies
+ for (InstrStage *Stage = NI->StageBegin, *E = NI->StageEnd;
+ Stage != E; Stage++) {
+ NI->Latency += Stage->Cycles;
+ }
+
+ // Sum up all the latencies for max tally size
+ NSlots += NI->Latency;
+ }
+
+ // Unify metrics if in a group
+ if (HasGroups) {
+ for (unsigned i = 0, N = NodeCount; i < N; i++) {
+ NodeInfo* NI = &Info[i];
+
+ if (NI->isInGroup()) {
+ NodeGroup *Group = NI->Group;
+
+ if (!Group->getDominator()) {
+ NIIterator NGI = Group->group_begin(), NGE = Group->group_end();
+ NodeInfo *Dominator = *NGI;
+ unsigned Latency = 0;
+
+ for (NGI++; NGI != NGE; NGI++) {
+ NodeInfo* NGNI = *NGI;
+ Latency += NGNI->Latency;
+ if (Dominator->Latency < NGNI->Latency) Dominator = NGNI;
+ }
+
+ Dominator->Latency = Latency;
+ Group->setDominator(Dominator);
+ }
+ }
+ }
+ }
+}
+
+/// VisitAll - Visit each node breadth-wise to produce an initial ordering.
+/// Note that the ordering in the Nodes vector is reversed.
+void ScheduleDAGSimple::VisitAll() {
+ // Add first element to list
+ NodeInfo *NI = getNI(DAG.getRoot().Val);
+ if (NI->isInGroup()) {
+ Ordering.push_back(NI->Group->getDominator());
+ } else {
+ Ordering.push_back(NI);
+ }
+
+ // Iterate through all nodes that have been added
+ for (unsigned i = 0; i < Ordering.size(); i++) { // note: size() varies
+ // Visit all operands
+ NodeGroupOpIterator NGI(Ordering[i]);
+ while (!NGI.isEnd()) {
+ // Get next operand
+ SDOperand Op = NGI.next();
+ // Get node
+ SDNode *Node = Op.Val;
+ // Ignore passive nodes
+ if (isPassiveNode(Node)) continue;
+ // Check out node
+ IncludeNode(getNI(Node));
+ }
+ }
+
+ // Add entry node last (IncludeNode filters entry nodes)
+ if (DAG.getEntryNode().Val != DAG.getRoot().Val)
+ Ordering.push_back(getNI(DAG.getEntryNode().Val));
+
+ // Reverse the order
+ std::reverse(Ordering.begin(), Ordering.end());
+}
+
+/// FakeGroupDominators - Set dominators for non-scheduling.
+///
+void ScheduleDAGSimple::FakeGroupDominators() {
+ for (unsigned i = 0, N = NodeCount; i < N; i++) {
+ NodeInfo* NI = &Info[i];
+
+ if (NI->isInGroup()) {
+ NodeGroup *Group = NI->Group;
+
+ if (!Group->getDominator()) {
+ Group->setDominator(NI);
+ }
+ }
+ }
+}
+
+/// isStrongDependency - Return true if node A has results used by node B.
+/// I.E., B must wait for latency of A.
+bool ScheduleDAGSimple::isStrongDependency(NodeInfo *A, NodeInfo *B) {
+ // If A defines for B then it's a strong dependency or
+ // if a load follows a store (may be dependent but why take a chance.)
+ return isDefiner(A, B) || (A->IsStore && B->IsLoad);
+}
+
+/// isWeakDependency Return true if node A produces a result that will
+/// conflict with operands of B. It is assumed that we have called
+/// isStrongDependency prior.
+bool ScheduleDAGSimple::isWeakDependency(NodeInfo *A, NodeInfo *B) {
+ // TODO check for conflicting real registers and aliases
+#if 0 // FIXME - Since we are in SSA form and not checking register aliasing
+ return A->Node->getOpcode() == ISD::EntryToken || isStrongDependency(B, A);
+#else
+ return A->Node->getOpcode() == ISD::EntryToken;
+#endif
+}
+
+/// ScheduleBackward - Schedule instructions so that any long latency
+/// instructions and the critical path get pushed back in time. Time is run in
+/// reverse to allow code reuse of the Tally and eliminate the overhead of
+/// biasing every slot indices against NSlots.
+void ScheduleDAGSimple::ScheduleBackward() {
+ // Size and clear the resource tally
+ Tally.Initialize(NSlots);
+ // Get number of nodes to schedule
+ unsigned N = Ordering.size();
+
+ // For each node being scheduled
+ for (unsigned i = N; 0 < i--;) {
+ NodeInfo *NI = Ordering[i];
+ // Track insertion
+ unsigned Slot = NotFound;
+
+ // Compare against those previously scheduled nodes
+ unsigned j = i + 1;
+ for (; j < N; j++) {
+ // Get following instruction
+ NodeInfo *Other = Ordering[j];
+
+ // Check dependency against previously inserted nodes
+ if (isStrongDependency(NI, Other)) {
+ Slot = Other->Slot + Other->Latency;
+ break;
+ } else if (isWeakDependency(NI, Other)) {
+ Slot = Other->Slot;
+ break;
+ }
+ }
+
+ // If independent of others (or first entry)
+ if (Slot == NotFound) Slot = 0;
+
+#if 0 // FIXME - measure later
+ // Find a slot where the needed resources are available
+ if (NI->StageBegin != NI->StageEnd)
+ Slot = Tally.FindAndReserve(Slot, NI->StageBegin, NI->StageEnd);
+#endif
+
+ // Set node slot
+ NI->Slot = Slot;
+
+ // Insert sort based on slot
+ j = i + 1;
+ for (; j < N; j++) {
+ // Get following instruction
+ NodeInfo *Other = Ordering[j];
+ // Should we look further (remember slots are in reverse time)
+ if (Slot >= Other->Slot) break;
+ // Shuffle other into ordering
+ Ordering[j - 1] = Other;
+ }
+ // Insert node in proper slot
+ if (j != i + 1) Ordering[j - 1] = NI;
+ }
+}
+
+/// ScheduleForward - Schedule instructions to maximize packing.
+///
+void ScheduleDAGSimple::ScheduleForward() {
+ // Size and clear the resource tally
+ Tally.Initialize(NSlots);
+ // Get number of nodes to schedule
+ unsigned N = Ordering.size();
+
+ // For each node being scheduled
+ for (unsigned i = 0; i < N; i++) {
+ NodeInfo *NI = Ordering[i];
+ // Track insertion
+ unsigned Slot = NotFound;
+
+ // Compare against those previously scheduled nodes
+ unsigned j = i;
+ for (; 0 < j--;) {
+ // Get following instruction
+ NodeInfo *Other = Ordering[j];
+
+ // Check dependency against previously inserted nodes
+ if (isStrongDependency(Other, NI)) {
+ Slot = Other->Slot + Other->Latency;
+ break;
+ } else if (Other->IsCall || isWeakDependency(Other, NI)) {
+ Slot = Other->Slot;
+ break;
+ }
+ }
+
+ // If independent of others (or first entry)
+ if (Slot == NotFound) Slot = 0;
+
+ // Find a slot where the needed resources are available
+ if (NI->StageBegin != NI->StageEnd)
+ Slot = Tally.FindAndReserve(Slot, NI->StageBegin, NI->StageEnd);
+
+ // Set node slot
+ NI->Slot = Slot;
+
+ // Insert sort based on slot
+ j = i;
+ for (; 0 < j--;) {
+ // Get prior instruction
+ NodeInfo *Other = Ordering[j];
+ // Should we look further
+ if (Slot >= Other->Slot) break;
+ // Shuffle other into ordering
+ Ordering[j + 1] = Other;
+ }
+ // Insert node in proper slot
+ if (j != i) Ordering[j + 1] = NI;
+ }
+}
+
+/// Schedule - Order nodes according to selected style.
+///
+void ScheduleDAGSimple::Schedule() {
+ // Number the nodes
+ NodeCount = std::distance(DAG.allnodes_begin(), DAG.allnodes_end());
+
+ // Set up minimum info for scheduling
+ PrepareNodeInfo();
+ // Construct node groups for flagged nodes
+ IdentifyGroups();
+
+ // Test to see if scheduling should occur
+ bool ShouldSchedule = NodeCount > 3 && !NoSched;
+ // Don't waste time if is only entry and return
+ if (ShouldSchedule) {
+ // Get latency and resource requirements
+ GatherSchedulingInfo();
+ } else if (HasGroups) {
+ // Make sure all the groups have dominators
+ FakeGroupDominators();
+ }
+
+ // Breadth first walk of DAG
+ VisitAll();
+
+#ifndef NDEBUG
+ static unsigned Count = 0;
+ Count++;
+ for (unsigned i = 0, N = Ordering.size(); i < N; i++) {
+ NodeInfo *NI = Ordering[i];
+ NI->Preorder = i;
+ }
+#endif
+
+ // Don't waste time if is only entry and return
+ if (ShouldSchedule) {
+ // Push back long instructions and critical path
+ ScheduleBackward();
+
+ // Pack instructions to maximize resource utilization
+ ScheduleForward();
+ }
+
+ DEBUG(printChanges(Count));
+
+ // Emit in scheduled order
+ EmitAll();
+}
+
+
+/// createSimpleDAGScheduler - This creates a simple two pass instruction
+/// scheduler using instruction itinerary.
+llvm::ScheduleDAG* llvm::createSimpleDAGScheduler(SelectionDAGISel *IS,
+ SelectionDAG *DAG,
+ MachineBasicBlock *BB) {
+ return new ScheduleDAGSimple(false, false, *DAG, BB, DAG->getTarget());
+}
+
+/// createNoItinsDAGScheduler - This creates a simple two pass instruction
+/// scheduler without using instruction itinerary.
+llvm::ScheduleDAG* llvm::createNoItinsDAGScheduler(SelectionDAGISel *IS,
+ SelectionDAG *DAG,
+ MachineBasicBlock *BB) {
+ return new ScheduleDAGSimple(false, true, *DAG, BB, DAG->getTarget());
+}
+
+/// createBFS_DAGScheduler - This creates a simple breadth first instruction
+/// scheduler.
+llvm::ScheduleDAG* llvm::createBFS_DAGScheduler(SelectionDAGISel *IS,
+ SelectionDAG *DAG,
+ MachineBasicBlock *BB) {
+ return new ScheduleDAGSimple(true, false, *DAG, BB, DAG->getTarget());
+}
diff --git a/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
new file mode 100644
index 0000000..9803ab8
--- /dev/null
+++ b/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -0,0 +1,3694 @@
+//===-- SelectionDAG.cpp - Implement the SelectionDAG data structures -----===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file was developed by the LLVM research group and is distributed under
+// the University of Illinois Open Source License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This implements the SelectionDAG class.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/CodeGen/SelectionDAG.h"
+#include "llvm/Constants.h"
+#include "llvm/GlobalVariable.h"
+#include "llvm/Intrinsics.h"
+#include "llvm/DerivedTypes.h"
+#include "llvm/Assembly/Writer.h"
+#include "llvm/CodeGen/MachineBasicBlock.h"
+#include "llvm/CodeGen/MachineConstantPool.h"
+#include "llvm/Support/MathExtras.h"
+#include "llvm/Target/MRegisterInfo.h"
+#include "llvm/Target/TargetData.h"
+#include "llvm/Target/TargetLowering.h"
+#include "llvm/Target/TargetInstrInfo.h"
+#include "llvm/Target/TargetMachine.h"
+#include "llvm/ADT/SetVector.h"
+#include "llvm/ADT/SmallPtrSet.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StringExtras.h"
+#include <algorithm>
+#include <cmath>
+using namespace llvm;
+
+/// makeVTList - Return an instance of the SDVTList struct initialized with the
+/// specified members.
+static SDVTList makeVTList(const MVT::ValueType *VTs, unsigned NumVTs) {
+ SDVTList Res = {VTs, NumVTs};
+ return Res;
+}
+
+//===----------------------------------------------------------------------===//
+// ConstantFPSDNode Class
+//===----------------------------------------------------------------------===//
+
+/// isExactlyValue - We don't rely on operator== working on double values, as
+/// it returns true for things that are clearly not equal, like -0.0 and 0.0.
+/// As such, this method can be used to do an exact bit-for-bit comparison of
+/// two floating point values.
+bool ConstantFPSDNode::isExactlyValue(double V) const {
+ return DoubleToBits(V) == DoubleToBits(Value);
+}
+
+//===----------------------------------------------------------------------===//
+// ISD Namespace
+//===----------------------------------------------------------------------===//
+
+/// isBuildVectorAllOnes - Return true if the specified node is a
+/// BUILD_VECTOR where all of the elements are ~0 or undef.
+bool ISD::isBuildVectorAllOnes(const SDNode *N) {
+ // Look through a bit convert.
+ if (N->getOpcode() == ISD::BIT_CONVERT)
+ N = N->getOperand(0).Val;
+
+ if (N->getOpcode() != ISD::BUILD_VECTOR) return false;
+
+ unsigned i = 0, e = N->getNumOperands();
+
+ // Skip over all of the undef values.
+ while (i != e && N->getOperand(i).getOpcode() == ISD::UNDEF)
+ ++i;
+
+ // Do not accept an all-undef vector.
+ if (i == e) return false;
+
+ // Do not accept build_vectors that aren't all constants or which have non-~0
+ // elements.
+ SDOperand NotZero = N->getOperand(i);
+ if (isa<ConstantSDNode>(NotZero)) {
+ if (!cast<ConstantSDNode>(NotZero)->isAllOnesValue())
+ return false;
+ } else if (isa<ConstantFPSDNode>(NotZero)) {
+ MVT::ValueType VT = NotZero.getValueType();
+ if (VT== MVT::f64) {
+ if (DoubleToBits(cast<ConstantFPSDNode>(NotZero)->getValue()) !=
+ (uint64_t)-1)
+ return false;
+ } else {
+ if (FloatToBits(cast<ConstantFPSDNode>(NotZero)->getValue()) !=
+ (uint32_t)-1)
+ return false;
+ }
+ } else
+ return false;
+
+ // Okay, we have at least one ~0 value, check to see if the rest match or are
+ // undefs.
+ for (++i; i != e; ++i)
+ if (N->getOperand(i) != NotZero &&
+ N->getOperand(i).getOpcode() != ISD::UNDEF)
+ return false;
+ return true;
+}
+
+
+/// isBuildVectorAllZeros - Return true if the specified node is a
+/// BUILD_VECTOR where all of the elements are 0 or undef.
+bool ISD::isBuildVectorAllZeros(const SDNode *N) {
+ // Look through a bit convert.
+ if (N->getOpcode() == ISD::BIT_CONVERT)
+ N = N->getOperand(0).Val;
+
+ if (N->getOpcode() != ISD::BUILD_VECTOR) return false;
+
+ unsigned i = 0, e = N->getNumOperands();
+
+ // Skip over all of the undef values.
+ while (i != e && N->getOperand(i).getOpcode() == ISD::UNDEF)
+ ++i;
+
+ // Do not accept an all-undef vector.
+ if (i == e) return false;
+
+ // Do not accept build_vectors that aren't all constants or which have non-~0
+ // elements.
+ SDOperand Zero = N->getOperand(i);
+ if (isa<ConstantSDNode>(Zero)) {
+ if (!cast<ConstantSDNode>(Zero)->isNullValue())
+ return false;
+ } else if (isa<ConstantFPSDNode>(Zero)) {
+ if (!cast<ConstantFPSDNode>(Zero)->isExactlyValue(0.0))
+ return false;
+ } else
+ return false;
+
+ // Okay, we have at least one ~0 value, check to see if the rest match or are
+ // undefs.
+ for (++i; i != e; ++i)
+ if (N->getOperand(i) != Zero &&
+ N->getOperand(i).getOpcode() != ISD::UNDEF)
+ return false;
+ return true;
+}
+
+/// getSetCCSwappedOperands - Return the operation corresponding to (Y op X)
+/// when given the operation for (X op Y).
+ISD::CondCode ISD::getSetCCSwappedOperands(ISD::CondCode Operation) {
+ // To perform this operation, we just need to swap the L and G bits of the
+ // operation.
+ unsigned OldL = (Operation >> 2) & 1;
+ unsigned OldG = (Operation >> 1) & 1;
+ return ISD::CondCode((Operation & ~6) | // Keep the N, U, E bits
+ (OldL << 1) | // New G bit
+ (OldG << 2)); // New L bit.
+}
+
+/// getSetCCInverse - Return the operation corresponding to !(X op Y), where
+/// 'op' is a valid SetCC operation.
+ISD::CondCode ISD::getSetCCInverse(ISD::CondCode Op, bool isInteger) {
+ unsigned Operation = Op;
+ if (isInteger)
+ Operation ^= 7; // Flip L, G, E bits, but not U.
+ else
+ Operation ^= 15; // Flip all of the condition bits.
+ if (Operation > ISD::SETTRUE2)
+ Operation &= ~8; // Don't let N and U bits get set.
+ return ISD::CondCode(Operation);
+}
+
+
+/// isSignedOp - For an integer comparison, return 1 if the comparison is a
+/// signed operation and 2 if the result is an unsigned comparison. Return zero
+/// if the operation does not depend on the sign of the input (setne and seteq).
+static int isSignedOp(ISD::CondCode Opcode) {
+ switch (Opcode) {
+ default: assert(0 && "Illegal integer setcc operation!");
+ case ISD::SETEQ:
+ case ISD::SETNE: return 0;
+ case ISD::SETLT:
+ case ISD::SETLE:
+ case ISD::SETGT:
+ case ISD::SETGE: return 1;
+ case ISD::SETULT:
+ case ISD::SETULE:
+ case ISD::SETUGT:
+ case ISD::SETUGE: return 2;
+ }
+}
+
+/// getSetCCOrOperation - Return the result of a logical OR between different
+/// comparisons of identical values: ((X op1 Y) | (X op2 Y)). This function
+/// returns SETCC_INVALID if it is not possible to represent the resultant
+/// comparison.
+ISD::CondCode ISD::getSetCCOrOperation(ISD::CondCode Op1, ISD::CondCode Op2,
+ bool isInteger) {
+ if (isInteger && (isSignedOp(Op1) | isSignedOp(Op2)) == 3)
+ // Cannot fold a signed integer setcc with an unsigned integer setcc.
+ return ISD::SETCC_INVALID;
+
+ unsigned Op = Op1 | Op2; // Combine all of the condition bits.
+
+ // If the N and U bits get set then the resultant comparison DOES suddenly
+ // care about orderedness, and is true when ordered.
+ if (Op > ISD::SETTRUE2)
+ Op &= ~16; // Clear the U bit if the N bit is set.
+
+ // Canonicalize illegal integer setcc's.
+ if (isInteger && Op == ISD::SETUNE) // e.g. SETUGT | SETULT
+ Op = ISD::SETNE;
+
+ return ISD::CondCode(Op);
+}
+
+/// getSetCCAndOperation - Return the result of a logical AND between different
+/// comparisons of identical values: ((X op1 Y) & (X op2 Y)). This
+/// function returns zero if it is not possible to represent the resultant
+/// comparison.
+ISD::CondCode ISD::getSetCCAndOperation(ISD::CondCode Op1, ISD::CondCode Op2,
+ bool isInteger) {
+ if (isInteger && (isSignedOp(Op1) | isSignedOp(Op2)) == 3)
+ // Cannot fold a signed setcc with an unsigned setcc.
+ return ISD::SETCC_INVALID;
+
+ // Combine all of the condition bits.
+ ISD::CondCode Result = ISD::CondCode(Op1 & Op2);
+
+ // Canonicalize illegal integer setcc's.
+ if (isInteger) {
+ switch (Result) {
+ default: break;
+ case ISD::SETUO : Result = ISD::SETFALSE; break; // SETUGT & SETULT
+ case ISD::SETUEQ: Result = ISD::SETEQ ; break; // SETUGE & SETULE
+ case ISD::SETOLT: Result = ISD::SETULT ; break; // SETULT & SETNE
+ case ISD::SETOGT: Result = ISD::SETUGT ; break; // SETUGT & SETNE
+ }
+ }
+
+ return Result;
+}
+
+const TargetMachine &SelectionDAG::getTarget() const {
+ return TLI.getTargetMachine();
+}
+
+//===----------------------------------------------------------------------===//
+// SDNode Profile Support
+//===----------------------------------------------------------------------===//
+
+/// AddNodeIDOpcode - Add the node opcode to the NodeID data.
+///
+static void AddNodeIDOpcode(FoldingSetNodeID &ID, unsigned OpC) {
+ ID.AddInteger(OpC);
+}
+
+/// AddNodeIDValueTypes - Value type lists are intern'd so we can represent them
+/// solely with their pointer.
+void AddNodeIDValueTypes(FoldingSetNodeID &ID, SDVTList VTList) {
+ ID.AddPointer(VTList.VTs);
+}
+
+/// AddNodeIDOperands - Various routines for adding operands to the NodeID data.
+///
+static void AddNodeIDOperands(FoldingSetNodeID &ID,
+ const SDOperand *Ops, unsigned NumOps) {
+ for (; NumOps; --NumOps, ++Ops) {
+ ID.AddPointer(Ops->Val);
+ ID.AddInteger(Ops->ResNo);
+ }
+}
+
+static void AddNodeIDNode(FoldingSetNodeID &ID,
+ unsigned short OpC, SDVTList VTList,
+ const SDOperand *OpList, unsigned N) {
+ AddNodeIDOpcode(ID, OpC);
+ AddNodeIDValueTypes(ID, VTList);
+ AddNodeIDOperands(ID, OpList, N);
+}
+
+/// AddNodeIDNode - Generic routine for adding a nodes info to the NodeID
+/// data.
+static void AddNodeIDNode(FoldingSetNodeID &ID, SDNode *N) {
+ AddNodeIDOpcode(ID, N->getOpcode());
+ // Add the return value info.
+ AddNodeIDValueTypes(ID, N->getVTList());
+ // Add the operand info.
+ AddNodeIDOperands(ID, N->op_begin(), N->getNumOperands());
+
+ // Handle SDNode leafs with special info.
+ switch (N->getOpcode()) {
+ default: break; // Normal nodes don't need extra info.
+ case ISD::TargetConstant:
+ case ISD::Constant:
+ ID.AddInteger(cast<ConstantSDNode>(N)->getValue());
+ break;
+ case ISD::TargetConstantFP:
+ case ISD::ConstantFP:
+ ID.AddDouble(cast<ConstantFPSDNode>(N)->getValue());
+ break;
+ case ISD::TargetGlobalAddress:
+ case ISD::GlobalAddress:
+ case ISD::TargetGlobalTLSAddress:
+ case ISD::GlobalTLSAddress: {
+ GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(N);
+ ID.AddPointer(GA->getGlobal());
+ ID.AddInteger(GA->getOffset());
+ break;
+ }
+ case ISD::BasicBlock:
+ ID.AddPointer(cast<BasicBlockSDNode>(N)->getBasicBlock());
+ break;
+ case ISD::Register:
+ ID.AddInteger(cast<RegisterSDNode>(N)->getReg());
+ break;
+ case ISD::SRCVALUE: {
+ SrcValueSDNode *SV = cast<SrcValueSDNode>(N);
+ ID.AddPointer(SV->getValue());
+ ID.AddInteger(SV->getOffset());
+ break;
+ }
+ case ISD::FrameIndex:
+ case ISD::TargetFrameIndex:
+ ID.AddInteger(cast<FrameIndexSDNode>(N)->getIndex());
+ break;
+ case ISD::JumpTable:
+ case ISD::TargetJumpTable:
+ ID.AddInteger(cast<JumpTableSDNode>(N)->getIndex());
+ break;
+ case ISD::ConstantPool:
+ case ISD::TargetConstantPool: {
+ ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(N);
+ ID.AddInteger(CP->getAlignment());
+ ID.AddInteger(CP->getOffset());
+ if (CP->isMachineConstantPoolEntry())
+ CP->getMachineCPVal()->AddSelectionDAGCSEId(ID);
+ else
+ ID.AddPointer(CP->getConstVal());
+ break;
+ }
+ case ISD::LOAD: {
+ LoadSDNode *LD = cast<LoadSDNode>(N);
+ ID.AddInteger(LD->getAddressingMode());
+ ID.AddInteger(LD->getExtensionType());
+ ID.AddInteger(LD->getLoadedVT());
+ ID.AddPointer(LD->getSrcValue());
+ ID.AddInteger(LD->getSrcValueOffset());
+ ID.AddInteger(LD->getAlignment());
+ ID.AddInteger(LD->isVolatile());
+ break;
+ }
+ case ISD::STORE: {
+ StoreSDNode *ST = cast<StoreSDNode>(N);
+ ID.AddInteger(ST->getAddressingMode());
+ ID.AddInteger(ST->isTruncatingStore());
+ ID.AddInteger(ST->getStoredVT());
+ ID.AddPointer(ST->getSrcValue());
+ ID.AddInteger(ST->getSrcValueOffset());
+ ID.AddInteger(ST->getAlignment());
+ ID.AddInteger(ST->isVolatile());
+ break;
+ }
+ }
+}
+
+//===----------------------------------------------------------------------===//
+// SelectionDAG Class
+//===----------------------------------------------------------------------===//
+
+/// RemoveDeadNodes - This method deletes all unreachable nodes in the
+/// SelectionDAG.
+void SelectionDAG::RemoveDeadNodes() {
+ // Create a dummy node (which is not added to allnodes), that adds a reference
+ // to the root node, preventing it from being deleted.
+ HandleSDNode Dummy(getRoot());
+
+ SmallVector<SDNode*, 128> DeadNodes;
+
+ // Add all obviously-dead nodes to the DeadNodes worklist.
+ for (allnodes_iterator I = allnodes_begin(), E = allnodes_end(); I != E; ++I)
+ if (I->use_empty())
+ DeadNodes.push_back(I);
+
+ // Process the worklist, deleting the nodes and adding their uses to the
+ // worklist.
+ while (!DeadNodes.empty()) {
+ SDNode *N = DeadNodes.back();
+ DeadNodes.pop_back();
+
+ // Take the node out of the appropriate CSE map.
+ RemoveNodeFromCSEMaps(N);
+
+ // Next, brutally remove the operand list. This is safe to do, as there are
+ // no cycles in the graph.
+ for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ++I) {
+ SDNode *Operand = I->Val;
+ Operand->removeUser(N);
+
+ // Now that we removed this operand, see if there are no uses of it left.
+ if (Operand->use_empty())
+ DeadNodes.push_back(Operand);
+ }
+ if (N->OperandsNeedDelete)
+ delete[] N->OperandList;
+ N->OperandList = 0;
+ N->NumOperands = 0;
+
+ // Finally, remove N itself.
+ AllNodes.erase(N);
+ }
+
+ // If the root changed (e.g. it was a dead load, update the root).
+ setRoot(Dummy.getValue());
+}
+
+void SelectionDAG::RemoveDeadNode(SDNode *N, std::vector<SDNode*> &Deleted) {
+ SmallVector<SDNode*, 16> DeadNodes;
+ DeadNodes.push_back(N);
+
+ // Process the worklist, deleting the nodes and adding their uses to the
+ // worklist.
+ while (!DeadNodes.empty()) {
+ SDNode *N = DeadNodes.back();
+ DeadNodes.pop_back();
+
+ // Take the node out of the appropriate CSE map.
+ RemoveNodeFromCSEMaps(N);
+
+ // Next, brutally remove the operand list. This is safe to do, as there are
+ // no cycles in the graph.
+ for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ++I) {
+ SDNode *Operand = I->Val;
+ Operand->removeUser(N);
+
+ // Now that we removed this operand, see if there are no uses of it left.
+ if (Operand->use_empty())
+ DeadNodes.push_back(Operand);
+ }
+ if (N->OperandsNeedDelete)
+ delete[] N->OperandList;
+ N->OperandList = 0;
+ N->NumOperands = 0;
+
+ // Finally, remove N itself.
+ Deleted.push_back(N);
+ AllNodes.erase(N);
+ }
+}
+
+void SelectionDAG::DeleteNode(SDNode *N) {
+ assert(N->use_empty() && "Cannot delete a node that is not dead!");
+
+ // First take this out of the appropriate CSE map.
+ RemoveNodeFromCSEMaps(N);
+
+ // Finally, remove uses due to operands of this node, remove from the
+ // AllNodes list, and delete the node.
+ DeleteNodeNotInCSEMaps(N);
+}
+
+void SelectionDAG::DeleteNodeNotInCSEMaps(SDNode *N) {
+
+ // Remove it from the AllNodes list.
+ AllNodes.remove(N);
+
+ // Drop all of the operands and decrement used nodes use counts.
+ for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ++I)
+ I->Val->removeUser(N);
+ if (N->OperandsNeedDelete)
+ delete[] N->OperandList;
+ N->OperandList = 0;
+ N->NumOperands = 0;
+
+ delete N;
+}
+
+/// RemoveNodeFromCSEMaps - Take the specified node out of the CSE map that
+/// correspond to it. This is useful when we're about to delete or repurpose
+/// the node. We don't want future request for structurally identical nodes
+/// to return N anymore.
+void SelectionDAG::RemoveNodeFromCSEMaps(SDNode *N) {
+ bool Erased = false;
+ switch (N->getOpcode()) {
+ case ISD::HANDLENODE: return; // noop.
+ case ISD::STRING:
+ Erased = StringNodes.erase(cast<StringSDNode>(N)->getValue());
+ break;
+ case ISD::CONDCODE:
+ assert(CondCodeNodes[cast<CondCodeSDNode>(N)->get()] &&
+ "Cond code doesn't exist!");
+ Erased = CondCodeNodes[cast<CondCodeSDNode>(N)->get()] != 0;
+ CondCodeNodes[cast<CondCodeSDNode>(N)->get()] = 0;
+ break;
+ case ISD::ExternalSymbol:
+ Erased = ExternalSymbols.erase(cast<ExternalSymbolSDNode>(N)->getSymbol());
+ break;
+ case ISD::TargetExternalSymbol:
+ Erased =
+ TargetExternalSymbols.erase(cast<ExternalSymbolSDNode>(N)->getSymbol());
+ break;
+ case ISD::VALUETYPE:
+ Erased = ValueTypeNodes[cast<VTSDNode>(N)->getVT()] != 0;
+ ValueTypeNodes[cast<VTSDNode>(N)->getVT()] = 0;
+ break;
+ default:
+ // Remove it from the CSE Map.
+ Erased = CSEMap.RemoveNode(N);
+ break;
+ }
+#ifndef NDEBUG
+ // Verify that the node was actually in one of the CSE maps, unless it has a
+ // flag result (which cannot be CSE'd) or is one of the special cases that are
+ // not subject to CSE.
+ if (!Erased && N->getValueType(N->getNumValues()-1) != MVT::Flag &&
+ !N->isTargetOpcode()) {
+ N->dump(this);
+ cerr << "\n";
+ assert(0 && "Node is not in map!");
+ }
+#endif
+}
+
+/// AddNonLeafNodeToCSEMaps - Add the specified node back to the CSE maps. It
+/// has been taken out and modified in some way. If the specified node already
+/// exists in the CSE maps, do not modify the maps, but return the existing node
+/// instead. If it doesn't exist, add it and return null.
+///
+SDNode *SelectionDAG::AddNonLeafNodeToCSEMaps(SDNode *N) {
+ assert(N->getNumOperands() && "This is a leaf node!");
+ if (N->getOpcode() == ISD::HANDLENODE || N->getValueType(0) == MVT::Flag)
+ return 0; // Never add these nodes.
+
+ // Check that remaining values produced are not flags.
+ for (unsigned i = 1, e = N->getNumValues(); i != e; ++i)
+ if (N->getValueType(i) == MVT::Flag)
+ return 0; // Never CSE anything that produces a flag.
+
+ SDNode *New = CSEMap.GetOrInsertNode(N);
+ if (New != N) return New; // Node already existed.
+ return 0;
+}
+
+/// FindModifiedNodeSlot - Find a slot for the specified node if its operands
+/// were replaced with those specified. If this node is never memoized,
+/// return null, otherwise return a pointer to the slot it would take. If a
+/// node already exists with these operands, the slot will be non-null.
+SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, SDOperand Op,
+ void *&InsertPos) {
+ if (N->getOpcode() == ISD::HANDLENODE || N->getValueType(0) == MVT::Flag)
+ return 0; // Never add these nodes.
+
+ // Check that remaining values produced are not flags.
+ for (unsigned i = 1, e = N->getNumValues(); i != e; ++i)
+ if (N->getValueType(i) == MVT::Flag)
+ return 0; // Never CSE anything that produces a flag.
+
+ SDOperand Ops[] = { Op };
+ FoldingSetNodeID ID;
+ AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops, 1);
+ return CSEMap.FindNodeOrInsertPos(ID, InsertPos);
+}
+
+/// FindModifiedNodeSlot - Find a slot for the specified node if its operands
+/// were replaced with those specified. If this node is never memoized,
+/// return null, otherwise return a pointer to the slot it would take. If a
+/// node already exists with these operands, the slot will be non-null.
+SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N,
+ SDOperand Op1, SDOperand Op2,
+ void *&InsertPos) {
+ if (N->getOpcode() == ISD::HANDLENODE || N->getValueType(0) == MVT::Flag)
+ return 0; // Never add these nodes.
+
+ // Check that remaining values produced are not flags.
+ for (unsigned i = 1, e = N->getNumValues(); i != e; ++i)
+ if (N->getValueType(i) == MVT::Flag)
+ return 0; // Never CSE anything that produces a flag.
+
+ SDOperand Ops[] = { Op1, Op2 };
+ FoldingSetNodeID ID;
+ AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops, 2);
+ return CSEMap.FindNodeOrInsertPos(ID, InsertPos);
+}
+
+
+/// FindModifiedNodeSlot - Find a slot for the specified node if its operands
+/// were replaced with those specified. If this node is never memoized,
+/// return null, otherwise return a pointer to the slot it would take. If a
+/// node already exists with these operands, the slot will be non-null.
+SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N,
+ const SDOperand *Ops,unsigned NumOps,
+ void *&InsertPos) {
+ if (N->getOpcode() == ISD::HANDLENODE || N->getValueType(0) == MVT::Flag)
+ return 0; // Never add these nodes.
+
+ // Check that remaining values produced are not flags.
+ for (unsigned i = 1, e = N->getNumValues(); i != e; ++i)
+ if (N->getValueType(i) == MVT::Flag)
+ return 0; // Never CSE anything that produces a flag.
+
+ FoldingSetNodeID ID;
+ AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops, NumOps);
+
+ if (const LoadSDNode *LD = dyn_cast<LoadSDNode>(N)) {
+ ID.AddInteger(LD->getAddressingMode());
+ ID.AddInteger(LD->getExtensionType());
+ ID.AddInteger(LD->getLoadedVT());
+ ID.AddPointer(LD->getSrcValue());
+ ID.AddInteger(LD->getSrcValueOffset());
+ ID.AddInteger(LD->getAlignment());
+ ID.AddInteger(LD->isVolatile());
+ } else if (const StoreSDNode *ST = dyn_cast<StoreSDNode>(N)) {
+ ID.AddInteger(ST->getAddressingMode());
+ ID.AddInteger(ST->isTruncatingStore());
+ ID.AddInteger(ST->getStoredVT());
+ ID.AddPointer(ST->getSrcValue());
+ ID.AddInteger(ST->getSrcValueOffset());
+ ID.AddInteger(ST->getAlignment());
+ ID.AddInteger(ST->isVolatile());
+ }
+
+ return CSEMap.FindNodeOrInsertPos(ID, InsertPos);
+}
+
+
+SelectionDAG::~SelectionDAG() {
+ while (!AllNodes.empty()) {
+ SDNode *N = AllNodes.begin();
+ N->SetNextInBucket(0);
+ if (N->OperandsNeedDelete)
+ delete [] N->OperandList;
+ N->OperandList = 0;
+ N->NumOperands = 0;
+ AllNodes.pop_front();
+ }
+}
+
+SDOperand SelectionDAG::getZeroExtendInReg(SDOperand Op, MVT::ValueType VT) {
+ if (Op.getValueType() == VT) return Op;
+ int64_t Imm = ~0ULL >> (64-MVT::getSizeInBits(VT));
+ return getNode(ISD::AND, Op.getValueType(), Op,
+ getConstant(Imm, Op.getValueType()));
+}
+
+SDOperand SelectionDAG::getString(const std::string &Val) {
+ StringSDNode *&N = StringNodes[Val];
+ if (!N) {
+ N = new StringSDNode(Val);
+ AllNodes.push_back(N);
+ }
+ return SDOperand(N, 0);
+}
+
+SDOperand SelectionDAG::getConstant(uint64_t Val, MVT::ValueType VT, bool isT) {
+ assert(MVT::isInteger(VT) && "Cannot create FP integer constant!");
+ assert(!MVT::isVector(VT) && "Cannot create Vector ConstantSDNodes!");
+
+ // Mask out any bits that are not valid for this constant.
+ Val &= MVT::getIntVTBitMask(VT);
+
+ unsigned Opc = isT ? ISD::TargetConstant : ISD::Constant;
+ FoldingSetNodeID ID;
+ AddNodeIDNode(ID, Opc, getVTList(VT), 0, 0);
+ ID.AddInteger(Val);
+ void *IP = 0;
+ if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
+ return SDOperand(E, 0);
+ SDNode *N = new ConstantSDNode(isT, Val, VT);
+ CSEMap.InsertNode(N, IP);
+ AllNodes.push_back(N);
+ return SDOperand(N, 0);
+}
+
+
+SDOperand SelectionDAG::getConstantFP(double Val, MVT::ValueType VT,
+ bool isTarget) {
+ assert(MVT::isFloatingPoint(VT) && "Cannot create integer FP constant!");
+ MVT::ValueType EltVT =
+ MVT::isVector(VT) ? MVT::getVectorElementType(VT) : VT;
+ if (EltVT == MVT::f32)
+ Val = (float)Val; // Mask out extra precision.
+
+ // Do the map lookup using the actual bit pattern for the floating point
+ // value, so that we don't have problems with 0.0 comparing equal to -0.0, and
+ // we don't have issues with SNANs.
+ unsigned Opc = isTarget ? ISD::TargetConstantFP : ISD::ConstantFP;
+ FoldingSetNodeID ID;
+ AddNodeIDNode(ID, Opc, getVTList(EltVT), 0, 0);
+ ID.AddDouble(Val);
+ void *IP = 0;
+ SDNode *N = NULL;
+ if ((N = CSEMap.FindNodeOrInsertPos(ID, IP)))
+ if (!MVT::isVector(VT))
+ return SDOperand(N, 0);
+ if (!N) {
+ N = new ConstantFPSDNode(isTarget, Val, EltVT);
+ CSEMap.InsertNode(N, IP);
+ AllNodes.push_back(N);
+ }
+
+ SDOperand Result(N, 0);
+ if (MVT::isVector(VT)) {
+ SmallVector<SDOperand, 8> Ops;
+ Ops.assign(MVT::getVectorNumElements(VT), Result);
+ Result = getNode(ISD::BUILD_VECTOR, VT, &Ops[0], Ops.size());
+ }
+ return Result;
+}
+
+SDOperand SelectionDAG::getGlobalAddress(const GlobalValue *GV,
+ MVT::ValueType VT, int Offset,
+ bool isTargetGA) {
+ const GlobalVariable *GVar = dyn_cast<GlobalVariable>(GV);
+ unsigned Opc;
+ if (GVar && GVar->isThreadLocal())
+ Opc = isTargetGA ? ISD::TargetGlobalTLSAddress : ISD::GlobalTLSAddress;
+ else
+ Opc = isTargetGA ? ISD::TargetGlobalAddress : ISD::GlobalAddress;
+ FoldingSetNodeID ID;
+ AddNodeIDNode(ID, Opc, getVTList(VT), 0, 0);
+ ID.AddPointer(GV);
+ ID.AddInteger(Offset);
+ void *IP = 0;
+ if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
+ return SDOperand(E, 0);
+ SDNode *N = new GlobalAddressSDNode(isTargetGA, GV, VT, Offset);
+ CSEMap.InsertNode(N, IP);
+ AllNodes.push_back(N);
+ return SDOperand(N, 0);
+}
+
+SDOperand SelectionDAG::getFrameIndex(int FI, MVT::ValueType VT,
+ bool isTarget) {
+ unsigned Opc = isTarget ? ISD::TargetFrameIndex : ISD::FrameIndex;
+ FoldingSetNodeID ID;
+ AddNodeIDNode(ID, Opc, getVTList(VT), 0, 0);
+ ID.AddInteger(FI);
+ void *IP = 0;
+ if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
+ return SDOperand(E, 0);
+ SDNode *N = new FrameIndexSDNode(FI, VT, isTarget);
+ CSEMap.InsertNode(N, IP);
+ AllNodes.push_back(N);
+ return SDOperand(N, 0);
+}
+
+SDOperand SelectionDAG::getJumpTable(int JTI, MVT::ValueType VT, bool isTarget){
+ unsigned Opc = isTarget ? ISD::TargetJumpTable : ISD::JumpTable;
+ FoldingSetNodeID ID;
+ AddNodeIDNode(ID, Opc, getVTList(VT), 0, 0);
+ ID.AddInteger(JTI);
+ void *IP = 0;
+ if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
+ return SDOperand(E, 0);
+ SDNode *N = new JumpTableSDNode(JTI, VT, isTarget);
+ CSEMap.InsertNode(N, IP);
+ AllNodes.push_back(N);
+ return SDOperand(N, 0);
+}
+
+SDOperand SelectionDAG::getConstantPool(Constant *C, MVT::ValueType VT,
+ unsigned Alignment, int Offset,
+ bool isTarget) {
+ unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool;
+ FoldingSetNodeID ID;
+ AddNodeIDNode(ID, Opc, getVTList(VT), 0, 0);
+ ID.AddInteger(Alignment);
+ ID.AddInteger(Offset);
+ ID.AddPointer(C);
+ void *IP = 0;
+ if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
+ return SDOperand(E, 0);
+ SDNode *N = new ConstantPoolSDNode(isTarget, C, VT, Offset, Alignment);
+ CSEMap.InsertNode(N, IP);
+ AllNodes.push_back(N);
+ return SDOperand(N, 0);
+}
+
+
+SDOperand SelectionDAG::getConstantPool(MachineConstantPoolValue *C,
+ MVT::ValueType VT,
+ unsigned Alignment, int Offset,
+ bool isTarget) {
+ unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool;
+ FoldingSetNodeID ID;
+ AddNodeIDNode(ID, Opc, getVTList(VT), 0, 0);
+ ID.AddInteger(Alignment);
+ ID.AddInteger(Offset);
+ C->AddSelectionDAGCSEId(ID);
+ void *IP = 0;
+ if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
+ return SDOperand(E, 0);
+ SDNode *N = new ConstantPoolSDNode(isTarget, C, VT, Offset, Alignment);
+ CSEMap.InsertNode(N, IP);
+ AllNodes.push_back(N);
+ return SDOperand(N, 0);
+}
+
+
+SDOperand SelectionDAG::getBasicBlock(MachineBasicBlock *MBB) {
+ FoldingSetNodeID ID;
+ AddNodeIDNode(ID, ISD::BasicBlock, getVTList(MVT::Other), 0, 0);
+ ID.AddPointer(MBB);
+ void *IP = 0;
+ if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
+ return SDOperand(E, 0);
+ SDNode *N = new BasicBlockSDNode(MBB);
+ CSEMap.InsertNode(N, IP);
+ AllNodes.push_back(N);
+ return SDOperand(N, 0);
+}
+
+SDOperand SelectionDAG::getValueType(MVT::ValueType VT) {
+ if ((unsigned)VT >= ValueTypeNodes.size())
+ ValueTypeNodes.resize(VT+1);
+ if (ValueTypeNodes[VT] == 0) {
+ ValueTypeNodes[VT] = new VTSDNode(VT);
+ AllNodes.push_back(ValueTypeNodes[VT]);
+ }
+
+ return SDOperand(ValueTypeNodes[VT], 0);
+}
+
+SDOperand SelectionDAG::getExternalSymbol(const char *Sym, MVT::ValueType VT) {
+ SDNode *&N = ExternalSymbols[Sym];
+ if (N) return SDOperand(N, 0);
+ N = new ExternalSymbolSDNode(false, Sym, VT);
+ AllNodes.push_back(N);
+ return SDOperand(N, 0);
+}
+
+SDOperand SelectionDAG::getTargetExternalSymbol(const char *Sym,
+ MVT::ValueType VT) {
+ SDNode *&N = TargetExternalSymbols[Sym];
+ if (N) return SDOperand(N, 0);
+ N = new ExternalSymbolSDNode(true, Sym, VT);
+ AllNodes.push_back(N);
+ return SDOperand(N, 0);
+}
+
+SDOperand SelectionDAG::getCondCode(ISD::CondCode Cond) {
+ if ((unsigned)Cond >= CondCodeNodes.size())
+ CondCodeNodes.resize(Cond+1);
+
+ if (CondCodeNodes[Cond] == 0) {
+ CondCodeNodes[Cond] = new CondCodeSDNode(Cond);
+ AllNodes.push_back(CondCodeNodes[Cond]);
+ }
+ return SDOperand(CondCodeNodes[Cond], 0);
+}
+
+SDOperand SelectionDAG::getRegister(unsigned RegNo, MVT::ValueType VT) {
+ FoldingSetNodeID ID;
+ AddNodeIDNode(ID, ISD::Register, getVTList(VT), 0, 0);
+ ID.AddInteger(RegNo);
+ void *IP = 0;
+ if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
+ return SDOperand(E, 0);
+ SDNode *N = new RegisterSDNode(RegNo, VT);
+ CSEMap.InsertNode(N, IP);
+ AllNodes.push_back(N);
+ return SDOperand(N, 0);
+}
+
+SDOperand SelectionDAG::getSrcValue(const Value *V, int Offset) {
+ assert((!V || isa<PointerType>(V->getType())) &&
+ "SrcValue is not a pointer?");
+
+ FoldingSetNodeID ID;
+ AddNodeIDNode(ID, ISD::SRCVALUE, getVTList(MVT::Other), 0, 0);
+ ID.AddPointer(V);
+ ID.AddInteger(Offset);
+ void *IP = 0;
+ if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
+ return SDOperand(E, 0);
+ SDNode *N = new SrcValueSDNode(V, Offset);
+ CSEMap.InsertNode(N, IP);
+ AllNodes.push_back(N);
+ return SDOperand(N, 0);
+}
+
+SDOperand SelectionDAG::FoldSetCC(MVT::ValueType VT, SDOperand N1,
+ SDOperand N2, ISD::CondCode Cond) {
+ // These setcc operations always fold.
+ switch (Cond) {
+ default: break;
+ case ISD::SETFALSE:
+ case ISD::SETFALSE2: return getConstant(0, VT);
+ case ISD::SETTRUE:
+ case ISD::SETTRUE2: return getConstant(1, VT);
+
+ case ISD::SETOEQ:
+ case ISD::SETOGT:
+ case ISD::SETOGE:
+ case ISD::SETOLT:
+ case ISD::SETOLE:
+ case ISD::SETONE:
+ case ISD::SETO:
+ case ISD::SETUO:
+ case ISD::SETUEQ:
+ case ISD::SETUNE:
+ assert(!MVT::isInteger(N1.getValueType()) && "Illegal setcc for integer!");
+ break;
+ }
+
+ if (ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2.Val)) {
+ uint64_t C2 = N2C->getValue();
+ if (ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.Val)) {
+ uint64_t C1 = N1C->getValue();
+
+ // Sign extend the operands if required
+ if (ISD::isSignedIntSetCC(Cond)) {
+ C1 = N1C->getSignExtended();
+ C2 = N2C->getSignExtended();
+ }
+
+ switch (Cond) {
+ default: assert(0 && "Unknown integer setcc!");
+ case ISD::SETEQ: return getConstant(C1 == C2, VT);
+ case ISD::SETNE: return getConstant(C1 != C2, VT);
+ case ISD::SETULT: return getConstant(C1 < C2, VT);
+ case ISD::SETUGT: return getConstant(C1 > C2, VT);
+ case ISD::SETULE: return getConstant(C1 <= C2, VT);
+ case ISD::SETUGE: return getConstant(C1 >= C2, VT);
+ case ISD::SETLT: return getConstant((int64_t)C1 < (int64_t)C2, VT);
+ case ISD::SETGT: return getConstant((int64_t)C1 > (int64_t)C2, VT);
+ case ISD::SETLE: return getConstant((int64_t)C1 <= (int64_t)C2, VT);
+ case ISD::SETGE: return getConstant((int64_t)C1 >= (int64_t)C2, VT);
+ }
+ }
+ }
+ if (ConstantFPSDNode *N1C = dyn_cast<ConstantFPSDNode>(N1.Val))
+ if (ConstantFPSDNode *N2C = dyn_cast<ConstantFPSDNode>(N2.Val)) {
+ double C1 = N1C->getValue(), C2 = N2C->getValue();
+
+ switch (Cond) {
+ default: break; // FIXME: Implement the rest of these!
+ case ISD::SETEQ: return getConstant(C1 == C2, VT);
+ case ISD::SETNE: return getConstant(C1 != C2, VT);
+ case ISD::SETLT: return getConstant(C1 < C2, VT);
+ case ISD::SETGT: return getConstant(C1 > C2, VT);
+ case ISD::SETLE: return getConstant(C1 <= C2, VT);
+ case ISD::SETGE: return getConstant(C1 >= C2, VT);
+ }
+ } else {
+ // Ensure that the constant occurs on the RHS.
+ return getSetCC(VT, N2, N1, ISD::getSetCCSwappedOperands(Cond));
+ }
+
+ // Could not fold it.
+ return SDOperand();
+}
+
+/// MaskedValueIsZero - Return true if 'V & Mask' is known to be zero. We use
+/// this predicate to simplify operations downstream. Mask is known to be zero
+/// for bits that V cannot have.
+bool SelectionDAG::MaskedValueIsZero(SDOperand Op, uint64_t Mask,
+ unsigned Depth) const {
+ // The masks are not wide enough to represent this type! Should use APInt.
+ if (Op.getValueType() == MVT::i128)
+ return false;
+
+ uint64_t KnownZero, KnownOne;
+ ComputeMaskedBits(Op, Mask, KnownZero, KnownOne, Depth);
+ assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?");
+ return (KnownZero & Mask) == Mask;
+}
+
+/// ComputeMaskedBits - Determine which of the bits specified in Mask are
+/// known to be either zero or one and return them in the KnownZero/KnownOne
+/// bitsets. This code only analyzes bits in Mask, in order to short-circuit
+/// processing.
+void SelectionDAG::ComputeMaskedBits(SDOperand Op, uint64_t Mask,
+ uint64_t &KnownZero, uint64_t &KnownOne,
+ unsigned Depth) const {
+ KnownZero = KnownOne = 0; // Don't know anything.
+ if (Depth == 6 || Mask == 0)
+ return; // Limit search depth.
+
+ // The masks are not wide enough to represent this type! Should use APInt.
+ if (Op.getValueType() == MVT::i128)
+ return;
+
+ uint64_t KnownZero2, KnownOne2;
+
+ switch (Op.getOpcode()) {
+ case ISD::Constant:
+ // We know all of the bits for a constant!
+ KnownOne = cast<ConstantSDNode>(Op)->getValue() & Mask;
+ KnownZero = ~KnownOne & Mask;
+ return;
+ case ISD::AND:
+ // If either the LHS or the RHS are Zero, the result is zero.
+ ComputeMaskedBits(Op.getOperand(1), Mask, KnownZero, KnownOne, Depth+1);
+ Mask &= ~KnownZero;
+ ComputeMaskedBits(Op.getOperand(0), Mask, KnownZero2, KnownOne2, Depth+1);
+ assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?");
+ assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?");
+
+ // Output known-1 bits are only known if set in both the LHS & RHS.
+ KnownOne &= KnownOne2;
+ // Output known-0 are known to be clear if zero in either the LHS | RHS.
+ KnownZero |= KnownZero2;
+ return;
+ case ISD::OR:
+ ComputeMaskedBits(Op.getOperand(1), Mask, KnownZero, KnownOne, Depth+1);
+ Mask &= ~KnownOne;
+ ComputeMaskedBits(Op.getOperand(0), Mask, KnownZero2, KnownOne2, Depth+1);
+ assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?");
+ assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?");
+
+ // Output known-0 bits are only known if clear in both the LHS & RHS.
+ KnownZero &= KnownZero2;
+ // Output known-1 are known to be set if set in either the LHS | RHS.
+ KnownOne |= KnownOne2;
+ return;
+ case ISD::XOR: {
+ ComputeMaskedBits(Op.getOperand(1), Mask, KnownZero, KnownOne, Depth+1);
+ ComputeMaskedBits(Op.getOperand(0), Mask, KnownZero2, KnownOne2, Depth+1);
+ assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?");
+ assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?");
+
+ // Output known-0 bits are known if clear or set in both the LHS & RHS.
+ uint64_t KnownZeroOut = (KnownZero & KnownZero2) | (KnownOne & KnownOne2);
+ // Output known-1 are known to be set if set in only one of the LHS, RHS.
+ KnownOne = (KnownZero & KnownOne2) | (KnownOne & KnownZero2);
+ KnownZero = KnownZeroOut;
+ return;
+ }
+ case ISD::SELECT:
+ ComputeMaskedBits(Op.getOperand(2), Mask, KnownZero, KnownOne, Depth+1);
+ ComputeMaskedBits(Op.getOperand(1), Mask, KnownZero2, KnownOne2, Depth+1);
+ assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?");
+ assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?");
+
+ // Only known if known in both the LHS and RHS.
+ KnownOne &= KnownOne2;
+ KnownZero &= KnownZero2;
+ return;
+ case ISD::SELECT_CC:
+ ComputeMaskedBits(Op.getOperand(3), Mask, KnownZero, KnownOne, Depth+1);
+ ComputeMaskedBits(Op.getOperand(2), Mask, KnownZero2, KnownOne2, Depth+1);
+ assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?");
+ assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?");
+
+ // Only known if known in both the LHS and RHS.
+ KnownOne &= KnownOne2;
+ KnownZero &= KnownZero2;
+ return;
+ case ISD::SETCC:
+ // If we know the result of a setcc has the top bits zero, use this info.
+ if (TLI.getSetCCResultContents() == TargetLowering::ZeroOrOneSetCCResult)
+ KnownZero |= (MVT::getIntVTBitMask(Op.getValueType()) ^ 1ULL);
+ return;
+ case ISD::SHL:
+ // (shl X, C1) & C2 == 0 iff (X & C2 >>u C1) == 0
+ if (ConstantSDNode *SA = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
+ ComputeMaskedBits(Op.getOperand(0), Mask >> SA->getValue(),
+ KnownZero, KnownOne, Depth+1);
+ assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?");
+ KnownZero <<= SA->getValue();
+ KnownOne <<= SA->getValue();
+ KnownZero |= (1ULL << SA->getValue())-1; // low bits known zero.
+ }
+ return;
+ case ISD::SRL:
+ // (ushr X, C1) & C2 == 0 iff (-1 >> C1) & C2 == 0
+ if (ConstantSDNode *SA = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
+ MVT::ValueType VT = Op.getValueType();
+ unsigned ShAmt = SA->getValue();
+
+ uint64_t TypeMask = MVT::getIntVTBitMask(VT);
+ ComputeMaskedBits(Op.getOperand(0), (Mask << ShAmt) & TypeMask,
+ KnownZero, KnownOne, Depth+1);
+ assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?");
+ KnownZero &= TypeMask;
+ KnownOne &= TypeMask;
+ KnownZero >>= ShAmt;
+ KnownOne >>= ShAmt;
+
+ uint64_t HighBits = (1ULL << ShAmt)-1;
+ HighBits <<= MVT::getSizeInBits(VT)-ShAmt;
+ KnownZero |= HighBits; // High bits known zero.
+ }
+ return;
+ case ISD::SRA:
+ if (ConstantSDNode *SA = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
+ MVT::ValueType VT = Op.getValueType();
+ unsigned ShAmt = SA->getValue();
+
+ // Compute the new bits that are at the top now.
+ uint64_t TypeMask = MVT::getIntVTBitMask(VT);
+
+ uint64_t InDemandedMask = (Mask << ShAmt) & TypeMask;
+ // If any of the demanded bits are produced by the sign extension, we also
+ // demand the input sign bit.
+ uint64_t HighBits = (1ULL << ShAmt)-1;
+ HighBits <<= MVT::getSizeInBits(VT) - ShAmt;
+ if (HighBits & Mask)
+ InDemandedMask |= MVT::getIntVTSignBit(VT);
+
+ ComputeMaskedBits(Op.getOperand(0), InDemandedMask, KnownZero, KnownOne,
+ Depth+1);
+ assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?");
+ KnownZero &= TypeMask;
+ KnownOne &= TypeMask;
+ KnownZero >>= ShAmt;
+ KnownOne >>= ShAmt;
+
+ // Handle the sign bits.
+ uint64_t SignBit = MVT::getIntVTSignBit(VT);
+ SignBit >>= ShAmt; // Adjust to where it is now in the mask.
+
+ if (KnownZero & SignBit) {
+ KnownZero |= HighBits; // New bits are known zero.
+ } else if (KnownOne & SignBit) {
+ KnownOne |= HighBits; // New bits are known one.
+ }
+ }
+ return;
+ case ISD::SIGN_EXTEND_INREG: {
+ MVT::ValueType EVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
+
+ // Sign extension. Compute the demanded bits in the result that are not
+ // present in the input.
+ uint64_t NewBits = ~MVT::getIntVTBitMask(EVT) & Mask;
+
+ uint64_t InSignBit = MVT::getIntVTSignBit(EVT);
+ int64_t InputDemandedBits = Mask & MVT::getIntVTBitMask(EVT);
+
+ // If the sign extended bits are demanded, we know that the sign
+ // bit is demanded.
+ if (NewBits)
+ InputDemandedBits |= InSignBit;
+
+ ComputeMaskedBits(Op.getOperand(0), InputDemandedBits,
+ KnownZero, KnownOne, Depth+1);
+ assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?");
+
+ // If the sign bit of the input is known set or clear, then we know the
+ // top bits of the result.
+ if (KnownZero & InSignBit) { // Input sign bit known clear
+ KnownZero |= NewBits;
+ KnownOne &= ~NewBits;
+ } else if (KnownOne & InSignBit) { // Input sign bit known set
+ KnownOne |= NewBits;
+ KnownZero &= ~NewBits;
+ } else { // Input sign bit unknown
+ KnownZero &= ~NewBits;
+ KnownOne &= ~NewBits;
+ }
+ return;
+ }
+ case ISD::CTTZ:
+ case ISD::CTLZ:
+ case ISD::CTPOP: {
+ MVT::ValueType VT = Op.getValueType();
+ unsigned LowBits = Log2_32(MVT::getSizeInBits(VT))+1;
+ KnownZero = ~((1ULL << LowBits)-1) & MVT::getIntVTBitMask(VT);
+ KnownOne = 0;
+ return;
+ }
+ case ISD::LOAD: {
+ if (ISD::isZEXTLoad(Op.Val)) {
+ LoadSDNode *LD = cast<LoadSDNode>(Op);
+ MVT::ValueType VT = LD->getLoadedVT();
+ KnownZero |= ~MVT::getIntVTBitMask(VT) & Mask;
+ }
+ return;
+ }
+ case ISD::ZERO_EXTEND: {
+ uint64_t InMask = MVT::getIntVTBitMask(Op.getOperand(0).getValueType());
+ uint64_t NewBits = (~InMask) & Mask;
+ ComputeMaskedBits(Op.getOperand(0), Mask & InMask, KnownZero,
+ KnownOne, Depth+1);
+ KnownZero |= NewBits & Mask;
+ KnownOne &= ~NewBits;
+ return;
+ }
+ case ISD::SIGN_EXTEND: {
+ MVT::ValueType InVT = Op.getOperand(0).getValueType();
+ unsigned InBits = MVT::getSizeInBits(InVT);
+ uint64_t InMask = MVT::getIntVTBitMask(InVT);
+ uint64_t InSignBit = 1ULL << (InBits-1);
+ uint64_t NewBits = (~InMask) & Mask;
+ uint64_t InDemandedBits = Mask & InMask;
+
+ // If any of the sign extended bits are demanded, we know that the sign
+ // bit is demanded.
+ if (NewBits & Mask)
+ InDemandedBits |= InSignBit;
+
+ ComputeMaskedBits(Op.getOperand(0), InDemandedBits, KnownZero,
+ KnownOne, Depth+1);
+ // If the sign bit is known zero or one, the top bits match.
+ if (KnownZero & InSignBit) {
+ KnownZero |= NewBits;
+ KnownOne &= ~NewBits;
+ } else if (KnownOne & InSignBit) {
+ KnownOne |= NewBits;
+ KnownZero &= ~NewBits;
+ } else { // Otherwise, top bits aren't known.
+ KnownOne &= ~NewBits;
+ KnownZero &= ~NewBits;
+ }
+ return;
+ }
+ case ISD::ANY_EXTEND: {
+ MVT::ValueType VT = Op.getOperand(0).getValueType();
+ ComputeMaskedBits(Op.getOperand(0), Mask & MVT::getIntVTBitMask(VT),
+ KnownZero, KnownOne, Depth+1);
+ return;
+ }
+ case ISD::TRUNCATE: {
+ ComputeMaskedBits(Op.getOperand(0), Mask, KnownZero, KnownOne, Depth+1);
+ assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?");
+ uint64_t OutMask = MVT::getIntVTBitMask(Op.getValueType());
+ KnownZero &= OutMask;
+ KnownOne &= OutMask;
+ break;
+ }
+ case ISD::AssertZext: {
+ MVT::ValueType VT = cast<VTSDNode>(Op.getOperand(1))->getVT();
+ uint64_t InMask = MVT::getIntVTBitMask(VT);
+ ComputeMaskedBits(Op.getOperand(0), Mask & InMask, KnownZero,
+ KnownOne, Depth+1);
+ KnownZero |= (~InMask) & Mask;
+ return;
+ }
+ case ISD::ADD: {
+ // If either the LHS or the RHS are Zero, the result is zero.
+ ComputeMaskedBits(Op.getOperand(1), Mask, KnownZero, KnownOne, Depth+1);
+ ComputeMaskedBits(Op.getOperand(0), Mask, KnownZero2, KnownOne2, Depth+1);
+ assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?");
+ assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?");
+
+ // Output known-0 bits are known if clear or set in both the low clear bits
+ // common to both LHS & RHS. For example, 8+(X<<3) is known to have the
+ // low 3 bits clear.
+ uint64_t KnownZeroOut = std::min(CountTrailingZeros_64(~KnownZero),
+ CountTrailingZeros_64(~KnownZero2));
+
+ KnownZero = (1ULL << KnownZeroOut) - 1;
+ KnownOne = 0;
+ return;
+ }
+ case ISD::SUB: {
+ ConstantSDNode *CLHS = dyn_cast<ConstantSDNode>(Op.getOperand(0));
+ if (!CLHS) return;
+
+ // We know that the top bits of C-X are clear if X contains less bits
+ // than C (i.e. no wrap-around can happen). For example, 20-X is
+ // positive if we can prove that X is >= 0 and < 16.
+ MVT::ValueType VT = CLHS->getValueType(0);
+ if ((CLHS->getValue() & MVT::getIntVTSignBit(VT)) == 0) { // sign bit clear
+ unsigned NLZ = CountLeadingZeros_64(CLHS->getValue()+1);
+ uint64_t MaskV = (1ULL << (63-NLZ))-1; // NLZ can't be 64 with no sign bit
+ MaskV = ~MaskV & MVT::getIntVTBitMask(VT);
+ ComputeMaskedBits(Op.getOperand(1), MaskV, KnownZero, KnownOne, Depth+1);
+
+ // If all of the MaskV bits are known to be zero, then we know the output
+ // top bits are zero, because we now know that the output is from [0-C].
+ if ((KnownZero & MaskV) == MaskV) {
+ unsigned NLZ2 = CountLeadingZeros_64(CLHS->getValue());
+ KnownZero = ~((1ULL << (64-NLZ2))-1) & Mask; // Top bits known zero.
+ KnownOne = 0; // No one bits known.
+ } else {
+ KnownZero = KnownOne = 0; // Otherwise, nothing known.
+ }
+ }
+ return;
+ }
+ default:
+ // Allow the target to implement this method for its nodes.
+ if (Op.getOpcode() >= ISD::BUILTIN_OP_END) {
+ case ISD::INTRINSIC_WO_CHAIN:
+ case ISD::INTRINSIC_W_CHAIN:
+ case ISD::INTRINSIC_VOID:
+ TLI.computeMaskedBitsForTargetNode(Op, Mask, KnownZero, KnownOne, *this);
+ }
+ return;
+ }
+}
+
+/// ComputeNumSignBits - Return the number of times the sign bit of the
+/// register is replicated into the other bits. We know that at least 1 bit
+/// is always equal to the sign bit (itself), but other cases can give us
+/// information. For example, immediately after an "SRA X, 2", we know that
+/// the top 3 bits are all equal to each other, so we return 3.
+unsigned SelectionDAG::ComputeNumSignBits(SDOperand Op, unsigned Depth) const{
+ MVT::ValueType VT = Op.getValueType();
+ assert(MVT::isInteger(VT) && "Invalid VT!");
+ unsigned VTBits = MVT::getSizeInBits(VT);
+ unsigned Tmp, Tmp2;
+
+ if (Depth == 6)
+ return 1; // Limit search depth.
+
+ switch (Op.getOpcode()) {
+ default: break;
+ case ISD::AssertSext:
+ Tmp = MVT::getSizeInBits(cast<VTSDNode>(Op.getOperand(1))->getVT());
+ return VTBits-Tmp+1;
+ case ISD::AssertZext:
+ Tmp = MVT::getSizeInBits(cast<VTSDNode>(Op.getOperand(1))->getVT());
+ return VTBits-Tmp;
+
+ case ISD::Constant: {
+ uint64_t Val = cast<ConstantSDNode>(Op)->getValue();
+ // If negative, invert the bits, then look at it.
+ if (Val & MVT::getIntVTSignBit(VT))
+ Val = ~Val;
+
+ // Shift the bits so they are the leading bits in the int64_t.
+ Val <<= 64-VTBits;
+
+ // Return # leading zeros. We use 'min' here in case Val was zero before
+ // shifting. We don't want to return '64' as for an i32 "0".
+ return std::min(VTBits, CountLeadingZeros_64(Val));
+ }
+
+ case ISD::SIGN_EXTEND:
+ Tmp = VTBits-MVT::getSizeInBits(Op.getOperand(0).getValueType());
+ return ComputeNumSignBits(Op.getOperand(0), Depth+1) + Tmp;
+
+ case ISD::SIGN_EXTEND_INREG:
+ // Max of the input and what this extends.
+ Tmp = MVT::getSizeInBits(cast<VTSDNode>(Op.getOperand(1))->getVT());
+ Tmp = VTBits-Tmp+1;
+
+ Tmp2 = ComputeNumSignBits(Op.getOperand(0), Depth+1);
+ return std::max(Tmp, Tmp2);
+
+ case ISD::SRA:
+ Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
+ // SRA X, C -> adds C sign bits.
+ if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
+ Tmp += C->getValue();
+ if (Tmp > VTBits) Tmp = VTBits;
+ }
+ return Tmp;
+ case ISD::SHL:
+ if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
+ // shl destroys sign bits.
+ Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
+ if (C->getValue() >= VTBits || // Bad shift.
+ C->getValue() >= Tmp) break; // Shifted all sign bits out.
+ return Tmp - C->getValue();
+ }
+ break;
+ case ISD::AND:
+ case ISD::OR:
+ case ISD::XOR: // NOT is handled here.
+ // Logical binary ops preserve the number of sign bits.
+ Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
+ if (Tmp == 1) return 1; // Early out.
+ Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth+1);
+ return std::min(Tmp, Tmp2);
+
+ case ISD::SELECT:
+ Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
+ if (Tmp == 1) return 1; // Early out.
+ Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth+1);
+ return std::min(Tmp, Tmp2);
+
+ case ISD::SETCC:
+ // If setcc returns 0/-1, all bits are sign bits.
+ if (TLI.getSetCCResultContents() ==
+ TargetLowering::ZeroOrNegativeOneSetCCResult)
+ return VTBits;
+ break;
+ case ISD::ROTL:
+ case ISD::ROTR:
+ if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
+ unsigned RotAmt = C->getValue() & (VTBits-1);
+
+ // Handle rotate right by N like a rotate left by 32-N.
+ if (Op.getOpcode() == ISD::ROTR)
+ RotAmt = (VTBits-RotAmt) & (VTBits-1);
+
+ // If we aren't rotating out all of the known-in sign bits, return the
+ // number that are left. This handles rotl(sext(x), 1) for example.
+ Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
+ if (Tmp > RotAmt+1) return Tmp-RotAmt;
+ }
+ break;
+ case ISD::ADD:
+ // Add can have at most one carry bit. Thus we know that the output
+ // is, at worst, one more bit than the inputs.
+ Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
+ if (Tmp == 1) return 1; // Early out.
+
+ // Special case decrementing a value (ADD X, -1):
+ if (ConstantSDNode *CRHS = dyn_cast<ConstantSDNode>(Op.getOperand(0)))
+ if (CRHS->isAllOnesValue()) {
+ uint64_t KnownZero, KnownOne;
+ uint64_t Mask = MVT::getIntVTBitMask(VT);
+ ComputeMaskedBits(Op.getOperand(0), Mask, KnownZero, KnownOne, Depth+1);
+
+ // If the input is known to be 0 or 1, the output is 0/-1, which is all
+ // sign bits set.
+ if ((KnownZero|1) == Mask)
+ return VTBits;
+
+ // If we are subtracting one from a positive number, there is no carry
+ // out of the result.
+ if (KnownZero & MVT::getIntVTSignBit(VT))
+ return Tmp;
+ }
+
+ Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth+1);
+ if (Tmp2 == 1) return 1;
+ return std::min(Tmp, Tmp2)-1;
+ break;
+
+ case ISD::SUB:
+ Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth+1);
+ if (Tmp2 == 1) return 1;
+
+ // Handle NEG.
+ if (ConstantSDNode *CLHS = dyn_cast<ConstantSDNode>(Op.getOperand(0)))
+ if (CLHS->getValue() == 0) {
+ uint64_t KnownZero, KnownOne;
+ uint64_t Mask = MVT::getIntVTBitMask(VT);
+ ComputeMaskedBits(Op.getOperand(1), Mask, KnownZero, KnownOne, Depth+1);
+ // If the input is known to be 0 or 1, the output is 0/-1, which is all
+ // sign bits set.
+ if ((KnownZero|1) == Mask)
+ return VTBits;
+
+ // If the input is known to be positive (the sign bit is known clear),
+ // the output of the NEG has the same number of sign bits as the input.
+ if (KnownZero & MVT::getIntVTSignBit(VT))
+ return Tmp2;
+
+ // Otherwise, we treat this like a SUB.
+ }
+
+ // Sub can have at most one carry bit. Thus we know that the output
+ // is, at worst, one more bit than the inputs.
+ Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
+ if (Tmp == 1) return 1; // Early out.
+ return std::min(Tmp, Tmp2)-1;
+ break;
+ case ISD::TRUNCATE:
+ // FIXME: it's tricky to do anything useful for this, but it is an important
+ // case for targets like X86.
+ break;
+ }
+
+ // Handle LOADX separately here. EXTLOAD case will fallthrough.
+ if (Op.getOpcode() == ISD::LOAD) {
+ LoadSDNode *LD = cast<LoadSDNode>(Op);
+ unsigned ExtType = LD->getExtensionType();
+ switch (ExtType) {
+ default: break;
+ case ISD::SEXTLOAD: // '17' bits known
+ Tmp = MVT::getSizeInBits(LD->getLoadedVT());
+ return VTBits-Tmp+1;
+ case ISD::ZEXTLOAD: // '16' bits known
+ Tmp = MVT::getSizeInBits(LD->getLoadedVT());
+ return VTBits-Tmp;
+ }
+ }
+
+ // Allow the target to implement this method for its nodes.
+ if (Op.getOpcode() >= ISD::BUILTIN_OP_END ||
+ Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
+ Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
+ Op.getOpcode() == ISD::INTRINSIC_VOID) {
+ unsigned NumBits = TLI.ComputeNumSignBitsForTargetNode(Op, Depth);
+ if (NumBits > 1) return NumBits;
+ }
+
+ // Finally, if we can prove that the top bits of the result are 0's or 1's,
+ // use this information.
+ uint64_t KnownZero, KnownOne;
+ uint64_t Mask = MVT::getIntVTBitMask(VT);
+ ComputeMaskedBits(Op, Mask, KnownZero, KnownOne, Depth);
+
+ uint64_t SignBit = MVT::getIntVTSignBit(VT);
+ if (KnownZero & SignBit) { // SignBit is 0
+ Mask = KnownZero;
+ } else if (KnownOne & SignBit) { // SignBit is 1;
+ Mask = KnownOne;
+ } else {
+ // Nothing known.
+ return 1;
+ }
+
+ // Okay, we know that the sign bit in Mask is set. Use CLZ to determine
+ // the number of identical bits in the top of the input value.
+ Mask ^= ~0ULL;
+ Mask <<= 64-VTBits;
+ // Return # leading zeros. We use 'min' here in case Val was zero before
+ // shifting. We don't want to return '64' as for an i32 "0".
+ return std::min(VTBits, CountLeadingZeros_64(Mask));
+}
+
+
+/// getNode - Gets or creates the specified node.
+///
+SDOperand SelectionDAG::getNode(unsigned Opcode, MVT::ValueType VT) {
+ FoldingSetNodeID ID;
+ AddNodeIDNode(ID, Opcode, getVTList(VT), 0, 0);
+ void *IP = 0;
+ if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
+ return SDOperand(E, 0);
+ SDNode *N = new SDNode(Opcode, SDNode::getSDVTList(VT));
+ CSEMap.InsertNode(N, IP);
+
+ AllNodes.push_back(N);
+ return SDOperand(N, 0);
+}
+
+SDOperand SelectionDAG::getNode(unsigned Opcode, MVT::ValueType VT,
+ SDOperand Operand) {
+ unsigned Tmp1;
+ // Constant fold unary operations with an integer constant operand.
+ if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Operand.Val)) {
+ uint64_t Val = C->getValue();
+ switch (Opcode) {
+ default: break;
+ case ISD::SIGN_EXTEND: return getConstant(C->getSignExtended(), VT);
+ case ISD::ANY_EXTEND:
+ case ISD::ZERO_EXTEND: return getConstant(Val, VT);
+ case ISD::TRUNCATE: return getConstant(Val, VT);
+ case ISD::SINT_TO_FP: return getConstantFP(C->getSignExtended(), VT);
+ case ISD::UINT_TO_FP: return getConstantFP(C->getValue(), VT);
+ case ISD::BIT_CONVERT:
+ if (VT == MVT::f32 && C->getValueType(0) == MVT::i32)
+ return getConstantFP(BitsToFloat(Val), VT);
+ else if (VT == MVT::f64 && C->getValueType(0) == MVT::i64)
+ return getConstantFP(BitsToDouble(Val), VT);
+ break;
+ case ISD::BSWAP:
+ switch(VT) {
+ default: assert(0 && "Invalid bswap!"); break;
+ case MVT::i16: return getConstant(ByteSwap_16((unsigned short)Val), VT);
+ case MVT::i32: return getConstant(ByteSwap_32((unsigned)Val), VT);
+ case MVT::i64: return getConstant(ByteSwap_64(Val), VT);
+ }
+ break;
+ case ISD::CTPOP:
+ switch(VT) {
+ default: assert(0 && "Invalid ctpop!"); break;
+ case MVT::i1: return getConstant(Val != 0, VT);
+ case MVT::i8:
+ Tmp1 = (unsigned)Val & 0xFF;
+ return getConstant(CountPopulation_32(Tmp1), VT);
+ case MVT::i16:
+ Tmp1 = (unsigned)Val & 0xFFFF;
+ return getConstant(CountPopulation_32(Tmp1), VT);
+ case MVT::i32:
+ return getConstant(CountPopulation_32((unsigned)Val), VT);
+ case MVT::i64:
+ return getConstant(CountPopulation_64(Val), VT);
+ }
+ case ISD::CTLZ:
+ switch(VT) {
+ default: assert(0 && "Invalid ctlz!"); break;
+ case MVT::i1: return getConstant(Val == 0, VT);
+ case MVT::i8:
+ Tmp1 = (unsigned)Val & 0xFF;
+ return getConstant(CountLeadingZeros_32(Tmp1)-24, VT);
+ case MVT::i16:
+ Tmp1 = (unsigned)Val & 0xFFFF;
+ return getConstant(CountLeadingZeros_32(Tmp1)-16, VT);
+ case MVT::i32:
+ return getConstant(CountLeadingZeros_32((unsigned)Val), VT);
+ case MVT::i64:
+ return getConstant(CountLeadingZeros_64(Val), VT);
+ }
+ case ISD::CTTZ:
+ switch(VT) {
+ default: assert(0 && "Invalid cttz!"); break;
+ case MVT::i1: return getConstant(Val == 0, VT);
+ case MVT::i8:
+ Tmp1 = (unsigned)Val | 0x100;
+ return getConstant(CountTrailingZeros_32(Tmp1), VT);
+ case MVT::i16:
+ Tmp1 = (unsigned)Val | 0x10000;
+ return getConstant(CountTrailingZeros_32(Tmp1), VT);
+ case MVT::i32:
+ return getConstant(CountTrailingZeros_32((unsigned)Val), VT);
+ case MVT::i64:
+ return getConstant(CountTrailingZeros_64(Val), VT);
+ }
+ }
+ }
+
+ // Constant fold unary operations with an floating point constant operand.
+ if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Operand.Val))
+ switch (Opcode) {
+ case ISD::FNEG:
+ return getConstantFP(-C->getValue(), VT);
+ case ISD::FABS:
+ return getConstantFP(fabs(C->getValue()), VT);
+ case ISD::FP_ROUND:
+ case ISD::FP_EXTEND:
+ return getConstantFP(C->getValue(), VT);
+ case ISD::FP_TO_SINT:
+ return getConstant((int64_t)C->getValue(), VT);
+ case ISD::FP_TO_UINT:
+ return getConstant((uint64_t)C->getValue(), VT);
+ case ISD::BIT_CONVERT:
+ if (VT == MVT::i32 && C->getValueType(0) == MVT::f32)
+ return getConstant(FloatToBits(C->getValue()), VT);
+ else if (VT == MVT::i64 && C->getValueType(0) == MVT::f64)
+ return getConstant(DoubleToBits(C->getValue()), VT);
+ break;
+ }
+
+ unsigned OpOpcode = Operand.Val->getOpcode();
+ switch (Opcode) {
+ case ISD::TokenFactor:
+ return Operand; // Factor of one node? No factor.
+ case ISD::FP_ROUND:
+ case ISD::FP_EXTEND:
+ assert(MVT::isFloatingPoint(VT) &&
+ MVT::isFloatingPoint(Operand.getValueType()) && "Invalid FP cast!");
+ break;
+ case ISD::SIGN_EXTEND:
+ assert(MVT::isInteger(VT) && MVT::isInteger(Operand.getValueType()) &&
+ "Invalid SIGN_EXTEND!");
+ if (Operand.getValueType() == VT) return Operand; // noop extension
+ assert(Operand.getValueType() < VT && "Invalid sext node, dst < src!");
+ if (OpOpcode == ISD::SIGN_EXTEND || OpOpcode == ISD::ZERO_EXTEND)
+ return getNode(OpOpcode, VT, Operand.Val->getOperand(0));
+ break;
+ case ISD::ZERO_EXTEND:
+ assert(MVT::isInteger(VT) && MVT::isInteger(Operand.getValueType()) &&
+ "Invalid ZERO_EXTEND!");
+ if (Operand.getValueType() == VT) return Operand; // noop extension
+ assert(Operand.getValueType() < VT && "Invalid zext node, dst < src!");
+ if (OpOpcode == ISD::ZERO_EXTEND) // (zext (zext x)) -> (zext x)
+ return getNode(ISD::ZERO_EXTEND, VT, Operand.Val->getOperand(0));
+ break;
+ case ISD::ANY_EXTEND:
+ assert(MVT::isInteger(VT) && MVT::isInteger(Operand.getValueType()) &&
+ "Invalid ANY_EXTEND!");
+ if (Operand.getValueType() == VT) return Operand; // noop extension
+ assert(Operand.getValueType() < VT && "Invalid anyext node, dst < src!");
+ if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND)
+ // (ext (zext x)) -> (zext x) and (ext (sext x)) -> (sext x)
+ return getNode(OpOpcode, VT, Operand.Val->getOperand(0));
+ break;
+ case ISD::TRUNCATE:
+ assert(MVT::isInteger(VT) && MVT::isInteger(Operand.getValueType()) &&
+ "Invalid TRUNCATE!");
+ if (Operand.getValueType() == VT) return Operand; // noop truncate
+ assert(Operand.getValueType() > VT && "Invalid truncate node, src < dst!");
+ if (OpOpcode == ISD::TRUNCATE)
+ return getNode(ISD::TRUNCATE, VT, Operand.Val->getOperand(0));
+ else if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND ||
+ OpOpcode == ISD::ANY_EXTEND) {
+ // If the source is smaller than the dest, we still need an extend.
+ if (Operand.Val->getOperand(0).getValueType() < VT)
+ return getNode(OpOpcode, VT, Operand.Val->getOperand(0));
+ else if (Operand.Val->getOperand(0).getValueType() > VT)
+ return getNode(ISD::TRUNCATE, VT, Operand.Val->getOperand(0));
+ else
+ return Operand.Val->getOperand(0);
+ }
+ break;
+ case ISD::BIT_CONVERT:
+ // Basic sanity checking.
+ assert(MVT::getSizeInBits(VT) == MVT::getSizeInBits(Operand.getValueType())
+ && "Cannot BIT_CONVERT between types of different sizes!");
+ if (VT == Operand.getValueType()) return Operand; // noop conversion.
+ if (OpOpcode == ISD::BIT_CONVERT) // bitconv(bitconv(x)) -> bitconv(x)
+ return getNode(ISD::BIT_CONVERT, VT, Operand.getOperand(0));
+ if (OpOpcode == ISD::UNDEF)
+ return getNode(ISD::UNDEF, VT);
+ break;
+ case ISD::SCALAR_TO_VECTOR:
+ assert(MVT::isVector(VT) && !MVT::isVector(Operand.getValueType()) &&
+ MVT::getVectorElementType(VT) == Operand.getValueType() &&
+ "Illegal SCALAR_TO_VECTOR node!");
+ break;
+ case ISD::FNEG:
+ if (OpOpcode == ISD::FSUB) // -(X-Y) -> (Y-X)
+ return getNode(ISD::FSUB, VT, Operand.Val->getOperand(1),
+ Operand.Val->getOperand(0));
+ if (OpOpcode == ISD::FNEG) // --X -> X
+ return Operand.Val->getOperand(0);
+ break;
+ case ISD::FABS:
+ if (OpOpcode == ISD::FNEG) // abs(-X) -> abs(X)
+ return getNode(ISD::FABS, VT, Operand.Val->getOperand(0));
+ break;
+ }
+
+ SDNode *N;
+ SDVTList VTs = getVTList(VT);
+ if (VT != MVT::Flag) { // Don't CSE flag producing nodes
+ FoldingSetNodeID ID;
+ SDOperand Ops[1] = { Operand };
+ AddNodeIDNode(ID, Opcode, VTs, Ops, 1);
+ void *IP = 0;
+ if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
+ return SDOperand(E, 0);
+ N = new UnarySDNode(Opcode, VTs, Operand);
+ CSEMap.InsertNode(N, IP);
+ } else {
+ N = new UnarySDNode(Opcode, VTs, Operand);
+ }
+ AllNodes.push_back(N);
+ return SDOperand(N, 0);
+}
+
+
+
+SDOperand SelectionDAG::getNode(unsigned Opcode, MVT::ValueType VT,
+ SDOperand N1, SDOperand N2) {
+#ifndef NDEBUG
+ switch (Opcode) {
+ case ISD::TokenFactor:
+ assert(VT == MVT::Other && N1.getValueType() == MVT::Other &&
+ N2.getValueType() == MVT::Other && "Invalid token factor!");
+ break;
+ case ISD::AND:
+ case ISD::OR:
+ case ISD::XOR:
+ case ISD::UDIV:
+ case ISD::UREM:
+ case ISD::MULHU:
+ case ISD::MULHS:
+ assert(MVT::isInteger(VT) && "This operator does not apply to FP types!");
+ // fall through
+ case ISD::ADD:
+ case ISD::SUB:
+ case ISD::MUL:
+ case ISD::SDIV:
+ case ISD::SREM:
+ assert(MVT::isInteger(N1.getValueType()) && "Should use F* for FP ops");
+ // fall through.
+ case ISD::FADD:
+ case ISD::FSUB:
+ case ISD::FMUL:
+ case ISD::FDIV:
+ case ISD::FREM:
+ assert(N1.getValueType() == N2.getValueType() &&
+ N1.getValueType() == VT && "Binary operator types must match!");
+ break;
+ case ISD::FCOPYSIGN: // N1 and result must match. N1/N2 need not match.
+ assert(N1.getValueType() == VT &&
+ MVT::isFloatingPoint(N1.getValueType()) &&
+ MVT::isFloatingPoint(N2.getValueType()) &&
+ "Invalid FCOPYSIGN!");
+ break;
+ case ISD::SHL:
+ case ISD::SRA:
+ case ISD::SRL:
+ case ISD::ROTL:
+ case ISD::ROTR:
+ assert(VT == N1.getValueType() &&
+ "Shift operators return type must be the same as their first arg");
+ assert(MVT::isInteger(VT) && MVT::isInteger(N2.getValueType()) &&
+ VT != MVT::i1 && "Shifts only work on integers");
+ break;
+ case ISD::FP_ROUND_INREG: {
+ MVT::ValueType EVT = cast<VTSDNode>(N2)->getVT();
+ assert(VT == N1.getValueType() && "Not an inreg round!");
+ assert(MVT::isFloatingPoint(VT) && MVT::isFloatingPoint(EVT) &&
+ "Cannot FP_ROUND_INREG integer types");
+ assert(EVT <= VT && "Not rounding down!");
+ break;
+ }
+ case ISD::AssertSext:
+ case ISD::AssertZext:
+ case ISD::SIGN_EXTEND_INREG: {
+ MVT::ValueType EVT = cast<VTSDNode>(N2)->getVT();
+ assert(VT == N1.getValueType() && "Not an inreg extend!");
+ assert(MVT::isInteger(VT) && MVT::isInteger(EVT) &&
+ "Cannot *_EXTEND_INREG FP types");
+ assert(EVT <= VT && "Not extending!");
+ }
+
+ default: break;
+ }
+#endif
+
+ ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.Val);
+ ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2.Val);
+ if (N1C) {
+ if (Opcode == ISD::SIGN_EXTEND_INREG) {
+ int64_t Val = N1C->getValue();
+ unsigned FromBits = MVT::getSizeInBits(cast<VTSDNode>(N2)->getVT());
+ Val <<= 64-FromBits;
+ Val >>= 64-FromBits;
+ return getConstant(Val, VT);
+ }
+
+ if (N2C) {
+ uint64_t C1 = N1C->getValue(), C2 = N2C->getValue();
+ switch (Opcode) {
+ case ISD::ADD: return getConstant(C1 + C2, VT);
+ case ISD::SUB: return getConstant(C1 - C2, VT);
+ case ISD::MUL: return getConstant(C1 * C2, VT);
+ case ISD::UDIV:
+ if (C2) return getConstant(C1 / C2, VT);
+ break;
+ case ISD::UREM :
+ if (C2) return getConstant(C1 % C2, VT);
+ break;
+ case ISD::SDIV :
+ if (C2) return getConstant(N1C->getSignExtended() /
+ N2C->getSignExtended(), VT);
+ break;
+ case ISD::SREM :
+ if (C2) return getConstant(N1C->getSignExtended() %
+ N2C->getSignExtended(), VT);
+ break;
+ case ISD::AND : return getConstant(C1 & C2, VT);
+ case ISD::OR : return getConstant(C1 | C2, VT);
+ case ISD::XOR : return getConstant(C1 ^ C2, VT);
+ case ISD::SHL : return getConstant(C1 << C2, VT);
+ case ISD::SRL : return getConstant(C1 >> C2, VT);
+ case ISD::SRA : return getConstant(N1C->getSignExtended() >>(int)C2, VT);
+ case ISD::ROTL :
+ return getConstant((C1 << C2) | (C1 >> (MVT::getSizeInBits(VT) - C2)),
+ VT);
+ case ISD::ROTR :
+ return getConstant((C1 >> C2) | (C1 << (MVT::getSizeInBits(VT) - C2)),
+ VT);
+ default: break;
+ }
+ } else { // Cannonicalize constant to RHS if commutative
+ if (isCommutativeBinOp(Opcode)) {
+ std::swap(N1C, N2C);
+ std::swap(N1, N2);
+ }
+ }
+ }
+
+ ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1.Val);
+ ConstantFPSDNode *N2CFP = dyn_cast<ConstantFPSDNode>(N2.Val);
+ if (N1CFP) {
+ if (N2CFP) {
+ double C1 = N1CFP->getValue(), C2 = N2CFP->getValue();
+ switch (Opcode) {
+ case ISD::FADD: return getConstantFP(C1 + C2, VT);
+ case ISD::FSUB: return getConstantFP(C1 - C2, VT);
+ case ISD::FMUL: return getConstantFP(C1 * C2, VT);
+ case ISD::FDIV:
+ if (C2) return getConstantFP(C1 / C2, VT);
+ break;
+ case ISD::FREM :
+ if (C2) return getConstantFP(fmod(C1, C2), VT);
+ break;
+ case ISD::FCOPYSIGN: {
+ union {
+ double F;
+ uint64_t I;
+ } u1;
+ u1.F = C1;
+ if (int64_t(DoubleToBits(C2)) < 0) // Sign bit of RHS set?
+ u1.I |= 1ULL << 63; // Set the sign bit of the LHS.
+ else
+ u1.I &= (1ULL << 63)-1; // Clear the sign bit of the LHS.
+ return getConstantFP(u1.F, VT);
+ }
+ default: break;
+ }
+ } else { // Cannonicalize constant to RHS if commutative
+ if (isCommutativeBinOp(Opcode)) {
+ std::swap(N1CFP, N2CFP);
+ std::swap(N1, N2);
+ }
+ }
+ }
+
+ // Canonicalize an UNDEF to the RHS, even over a constant.
+ if (N1.getOpcode() == ISD::UNDEF) {
+ if (isCommutativeBinOp(Opcode)) {
+ std::swap(N1, N2);
+ } else {
+ switch (Opcode) {
+ case ISD::FP_ROUND_INREG:
+ case ISD::SIGN_EXTEND_INREG:
+ case ISD::SUB:
+ case ISD::FSUB:
+ case ISD::FDIV:
+ case ISD::FREM:
+ case ISD::SRA:
+ return N1; // fold op(undef, arg2) -> undef
+ case ISD::UDIV:
+ case ISD::SDIV:
+ case ISD::UREM:
+ case ISD::SREM:
+ case ISD::SRL:
+ case ISD::SHL:
+ if (!MVT::isVector(VT))
+ return getConstant(0, VT); // fold op(undef, arg2) -> 0
+ // For vectors, we can't easily build an all zero vector, just return
+ // the LHS.
+ return N2;
+ }
+ }
+ }
+
+ // Fold a bunch of operators when the RHS is undef.
+ if (N2.getOpcode() == ISD::UNDEF) {
+ switch (Opcode) {
+ case ISD::ADD:
+ case ISD::ADDC:
+ case ISD::ADDE:
+ case ISD::SUB:
+ case ISD::FADD:
+ case ISD::FSUB:
+ case ISD::FMUL:
+ case ISD::FDIV:
+ case ISD::FREM:
+ case ISD::UDIV:
+ case ISD::SDIV:
+ case ISD::UREM:
+ case ISD::SREM:
+ case ISD::XOR:
+ return N2; // fold op(arg1, undef) -> undef
+ case ISD::MUL:
+ case ISD::AND:
+ case ISD::SRL:
+ case ISD::SHL:
+ if (!MVT::isVector(VT))
+ return getConstant(0, VT); // fold op(arg1, undef) -> 0
+ // For vectors, we can't easily build an all zero vector, just return
+ // the LHS.
+ return N1;
+ case ISD::OR:
+ if (!MVT::isVector(VT))
+ return getConstant(MVT::getIntVTBitMask(VT), VT);
+ // For vectors, we can't easily build an all one vector, just return
+ // the LHS.
+ return N1;
+ case ISD::SRA:
+ return N1;
+ }
+ }
+
+ // Fold operations.
+ switch (Opcode) {
+ case ISD::TokenFactor:
+ // Fold trivial token factors.
+ if (N1.getOpcode() == ISD::EntryToken) return N2;
+ if (N2.getOpcode() == ISD::EntryToken) return N1;
+ break;
+
+ case ISD::AND:
+ // (X & 0) -> 0. This commonly occurs when legalizing i64 values, so it's
+ // worth handling here.
+ if (N2C && N2C->getValue() == 0)
+ return N2;
+ break;
+ case ISD::OR:
+ case ISD::XOR:
+ // (X ^| 0) -> X. This commonly occurs when legalizing i64 values, so it's
+ // worth handling here.
+ if (N2C && N2C->getValue() == 0)
+ return N1;
+ break;
+ case ISD::FP_ROUND_INREG:
+ if (cast<VTSDNode>(N2)->getVT() == VT) return N1; // Not actually rounding.
+ break;
+ case ISD::SIGN_EXTEND_INREG: {
+ MVT::ValueType EVT = cast<VTSDNode>(N2)->getVT();
+ if (EVT == VT) return N1; // Not actually extending
+ break;
+ }
+ case ISD::EXTRACT_VECTOR_ELT:
+ assert(N2C && "Bad EXTRACT_VECTOR_ELT!");
+
+ // EXTRACT_VECTOR_ELT of CONCAT_VECTORS is often formed while lowering is
+ // expanding copies of large vectors from registers.
+ if (N1.getOpcode() == ISD::CONCAT_VECTORS &&
+ N1.getNumOperands() > 0) {
+ unsigned Factor =
+ MVT::getVectorNumElements(N1.getOperand(0).getValueType());
+ return getNode(ISD::EXTRACT_VECTOR_ELT, VT,
+ N1.getOperand(N2C->getValue() / Factor),
+ getConstant(N2C->getValue() % Factor, N2.getValueType()));
+ }
+
+ // EXTRACT_VECTOR_ELT of BUILD_VECTOR is often formed while lowering is
+ // expanding large vector constants.
+ if (N1.getOpcode() == ISD::BUILD_VECTOR)
+ return N1.getOperand(N2C->getValue());
+
+ // EXTRACT_VECTOR_ELT of INSERT_VECTOR_ELT is often formed when vector
+ // operations are lowered to scalars.
+ if (N1.getOpcode() == ISD::INSERT_VECTOR_ELT)
+ if (ConstantSDNode *IEC = dyn_cast<ConstantSDNode>(N1.getOperand(2))) {
+ if (IEC == N2C)
+ return N1.getOperand(1);
+ else
+ return getNode(ISD::EXTRACT_VECTOR_ELT, VT, N1.getOperand(0), N2);
+ }
+ break;
+ case ISD::EXTRACT_ELEMENT:
+ assert(N2C && (unsigned)N2C->getValue() < 2 && "Bad EXTRACT_ELEMENT!");
+
+ // EXTRACT_ELEMENT of BUILD_PAIR is often formed while legalize is expanding
+ // 64-bit integers into 32-bit parts. Instead of building the extract of
+ // the BUILD_PAIR, only to have legalize rip it apart, just do it now.
+ if (N1.getOpcode() == ISD::BUILD_PAIR)
+ return N1.getOperand(N2C->getValue());
+
+ // EXTRACT_ELEMENT of a constant int is also very common.
+ if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N1)) {
+ unsigned Shift = MVT::getSizeInBits(VT) * N2C->getValue();
+ return getConstant(C->getValue() >> Shift, VT);
+ }
+ break;
+
+ // FIXME: figure out how to safely handle things like
+ // int foo(int x) { return 1 << (x & 255); }
+ // int bar() { return foo(256); }
+#if 0
+ case ISD::SHL:
+ case ISD::SRL:
+ case ISD::SRA:
+ if (N2.getOpcode() == ISD::SIGN_EXTEND_INREG &&
+ cast<VTSDNode>(N2.getOperand(1))->getVT() != MVT::i1)
+ return getNode(Opcode, VT, N1, N2.getOperand(0));
+ else if (N2.getOpcode() == ISD::AND)
+ if (ConstantSDNode *AndRHS = dyn_cast<ConstantSDNode>(N2.getOperand(1))) {
+ // If the and is only masking out bits that cannot effect the shift,
+ // eliminate the and.
+ unsigned NumBits = MVT::getSizeInBits(VT);
+ if ((AndRHS->getValue() & (NumBits-1)) == NumBits-1)
+ return getNode(Opcode, VT, N1, N2.getOperand(0));
+ }
+ break;
+#endif
+ }
+
+ // Memoize this node if possible.
+ SDNode *N;
+ SDVTList VTs = getVTList(VT);
+ if (VT != MVT::Flag) {
+ SDOperand Ops[] = { N1, N2 };
+ FoldingSetNodeID ID;
+ AddNodeIDNode(ID, Opcode, VTs, Ops, 2);
+ void *IP = 0;
+ if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
+ return SDOperand(E, 0);
+ N = new BinarySDNode(Opcode, VTs, N1, N2);
+ CSEMap.InsertNode(N, IP);
+ } else {
+ N = new BinarySDNode(Opcode, VTs, N1, N2);
+ }
+
+ AllNodes.push_back(N);
+ return SDOperand(N, 0);
+}
+
+SDOperand SelectionDAG::getNode(unsigned Opcode, MVT::ValueType VT,
+ SDOperand N1, SDOperand N2, SDOperand N3) {
+ // Perform various simplifications.
+ ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.Val);
+ ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2.Val);
+ switch (Opcode) {
+ case ISD::SETCC: {
+ // Use FoldSetCC to simplify SETCC's.
+ SDOperand Simp = FoldSetCC(VT, N1, N2, cast<CondCodeSDNode>(N3)->get());
+ if (Simp.Val) return Simp;
+ break;
+ }
+ case ISD::SELECT:
+ if (N1C)
+ if (N1C->getValue())
+ return N2; // select true, X, Y -> X
+ else
+ return N3; // select false, X, Y -> Y
+
+ if (N2 == N3) return N2; // select C, X, X -> X
+ break;
+ case ISD::BRCOND:
+ if (N2C)
+ if (N2C->getValue()) // Unconditional branch
+ return getNode(ISD::BR, MVT::Other, N1, N3);
+ else
+ return N1; // Never-taken branch
+ break;
+ case ISD::VECTOR_SHUFFLE:
+ assert(VT == N1.getValueType() && VT == N2.getValueType() &&
+ MVT::isVector(VT) && MVT::isVector(N3.getValueType()) &&
+ N3.getOpcode() == ISD::BUILD_VECTOR &&
+ MVT::getVectorNumElements(VT) == N3.getNumOperands() &&
+ "Illegal VECTOR_SHUFFLE node!");
+ break;
+ case ISD::BIT_CONVERT:
+ // Fold bit_convert nodes from a type to themselves.
+ if (N1.getValueType() == VT)
+ return N1;
+ break;
+ }
+
+ // Memoize node if it doesn't produce a flag.
+ SDNode *N;
+ SDVTList VTs = getVTList(VT);
+ if (VT != MVT::Flag) {
+ SDOperand Ops[] = { N1, N2, N3 };
+ FoldingSetNodeID ID;
+ AddNodeIDNode(ID, Opcode, VTs, Ops, 3);
+ void *IP = 0;
+ if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
+ return SDOperand(E, 0);
+ N = new TernarySDNode(Opcode, VTs, N1, N2, N3);
+ CSEMap.InsertNode(N, IP);
+ } else {
+ N = new TernarySDNode(Opcode, VTs, N1, N2, N3);
+ }
+ AllNodes.push_back(N);
+ return SDOperand(N, 0);
+}
+
+SDOperand SelectionDAG::getNode(unsigned Opcode, MVT::ValueType VT,
+ SDOperand N1, SDOperand N2, SDOperand N3,
+ SDOperand N4) {
+ SDOperand Ops[] = { N1, N2, N3, N4 };
+ return getNode(Opcode, VT, Ops, 4);
+}
+
+SDOperand SelectionDAG::getNode(unsigned Opcode, MVT::ValueType VT,
+ SDOperand N1, SDOperand N2, SDOperand N3,
+ SDOperand N4, SDOperand N5) {
+ SDOperand Ops[] = { N1, N2, N3, N4, N5 };
+ return getNode(Opcode, VT, Ops, 5);
+}
+
+SDOperand SelectionDAG::getLoad(MVT::ValueType VT,
+ SDOperand Chain, SDOperand Ptr,
+ const Value *SV, int SVOffset,
+ bool isVolatile, unsigned Alignment) {
+ if (Alignment == 0) { // Ensure that codegen never sees alignment 0
+ const Type *Ty = 0;
+ if (VT != MVT::iPTR) {
+ Ty = MVT::getTypeForValueType(VT);
+ } else if (SV) {
+ const PointerType *PT = dyn_cast<PointerType>(SV->getType());
+ assert(PT && "Value for load must be a pointer");
+ Ty = PT->getElementType();
+ }
+ assert(Ty && "Could not get type information for load");
+ Alignment = TLI.getTargetData()->getABITypeAlignment(Ty);
+ }
+ SDVTList VTs = getVTList(VT, MVT::Other);
+ SDOperand Undef = getNode(ISD::UNDEF, Ptr.getValueType());
+ SDOperand Ops[] = { Chain, Ptr, Undef };
+ FoldingSetNodeID ID;
+ AddNodeIDNode(ID, ISD::LOAD, VTs, Ops, 3);
+ ID.AddInteger(ISD::UNINDEXED);
+ ID.AddInteger(ISD::NON_EXTLOAD);
+ ID.AddInteger(VT);
+ ID.AddPointer(SV);
+ ID.AddInteger(SVOffset);
+ ID.AddInteger(Alignment);
+ ID.AddInteger(isVolatile);
+ void *IP = 0;
+ if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
+ return SDOperand(E, 0);
+ SDNode *N = new LoadSDNode(Ops, VTs, ISD::UNINDEXED,
+ ISD::NON_EXTLOAD, VT, SV, SVOffset, Alignment,
+ isVolatile);
+ CSEMap.InsertNode(N, IP);
+ AllNodes.push_back(N);
+ return SDOperand(N, 0);
+}
+
+SDOperand SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, MVT::ValueType VT,
+ SDOperand Chain, SDOperand Ptr,
+ const Value *SV,
+ int SVOffset, MVT::ValueType EVT,
+ bool isVolatile, unsigned Alignment) {
+ // If they are asking for an extending load from/to the same thing, return a
+ // normal load.
+ if (VT == EVT)
+ ExtType = ISD::NON_EXTLOAD;
+
+ if (MVT::isVector(VT))
+ assert(EVT == MVT::getVectorElementType(VT) && "Invalid vector extload!");
+ else
+ assert(EVT < VT && "Should only be an extending load, not truncating!");
+ assert((ExtType == ISD::EXTLOAD || MVT::isInteger(VT)) &&
+ "Cannot sign/zero extend a FP/Vector load!");
+ assert(MVT::isInteger(VT) == MVT::isInteger(EVT) &&
+ "Cannot convert from FP to Int or Int -> FP!");
+
+ if (Alignment == 0) { // Ensure that codegen never sees alignment 0
+ const Type *Ty = 0;
+ if (VT != MVT::iPTR) {
+ Ty = MVT::getTypeForValueType(VT);
+ } else if (SV) {
+ const PointerType *PT = dyn_cast<PointerType>(SV->getType());
+ assert(PT && "Value for load must be a pointer");
+ Ty = PT->getElementType();
+ }
+ assert(Ty && "Could not get type information for load");
+ Alignment = TLI.getTargetData()->getABITypeAlignment(Ty);
+ }
+ SDVTList VTs = getVTList(VT, MVT::Other);
+ SDOperand Undef = getNode(ISD::UNDEF, Ptr.getValueType());
+ SDOperand Ops[] = { Chain, Ptr, Undef };
+ FoldingSetNodeID ID;
+ AddNodeIDNode(ID, ISD::LOAD, VTs, Ops, 3);
+ ID.AddInteger(ISD::UNINDEXED);
+ ID.AddInteger(ExtType);
+ ID.AddInteger(EVT);
+ ID.AddPointer(SV);
+ ID.AddInteger(SVOffset);
+ ID.AddInteger(Alignment);
+ ID.AddInteger(isVolatile);
+ void *IP = 0;
+ if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
+ return SDOperand(E, 0);
+ SDNode *N = new LoadSDNode(Ops, VTs, ISD::UNINDEXED, ExtType, EVT,
+ SV, SVOffset, Alignment, isVolatile);
+ CSEMap.InsertNode(N, IP);
+ AllNodes.push_back(N);
+ return SDOperand(N, 0);
+}
+
+SDOperand
+SelectionDAG::getIndexedLoad(SDOperand OrigLoad, SDOperand Base,
+ SDOperand Offset, ISD::MemIndexedMode AM) {
+ LoadSDNode *LD = cast<LoadSDNode>(OrigLoad);
+ assert(LD->getOffset().getOpcode() == ISD::UNDEF &&
+ "Load is already a indexed load!");
+ MVT::ValueType VT = OrigLoad.getValueType();
+ SDVTList VTs = getVTList(VT, Base.getValueType(), MVT::Other);
+ SDOperand Ops[] = { LD->getChain(), Base, Offset };
+ FoldingSetNodeID ID;
+ AddNodeIDNode(ID, ISD::LOAD, VTs, Ops, 3);
+ ID.AddInteger(AM);
+ ID.AddInteger(LD->getExtensionType());
+ ID.AddInteger(LD->getLoadedVT());
+ ID.AddPointer(LD->getSrcValue());
+ ID.AddInteger(LD->getSrcValueOffset());
+ ID.AddInteger(LD->getAlignment());
+ ID.AddInteger(LD->isVolatile());
+ void *IP = 0;
+ if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
+ return SDOperand(E, 0);
+ SDNode *N = new LoadSDNode(Ops, VTs, AM,
+ LD->getExtensionType(), LD->getLoadedVT(),
+ LD->getSrcValue(), LD->getSrcValueOffset(),
+ LD->getAlignment(), LD->isVolatile());
+ CSEMap.InsertNode(N, IP);
+ AllNodes.push_back(N);
+ return SDOperand(N, 0);
+}
+
+SDOperand SelectionDAG::getStore(SDOperand Chain, SDOperand Val,
+ SDOperand Ptr, const Value *SV, int SVOffset,
+ bool isVolatile, unsigned Alignment) {
+ MVT::ValueType VT = Val.getValueType();
+
+ if (Alignment == 0) { // Ensure that codegen never sees alignment 0
+ const Type *Ty = 0;
+ if (VT != MVT::iPTR) {
+ Ty = MVT::getTypeForValueType(VT);
+ } else if (SV) {
+ const PointerType *PT = dyn_cast<PointerType>(SV->getType());
+ assert(PT && "Value for store must be a pointer");
+ Ty = PT->getElementType();
+ }
+ assert(Ty && "Could not get type information for store");
+ Alignment = TLI.getTargetData()->getABITypeAlignment(Ty);
+ }
+ SDVTList VTs = getVTList(MVT::Other);
+ SDOperand Undef = getNode(ISD::UNDEF, Ptr.getValueType());
+ SDOperand Ops[] = { Chain, Val, Ptr, Undef };
+ FoldingSetNodeID ID;
+ AddNodeIDNode(ID, ISD::STORE, VTs, Ops, 4);
+ ID.AddInteger(ISD::UNINDEXED);
+ ID.AddInteger(false);
+ ID.AddInteger(VT);
+ ID.AddPointer(SV);
+ ID.AddInteger(SVOffset);
+ ID.AddInteger(Alignment);
+ ID.AddInteger(isVolatile);
+ void *IP = 0;
+ if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
+ return SDOperand(E, 0);
+ SDNode *N = new StoreSDNode(Ops, VTs, ISD::UNINDEXED, false,
+ VT, SV, SVOffset, Alignment, isVolatile);
+ CSEMap.InsertNode(N, IP);
+ AllNodes.push_back(N);
+ return SDOperand(N, 0);
+}
+
+SDOperand SelectionDAG::getTruncStore(SDOperand Chain, SDOperand Val,
+ SDOperand Ptr, const Value *SV,
+ int SVOffset, MVT::ValueType SVT,
+ bool isVolatile, unsigned Alignment) {
+ MVT::ValueType VT = Val.getValueType();
+ bool isTrunc = VT != SVT;
+
+ assert(VT > SVT && "Not a truncation?");
+ assert(MVT::isInteger(VT) == MVT::isInteger(SVT) &&
+ "Can't do FP-INT conversion!");
+
+ if (Alignment == 0) { // Ensure that codegen never sees alignment 0
+ const Type *Ty = 0;
+ if (VT != MVT::iPTR) {
+ Ty = MVT::getTypeForValueType(VT);
+ } else if (SV) {
+ const PointerType *PT = dyn_cast<PointerType>(SV->getType());
+ assert(PT && "Value for store must be a pointer");
+ Ty = PT->getElementType();
+ }
+ assert(Ty && "Could not get type information for store");
+ Alignment = TLI.getTargetData()->getABITypeAlignment(Ty);
+ }
+ SDVTList VTs = getVTList(MVT::Other);
+ SDOperand Undef = getNode(ISD::UNDEF, Ptr.getValueType());
+ SDOperand Ops[] = { Chain, Val, Ptr, Undef };
+ FoldingSetNodeID ID;
+ AddNodeIDNode(ID, ISD::STORE, VTs, Ops, 4);
+ ID.AddInteger(ISD::UNINDEXED);
+ ID.AddInteger(isTrunc);
+ ID.AddInteger(SVT);
+ ID.AddPointer(SV);
+ ID.AddInteger(SVOffset);
+ ID.AddInteger(Alignment);
+ ID.AddInteger(isVolatile);
+ void *IP = 0;
+ if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
+ return SDOperand(E, 0);
+ SDNode *N = new StoreSDNode(Ops, VTs, ISD::UNINDEXED, isTrunc,
+ SVT, SV, SVOffset, Alignment, isVolatile);
+ CSEMap.InsertNode(N, IP);
+ AllNodes.push_back(N);
+ return SDOperand(N, 0);
+}
+
+SDOperand
+SelectionDAG::getIndexedStore(SDOperand OrigStore, SDOperand Base,
+ SDOperand Offset, ISD::MemIndexedMode AM) {
+ StoreSDNode *ST = cast<StoreSDNode>(OrigStore);
+ assert(ST->getOffset().getOpcode() == ISD::UNDEF &&
+ "Store is already a indexed store!");
+ SDVTList VTs = getVTList(Base.getValueType(), MVT::Other);
+ SDOperand Ops[] = { ST->getChain(), ST->getValue(), Base, Offset };
+ FoldingSetNodeID ID;
+ AddNodeIDNode(ID, ISD::STORE, VTs, Ops, 4);
+ ID.AddInteger(AM);
+ ID.AddInteger(ST->isTruncatingStore());
+ ID.AddInteger(ST->getStoredVT());
+ ID.AddPointer(ST->getSrcValue());
+ ID.AddInteger(ST->getSrcValueOffset());
+ ID.AddInteger(ST->getAlignment());
+ ID.AddInteger(ST->isVolatile());
+ void *IP = 0;
+ if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
+ return SDOperand(E, 0);
+ SDNode *N = new StoreSDNode(Ops, VTs, AM,
+ ST->isTruncatingStore(), ST->getStoredVT(),
+ ST->getSrcValue(), ST->getSrcValueOffset(),
+ ST->getAlignment(), ST->isVolatile());
+ CSEMap.InsertNode(N, IP);
+ AllNodes.push_back(N);
+ return SDOperand(N, 0);
+}
+
+SDOperand SelectionDAG::getVAArg(MVT::ValueType VT,
+ SDOperand Chain, SDOperand Ptr,
+ SDOperand SV) {
+ SDOperand Ops[] = { Chain, Ptr, SV };
+ return getNode(ISD::VAARG, getVTList(VT, MVT::Other), Ops, 3);
+}
+
+SDOperand SelectionDAG::getNode(unsigned Opcode, MVT::ValueType VT,
+ const SDOperand *Ops, unsigned NumOps) {
+ switch (NumOps) {
+ case 0: return getNode(Opcode, VT);
+ case 1: return getNode(Opcode, VT, Ops[0]);
+ case 2: return getNode(Opcode, VT, Ops[0], Ops[1]);
+ case 3: return getNode(Opcode, VT, Ops[0], Ops[1], Ops[2]);
+ default: break;
+ }
+
+ switch (Opcode) {
+ default: break;
+ case ISD::SELECT_CC: {
+ assert(NumOps == 5 && "SELECT_CC takes 5 operands!");
+ assert(Ops[0].getValueType() == Ops[1].getValueType() &&
+ "LHS and RHS of condition must have same type!");
+ assert(Ops[2].getValueType() == Ops[3].getValueType() &&
+ "True and False arms of SelectCC must have same type!");
+ assert(Ops[2].getValueType() == VT &&
+ "select_cc node must be of same type as true and false value!");
+ break;
+ }
+ case ISD::BR_CC: {
+ assert(NumOps == 5 && "BR_CC takes 5 operands!");
+ assert(Ops[2].getValueType() == Ops[3].getValueType() &&
+ "LHS/RHS of comparison should match types!");
+ break;
+ }
+ }
+
+ // Memoize nodes.
+ SDNode *N;
+ SDVTList VTs = getVTList(VT);
+ if (VT != MVT::Flag) {
+ FoldingSetNodeID ID;
+ AddNodeIDNode(ID, Opcode, VTs, Ops, NumOps);
+ void *IP = 0;
+ if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
+ return SDOperand(E, 0);
+ N = new SDNode(Opcode, VTs, Ops, NumOps);
+ CSEMap.InsertNode(N, IP);
+ } else {
+ N = new SDNode(Opcode, VTs, Ops, NumOps);
+ }
+ AllNodes.push_back(N);
+ return SDOperand(N, 0);
+}
+
+SDOperand SelectionDAG::getNode(unsigned Opcode,
+ std::vector<MVT::ValueType> &ResultTys,
+ const SDOperand *Ops, unsigned NumOps) {
+ return getNode(Opcode, getNodeValueTypes(ResultTys), ResultTys.size(),
+ Ops, NumOps);
+}
+
+SDOperand SelectionDAG::getNode(unsigned Opcode,
+ const MVT::ValueType *VTs, unsigned NumVTs,
+ const SDOperand *Ops, unsigned NumOps) {
+ if (NumVTs == 1)
+ return getNode(Opcode, VTs[0], Ops, NumOps);
+ return getNode(Opcode, makeVTList(VTs, NumVTs), Ops, NumOps);
+}
+
+SDOperand SelectionDAG::getNode(unsigned Opcode, SDVTList VTList,
+ const SDOperand *Ops, unsigned NumOps) {
+ if (VTList.NumVTs == 1)
+ return getNode(Opcode, VTList.VTs[0], Ops, NumOps);
+
+ switch (Opcode) {
+ // FIXME: figure out how to safely handle things like
+ // int foo(int x) { return 1 << (x & 255); }
+ // int bar() { return foo(256); }
+#if 0
+ case ISD::SRA_PARTS:
+ case ISD::SRL_PARTS:
+ case ISD::SHL_PARTS:
+ if (N3.getOpcode() == ISD::SIGN_EXTEND_INREG &&
+ cast<VTSDNode>(N3.getOperand(1))->getVT() != MVT::i1)
+ return getNode(Opcode, VT, N1, N2, N3.getOperand(0));
+ else if (N3.getOpcode() == ISD::AND)
+ if (ConstantSDNode *AndRHS = dyn_cast<ConstantSDNode>(N3.getOperand(1))) {
+ // If the and is only masking out bits that cannot effect the shift,
+ // eliminate the and.
+ unsigned NumBits = MVT::getSizeInBits(VT)*2;
+ if ((AndRHS->getValue() & (NumBits-1)) == NumBits-1)
+ return getNode(Opcode, VT, N1, N2, N3.getOperand(0));
+ }
+ break;
+#endif
+ }
+
+ // Memoize the node unless it returns a flag.
+ SDNode *N;
+ if (VTList.VTs[VTList.NumVTs-1] != MVT::Flag) {
+ FoldingSetNodeID ID;
+ AddNodeIDNode(ID, Opcode, VTList, Ops, NumOps);
+ void *IP = 0;
+ if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
+ return SDOperand(E, 0);
+ if (NumOps == 1)
+ N = new UnarySDNode(Opcode, VTList, Ops[0]);
+ else if (NumOps == 2)
+ N = new BinarySDNode(Opcode, VTList, Ops[0], Ops[1]);
+ else if (NumOps == 3)
+ N = new TernarySDNode(Opcode, VTList, Ops[0], Ops[1], Ops[2]);
+ else
+ N = new SDNode(Opcode, VTList, Ops, NumOps);
+ CSEMap.InsertNode(N, IP);
+ } else {
+ if (NumOps == 1)
+ N = new UnarySDNode(Opcode, VTList, Ops[0]);
+ else if (NumOps == 2)
+ N = new BinarySDNode(Opcode, VTList, Ops[0], Ops[1]);
+ else if (NumOps == 3)
+ N = new TernarySDNode(Opcode, VTList, Ops[0], Ops[1], Ops[2]);
+ else
+ N = new SDNode(Opcode, VTList, Ops, NumOps);
+ }
+ AllNodes.push_back(N);
+ return SDOperand(N, 0);
+}
+
+SDVTList SelectionDAG::getVTList(MVT::ValueType VT) {
+ if (!MVT::isExtendedVT(VT))
+ return makeVTList(SDNode::getValueTypeList(VT), 1);
+
+ for (std::list<std::vector<MVT::ValueType> >::iterator I = VTList.begin(),
+ E = VTList.end(); I != E; ++I) {
+ if (I->size() == 1 && (*I)[0] == VT)
+ return makeVTList(&(*I)[0], 1);
+ }
+ std::vector<MVT::ValueType> V;
+ V.push_back(VT);
+ VTList.push_front(V);
+ return makeVTList(&(*VTList.begin())[0], 1);
+}
+
+SDVTList SelectionDAG::getVTList(MVT::ValueType VT1, MVT::ValueType VT2) {
+ for (std::list<std::vector<MVT::ValueType> >::iterator I = VTList.begin(),
+ E = VTList.end(); I != E; ++I) {
+ if (I->size() == 2 && (*I)[0] == VT1 && (*I)[1] == VT2)
+ return makeVTList(&(*I)[0], 2);
+ }
+ std::vector<MVT::ValueType> V;
+ V.push_back(VT1);
+ V.push_back(VT2);
+ VTList.push_front(V);
+ return makeVTList(&(*VTList.begin())[0], 2);
+}
+SDVTList SelectionDAG::getVTList(MVT::ValueType VT1, MVT::ValueType VT2,
+ MVT::ValueType VT3) {
+ for (std::list<std::vector<MVT::ValueType> >::iterator I = VTList.begin(),
+ E = VTList.end(); I != E; ++I) {
+ if (I->size() == 3 && (*I)[0] == VT1 && (*I)[1] == VT2 &&
+ (*I)[2] == VT3)
+ return makeVTList(&(*I)[0], 3);
+ }
+ std::vector<MVT::ValueType> V;
+ V.push_back(VT1);
+ V.push_back(VT2);
+ V.push_back(VT3);
+ VTList.push_front(V);
+ return makeVTList(&(*VTList.begin())[0], 3);
+}
+
+SDVTList SelectionDAG::getVTList(const MVT::ValueType *VTs, unsigned NumVTs) {
+ switch (NumVTs) {
+ case 0: assert(0 && "Cannot have nodes without results!");
+ case 1: return getVTList(VTs[0]);
+ case 2: return getVTList(VTs[0], VTs[1]);
+ case 3: return getVTList(VTs[0], VTs[1], VTs[2]);
+ default: break;
+ }
+
+ for (std::list<std::vector<MVT::ValueType> >::iterator I = VTList.begin(),
+ E = VTList.end(); I != E; ++I) {
+ if (I->size() != NumVTs || VTs[0] != (*I)[0] || VTs[1] != (*I)[1]) continue;
+
+ bool NoMatch = false;
+ for (unsigned i = 2; i != NumVTs; ++i)
+ if (VTs[i] != (*I)[i]) {
+ NoMatch = true;
+ break;
+ }
+ if (!NoMatch)
+ return makeVTList(&*I->begin(), NumVTs);
+ }
+
+ VTList.push_front(std::vector<MVT::ValueType>(VTs, VTs+NumVTs));
+ return makeVTList(&*VTList.begin()->begin(), NumVTs);
+}
+
+
+/// UpdateNodeOperands - *Mutate* the specified node in-place to have the
+/// specified operands. If the resultant node already exists in the DAG,
+/// this does not modify the specified node, instead it returns the node that
+/// already exists. If the resultant node does not exist in the DAG, the
+/// input node is returned. As a degenerate case, if you specify the same
+/// input operands as the node already has, the input node is returned.
+SDOperand SelectionDAG::
+UpdateNodeOperands(SDOperand InN, SDOperand Op) {
+ SDNode *N = InN.Val;
+ assert(N->getNumOperands() == 1 && "Update with wrong number of operands");
+
+ // Check to see if there is no change.
+ if (Op == N->getOperand(0)) return InN;
+
+ // See if the modified node already exists.
+ void *InsertPos = 0;
+ if (SDNode *Existing = FindModifiedNodeSlot(N, Op, InsertPos))
+ return SDOperand(Existing, InN.ResNo);
+
+ // Nope it doesn't. Remove the node from it's current place in the maps.
+ if (InsertPos)
+ RemoveNodeFromCSEMaps(N);
+
+ // Now we update the operands.
+ N->OperandList[0].Val->removeUser(N);
+ Op.Val->addUser(N);
+ N->OperandList[0] = Op;
+
+ // If this gets put into a CSE map, add it.
+ if (InsertPos) CSEMap.InsertNode(N, InsertPos);
+ return InN;
+}
+
+SDOperand SelectionDAG::
+UpdateNodeOperands(SDOperand InN, SDOperand Op1, SDOperand Op2) {
+ SDNode *N = InN.Val;
+ assert(N->getNumOperands() == 2 && "Update with wrong number of operands");
+
+ // Check to see if there is no change.
+ if (Op1 == N->getOperand(0) && Op2 == N->getOperand(1))
+ return InN; // No operands changed, just return the input node.
+
+ // See if the modified node already exists.
+ void *InsertPos = 0;
+ if (SDNode *Existing = FindModifiedNodeSlot(N, Op1, Op2, InsertPos))
+ return SDOperand(Existing, InN.ResNo);
+
+ // Nope it doesn't. Remove the node from it's current place in the maps.
+ if (InsertPos)
+ RemoveNodeFromCSEMaps(N);
+
+ // Now we update the operands.
+ if (N->OperandList[0] != Op1) {
+ N->OperandList[0].Val->removeUser(N);
+ Op1.Val->addUser(N);
+ N->OperandList[0] = Op1;
+ }
+ if (N->OperandList[1] != Op2) {
+ N->OperandList[1].Val->removeUser(N);
+ Op2.Val->addUser(N);
+ N->OperandList[1] = Op2;
+ }
+
+ // If this gets put into a CSE map, add it.
+ if (InsertPos) CSEMap.InsertNode(N, InsertPos);
+ return InN;
+}
+
+SDOperand SelectionDAG::
+UpdateNodeOperands(SDOperand N, SDOperand Op1, SDOperand Op2, SDOperand Op3) {
+ SDOperand Ops[] = { Op1, Op2, Op3 };
+ return UpdateNodeOperands(N, Ops, 3);
+}
+
+SDOperand SelectionDAG::
+UpdateNodeOperands(SDOperand N, SDOperand Op1, SDOperand Op2,
+ SDOperand Op3, SDOperand Op4) {
+ SDOperand Ops[] = { Op1, Op2, Op3, Op4 };
+ return UpdateNodeOperands(N, Ops, 4);
+}
+
+SDOperand SelectionDAG::
+UpdateNodeOperands(SDOperand N, SDOperand Op1, SDOperand Op2,
+ SDOperand Op3, SDOperand Op4, SDOperand Op5) {
+ SDOperand Ops[] = { Op1, Op2, Op3, Op4, Op5 };
+ return UpdateNodeOperands(N, Ops, 5);
+}
+
+
+SDOperand SelectionDAG::
+UpdateNodeOperands(SDOperand InN, SDOperand *Ops, unsigned NumOps) {
+ SDNode *N = InN.Val;
+ assert(N->getNumOperands() == NumOps &&
+ "Update with wrong number of operands");
+
+ // Check to see if there is no change.
+ bool AnyChange = false;
+ for (unsigned i = 0; i != NumOps; ++i) {
+ if (Ops[i] != N->getOperand(i)) {
+ AnyChange = true;
+ break;
+ }
+ }
+
+ // No operands changed, just return the input node.
+ if (!AnyChange) return InN;
+
+ // See if the modified node already exists.
+ void *InsertPos = 0;
+ if (SDNode *Existing = FindModifiedNodeSlot(N, Ops, NumOps, InsertPos))
+ return SDOperand(Existing, InN.ResNo);
+
+ // Nope it doesn't. Remove the node from it's current place in the maps.
+ if (InsertPos)
+ RemoveNodeFromCSEMaps(N);
+
+ // Now we update the operands.
+ for (unsigned i = 0; i != NumOps; ++i) {
+ if (N->OperandList[i] != Ops[i]) {
+ N->OperandList[i].Val->removeUser(N);
+ Ops[i].Val->addUser(N);
+ N->OperandList[i] = Ops[i];
+ }
+ }
+
+ // If this gets put into a CSE map, add it.
+ if (InsertPos) CSEMap.InsertNode(N, InsertPos);
+ return InN;
+}
+
+
+/// MorphNodeTo - This frees the operands of the current node, resets the
+/// opcode, types, and operands to the specified value. This should only be
+/// used by the SelectionDAG class.
+void SDNode::MorphNodeTo(unsigned Opc, SDVTList L,
+ const SDOperand *Ops, unsigned NumOps) {
+ NodeType = Opc;
+ ValueList = L.VTs;
+ NumValues = L.NumVTs;
+
+ // Clear the operands list, updating used nodes to remove this from their
+ // use list.
+ for (op_iterator I = op_begin(), E = op_end(); I != E; ++I)
+ I->Val->removeUser(this);
+
+ // If NumOps is larger than the # of operands we currently have, reallocate
+ // the operand list.
+ if (NumOps > NumOperands) {
+ if (OperandsNeedDelete)
+ delete [] OperandList;
+ OperandList = new SDOperand[NumOps];
+ OperandsNeedDelete = true;
+ }
+
+ // Assign the new operands.
+ NumOperands = NumOps;
+
+ for (unsigned i = 0, e = NumOps; i != e; ++i) {
+ OperandList[i] = Ops[i];
+ SDNode *N = OperandList[i].Val;
+ N->Uses.push_back(this);
+ }
+}
+
+/// SelectNodeTo - These are used for target selectors to *mutate* the
+/// specified node to have the specified return type, Target opcode, and
+/// operands. Note that target opcodes are stored as
+/// ISD::BUILTIN_OP_END+TargetOpcode in the node opcode field.
+///
+/// Note that SelectNodeTo returns the resultant node. If there is already a
+/// node of the specified opcode and operands, it returns that node instead of
+/// the current one.
+SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned TargetOpc,
+ MVT::ValueType VT) {
+ SDVTList VTs = getVTList(VT);
+ FoldingSetNodeID ID;
+ AddNodeIDNode(ID, ISD::BUILTIN_OP_END+TargetOpc, VTs, 0, 0);
+ void *IP = 0;
+ if (SDNode *ON = CSEMap.FindNodeOrInsertPos(ID, IP))
+ return ON;
+
+ RemoveNodeFromCSEMaps(N);
+
+ N->MorphNodeTo(ISD::BUILTIN_OP_END+TargetOpc, VTs, 0, 0);
+
+ CSEMap.InsertNode(N, IP);
+ return N;
+}
+
+SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned TargetOpc,
+ MVT::ValueType VT, SDOperand Op1) {
+ // If an identical node already exists, use it.
+ SDVTList VTs = getVTList(VT);
+ SDOperand Ops[] = { Op1 };
+
+ FoldingSetNodeID ID;
+ AddNodeIDNode(ID, ISD::BUILTIN_OP_END+TargetOpc, VTs, Ops, 1);
+ void *IP = 0;
+ if (SDNode *ON = CSEMap.FindNodeOrInsertPos(ID, IP))
+ return ON;
+
+ RemoveNodeFromCSEMaps(N);
+ N->MorphNodeTo(ISD::BUILTIN_OP_END+TargetOpc, VTs, Ops, 1);
+ CSEMap.InsertNode(N, IP);
+ return N;
+}
+
+SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned TargetOpc,
+ MVT::ValueType VT, SDOperand Op1,
+ SDOperand Op2) {
+ // If an identical node already exists, use it.
+ SDVTList VTs = getVTList(VT);
+ SDOperand Ops[] = { Op1, Op2 };
+
+ FoldingSetNodeID ID;
+ AddNodeIDNode(ID, ISD::BUILTIN_OP_END+TargetOpc, VTs, Ops, 2);
+ void *IP = 0;
+ if (SDNode *ON = CSEMap.FindNodeOrInsertPos(ID, IP))
+ return ON;
+
+ RemoveNodeFromCSEMaps(N);
+
+ N->MorphNodeTo(ISD::BUILTIN_OP_END+TargetOpc, VTs, Ops, 2);
+
+ CSEMap.InsertNode(N, IP); // Memoize the new node.
+ return N;
+}
+
+SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned TargetOpc,
+ MVT::ValueType VT, SDOperand Op1,
+ SDOperand Op2, SDOperand Op3) {
+ // If an identical node already exists, use it.
+ SDVTList VTs = getVTList(VT);
+ SDOperand Ops[] = { Op1, Op2, Op3 };
+ FoldingSetNodeID ID;
+ AddNodeIDNode(ID, ISD::BUILTIN_OP_END+TargetOpc, VTs, Ops, 3);
+ void *IP = 0;
+ if (SDNode *ON = CSEMap.FindNodeOrInsertPos(ID, IP))
+ return ON;
+
+ RemoveNodeFromCSEMaps(N);
+
+ N->MorphNodeTo(ISD::BUILTIN_OP_END+TargetOpc, VTs, Ops, 3);
+
+ CSEMap.InsertNode(N, IP); // Memoize the new node.
+ return N;
+}
+
+SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned TargetOpc,
+ MVT::ValueType VT, const SDOperand *Ops,
+ unsigned NumOps) {
+ // If an identical node already exists, use it.
+ SDVTList VTs = getVTList(VT);
+ FoldingSetNodeID ID;
+ AddNodeIDNode(ID, ISD::BUILTIN_OP_END+TargetOpc, VTs, Ops, NumOps);
+ void *IP = 0;
+ if (SDNode *ON = CSEMap.FindNodeOrInsertPos(ID, IP))
+ return ON;
+
+ RemoveNodeFromCSEMaps(N);
+ N->MorphNodeTo(ISD::BUILTIN_OP_END+TargetOpc, VTs, Ops, NumOps);
+
+ CSEMap.InsertNode(N, IP); // Memoize the new node.
+ return N;
+}
+
+SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned TargetOpc,
+ MVT::ValueType VT1, MVT::ValueType VT2,
+ SDOperand Op1, SDOperand Op2) {
+ SDVTList VTs = getVTList(VT1, VT2);
+ FoldingSetNodeID ID;
+ SDOperand Ops[] = { Op1, Op2 };
+ AddNodeIDNode(ID, ISD::BUILTIN_OP_END+TargetOpc, VTs, Ops, 2);
+ void *IP = 0;
+ if (SDNode *ON = CSEMap.FindNodeOrInsertPos(ID, IP))
+ return ON;
+
+ RemoveNodeFromCSEMaps(N);
+ N->MorphNodeTo(ISD::BUILTIN_OP_END+TargetOpc, VTs, Ops, 2);
+ CSEMap.InsertNode(N, IP); // Memoize the new node.
+ return N;
+}
+
+SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned TargetOpc,
+ MVT::ValueType VT1, MVT::ValueType VT2,
+ SDOperand Op1, SDOperand Op2,
+ SDOperand Op3) {
+ // If an identical node already exists, use it.
+ SDVTList VTs = getVTList(VT1, VT2);
+ SDOperand Ops[] = { Op1, Op2, Op3 };
+ FoldingSetNodeID ID;
+ AddNodeIDNode(ID, ISD::BUILTIN_OP_END+TargetOpc, VTs, Ops, 3);
+ void *IP = 0;
+ if (SDNode *ON = CSEMap.FindNodeOrInsertPos(ID, IP))
+ return ON;
+
+ RemoveNodeFromCSEMaps(N);
+
+ N->MorphNodeTo(ISD::BUILTIN_OP_END+TargetOpc, VTs, Ops, 3);
+ CSEMap.InsertNode(N, IP); // Memoize the new node.
+ return N;
+}
+
+
+/// getTargetNode - These are used for target selectors to create a new node
+/// with specified return type(s), target opcode, and operands.
+///
+/// Note that getTargetNode returns the resultant node. If there is already a
+/// node of the specified opcode and operands, it returns that node instead of
+/// the current one.
+SDNode *SelectionDAG::getTargetNode(unsigned Opcode, MVT::ValueType VT) {
+ return getNode(ISD::BUILTIN_OP_END+Opcode, VT).Val;
+}
+SDNode *SelectionDAG::getTargetNode(unsigned Opcode, MVT::ValueType VT,
+ SDOperand Op1) {
+ return getNode(ISD::BUILTIN_OP_END+Opcode, VT, Op1).Val;
+}
+SDNode *SelectionDAG::getTargetNode(unsigned Opcode, MVT::ValueType VT,
+ SDOperand Op1, SDOperand Op2) {
+ return getNode(ISD::BUILTIN_OP_END+Opcode, VT, Op1, Op2).Val;
+}
+SDNode *SelectionDAG::getTargetNode(unsigned Opcode, MVT::ValueType VT,
+ SDOperand Op1, SDOperand Op2,
+ SDOperand Op3) {
+ return getNode(ISD::BUILTIN_OP_END+Opcode, VT, Op1, Op2, Op3).Val;
+}
+SDNode *SelectionDAG::getTargetNode(unsigned Opcode, MVT::ValueType VT,
+ const SDOperand *Ops, unsigned NumOps) {
+ return getNode(ISD::BUILTIN_OP_END+Opcode, VT, Ops, NumOps).Val;
+}
+SDNode *SelectionDAG::getTargetNode(unsigned Opcode, MVT::ValueType VT1,
+ MVT::ValueType VT2, SDOperand Op1) {
+ const MVT::ValueType *VTs = getNodeValueTypes(VT1, VT2);
+ return getNode(ISD::BUILTIN_OP_END+Opcode, VTs, 2, &Op1, 1).Val;
+}
+SDNode *SelectionDAG::getTargetNode(unsigned Opcode, MVT::ValueType VT1,
+ MVT::ValueType VT2, SDOperand Op1,
+ SDOperand Op2) {
+ const MVT::ValueType *VTs = getNodeValueTypes(VT1, VT2);
+ SDOperand Ops[] = { Op1, Op2 };
+ return getNode(ISD::BUILTIN_OP_END+Opcode, VTs, 2, Ops, 2).Val;
+}
+SDNode *SelectionDAG::getTargetNode(unsigned Opcode, MVT::ValueType VT1,
+ MVT::ValueType VT2, SDOperand Op1,
+ SDOperand Op2, SDOperand Op3) {
+ const MVT::ValueType *VTs = getNodeValueTypes(VT1, VT2);
+ SDOperand Ops[] = { Op1, Op2, Op3 };
+ return getNode(ISD::BUILTIN_OP_END+Opcode, VTs, 2, Ops, 3).Val;
+}
+SDNode *SelectionDAG::getTargetNode(unsigned Opcode, MVT::ValueType VT1,
+ MVT::ValueType VT2,
+ const SDOperand *Ops, unsigned NumOps) {
+ const MVT::ValueType *VTs = getNodeValueTypes(VT1, VT2);
+ return getNode(ISD::BUILTIN_OP_END+Opcode, VTs, 2, Ops, NumOps).Val;
+}
+SDNode *SelectionDAG::getTargetNode(unsigned Opcode, MVT::ValueType VT1,
+ MVT::ValueType VT2, MVT::ValueType VT3,
+ SDOperand Op1, SDOperand Op2) {
+ const MVT::ValueType *VTs = getNodeValueTypes(VT1, VT2, VT3);
+ SDOperand Ops[] = { Op1, Op2 };
+ return getNode(ISD::BUILTIN_OP_END+Opcode, VTs, 3, Ops, 2).Val;
+}
+SDNode *SelectionDAG::getTargetNode(unsigned Opcode, MVT::ValueType VT1,
+ MVT::ValueType VT2, MVT::ValueType VT3,
+ SDOperand Op1, SDOperand Op2,
+ SDOperand Op3) {
+ const MVT::ValueType *VTs = getNodeValueTypes(VT1, VT2, VT3);
+ SDOperand Ops[] = { Op1, Op2, Op3 };
+ return getNode(ISD::BUILTIN_OP_END+Opcode, VTs, 3, Ops, 3).Val;
+}
+SDNode *SelectionDAG::getTargetNode(unsigned Opcode, MVT::ValueType VT1,
+ MVT::ValueType VT2, MVT::ValueType VT3,
+ const SDOperand *Ops, unsigned NumOps) {
+ const MVT::ValueType *VTs = getNodeValueTypes(VT1, VT2, VT3);
+ return getNode(ISD::BUILTIN_OP_END+Opcode, VTs, 3, Ops, NumOps).Val;
+}
+
+/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
+/// This can cause recursive merging of nodes in the DAG.
+///
+/// This version assumes From/To have a single result value.
+///
+void SelectionDAG::ReplaceAllUsesWith(SDOperand FromN, SDOperand ToN,
+ std::vector<SDNode*> *Deleted) {
+ SDNode *From = FromN.Val, *To = ToN.Val;
+ assert(From->getNumValues() == 1 && To->getNumValues() == 1 &&
+ "Cannot replace with this method!");
+ assert(From != To && "Cannot replace uses of with self");
+
+ while (!From->use_empty()) {
+ // Process users until they are all gone.
+ SDNode *U = *From->use_begin();
+
+ // This node is about to morph, remove its old self from the CSE maps.
+ RemoveNodeFromCSEMaps(U);
+
+ for (SDOperand *I = U->OperandList, *E = U->OperandList+U->NumOperands;
+ I != E; ++I)
+ if (I->Val == From) {
+ From->removeUser(U);
+ I->Val = To;
+ To->addUser(U);
+ }
+
+ // Now that we have modified U, add it back to the CSE maps. If it already
+ // exists there, recursively merge the results together.
+ if (SDNode *Existing = AddNonLeafNodeToCSEMaps(U)) {
+ ReplaceAllUsesWith(U, Existing, Deleted);
+ // U is now dead.
+ if (Deleted) Deleted->push_back(U);
+ DeleteNodeNotInCSEMaps(U);
+ }
+ }
+}
+
+/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
+/// This can cause recursive merging of nodes in the DAG.
+///
+/// This version assumes From/To have matching types and numbers of result
+/// values.
+///
+void SelectionDAG::ReplaceAllUsesWith(SDNode *From, SDNode *To,
+ std::vector<SDNode*> *Deleted) {
+ assert(From != To && "Cannot replace uses of with self");
+ assert(From->getNumValues() == To->getNumValues() &&
+ "Cannot use this version of ReplaceAllUsesWith!");
+ if (From->getNumValues() == 1) { // If possible, use the faster version.
+ ReplaceAllUsesWith(SDOperand(From, 0), SDOperand(To, 0), Deleted);
+ return;
+ }
+
+ while (!From->use_empty()) {
+ // Process users until they are all gone.
+ SDNode *U = *From->use_begin();
+
+ // This node is about to morph, remove its old self from the CSE maps.
+ RemoveNodeFromCSEMaps(U);
+
+ for (SDOperand *I = U->OperandList, *E = U->OperandList+U->NumOperands;
+ I != E; ++I)
+ if (I->Val == From) {
+ From->removeUser(U);
+ I->Val = To;
+ To->addUser(U);
+ }
+
+ // Now that we have modified U, add it back to the CSE maps. If it already
+ // exists there, recursively merge the results together.
+ if (SDNode *Existing = AddNonLeafNodeToCSEMaps(U)) {
+ ReplaceAllUsesWith(U, Existing, Deleted);
+ // U is now dead.
+ if (Deleted) Deleted->push_back(U);
+ DeleteNodeNotInCSEMaps(U);
+ }
+ }
+}
+
+/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
+/// This can cause recursive merging of nodes in the DAG.
+///
+/// This version can replace From with any result values. To must match the
+/// number and types of values returned by From.
+void SelectionDAG::ReplaceAllUsesWith(SDNode *From,
+ const SDOperand *To,
+ std::vector<SDNode*> *Deleted) {
+ if (From->getNumValues() == 1 && To[0].Val->getNumValues() == 1) {
+ // Degenerate case handled above.
+ ReplaceAllUsesWith(SDOperand(From, 0), To[0], Deleted);
+ return;
+ }
+
+ while (!From->use_empty()) {
+ // Process users until they are all gone.
+ SDNode *U = *From->use_begin();
+
+ // This node is about to morph, remove its old self from the CSE maps.
+ RemoveNodeFromCSEMaps(U);
+
+ for (SDOperand *I = U->OperandList, *E = U->OperandList+U->NumOperands;
+ I != E; ++I)
+ if (I->Val == From) {
+ const SDOperand &ToOp = To[I->ResNo];
+ From->removeUser(U);
+ *I = ToOp;
+ ToOp.Val->addUser(U);
+ }
+
+ // Now that we have modified U, add it back to the CSE maps. If it already
+ // exists there, recursively merge the results together.
+ if (SDNode *Existing = AddNonLeafNodeToCSEMaps(U)) {
+ ReplaceAllUsesWith(U, Existing, Deleted);
+ // U is now dead.
+ if (Deleted) Deleted->push_back(U);
+ DeleteNodeNotInCSEMaps(U);
+ }
+ }
+}
+
+/// ReplaceAllUsesOfValueWith - Replace any uses of From with To, leaving
+/// uses of other values produced by From.Val alone. The Deleted vector is
+/// handled the same was as for ReplaceAllUsesWith.
+void SelectionDAG::ReplaceAllUsesOfValueWith(SDOperand From, SDOperand To,
+ std::vector<SDNode*> &Deleted) {
+ assert(From != To && "Cannot replace a value with itself");
+ // Handle the simple, trivial, case efficiently.
+ if (From.Val->getNumValues() == 1 && To.Val->getNumValues() == 1) {
+ ReplaceAllUsesWith(From, To, &Deleted);
+ return;
+ }
+
+ // Get all of the users of From.Val. We want these in a nice,
+ // deterministically ordered and uniqued set, so we use a SmallSetVector.
+ SmallSetVector<SDNode*, 16> Users(From.Val->use_begin(), From.Val->use_end());
+
+ while (!Users.empty()) {
+ // We know that this user uses some value of From. If it is the right
+ // value, update it.
+ SDNode *User = Users.back();
+ Users.pop_back();
+
+ for (SDOperand *Op = User->OperandList,
+ *E = User->OperandList+User->NumOperands; Op != E; ++Op) {
+ if (*Op == From) {
+ // Okay, we know this user needs to be updated. Remove its old self
+ // from the CSE maps.
+ RemoveNodeFromCSEMaps(User);
+
+ // Update all operands that match "From".
+ for (; Op != E; ++Op) {
+ if (*Op == From) {
+ From.Val->removeUser(User);
+ *Op = To;
+ To.Val->addUser(User);
+ }
+ }
+
+ // Now that we have modified User, add it back to the CSE maps. If it
+ // already exists there, recursively merge the results together.
+ if (SDNode *Existing = AddNonLeafNodeToCSEMaps(User)) {
+ unsigned NumDeleted = Deleted.size();
+ ReplaceAllUsesWith(User, Existing, &Deleted);
+
+ // User is now dead.
+ Deleted.push_back(User);
+ DeleteNodeNotInCSEMaps(User);
+
+ // We have to be careful here, because ReplaceAllUsesWith could have
+ // deleted a user of From, which means there may be dangling pointers
+ // in the "Users" setvector. Scan over the deleted node pointers and
+ // remove them from the setvector.
+ for (unsigned i = NumDeleted, e = Deleted.size(); i != e; ++i)
+ Users.remove(Deleted[i]);
+ }
+ break; // Exit the operand scanning loop.
+ }
+ }
+ }
+}
+
+
+/// AssignNodeIds - Assign a unique node id for each node in the DAG based on
+/// their allnodes order. It returns the maximum id.
+unsigned SelectionDAG::AssignNodeIds() {
+ unsigned Id = 0;
+ for (allnodes_iterator I = allnodes_begin(), E = allnodes_end(); I != E; ++I){
+ SDNode *N = I;
+ N->setNodeId(Id++);
+ }
+ return Id;
+}
+
+/// AssignTopologicalOrder - Assign a unique node id for each node in the DAG
+/// based on their topological order. It returns the maximum id and a vector
+/// of the SDNodes* in assigned order by reference.
+unsigned SelectionDAG::AssignTopologicalOrder(std::vector<SDNode*> &TopOrder) {
+ unsigned DAGSize = AllNodes.size();
+ std::vector<unsigned> InDegree(DAGSize);
+ std::vector<SDNode*> Sources;
+
+ // Use a two pass approach to avoid using a std::map which is slow.
+ unsigned Id = 0;
+ for (allnodes_iterator I = allnodes_begin(),E = allnodes_end(); I != E; ++I){
+ SDNode *N = I;
+ N->setNodeId(Id++);
+ unsigned Degree = N->use_size();
+ InDegree[N->getNodeId()] = Degree;
+ if (Degree == 0)
+ Sources.push_back(N);
+ }
+
+ TopOrder.clear();
+ while (!Sources.empty()) {
+ SDNode *N = Sources.back();
+ Sources.pop_back();
+ TopOrder.push_back(N);
+ for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ++I) {
+ SDNode *P = I->Val;
+ unsigned Degree = --InDegree[P->getNodeId()];
+ if (Degree == 0)
+ Sources.push_back(P);
+ }
+ }
+
+ // Second pass, assign the actual topological order as node ids.
+ Id = 0;
+ for (std::vector<SDNode*>::iterator TI = TopOrder.begin(),TE = TopOrder.end();
+ TI != TE; ++TI)
+ (*TI)->setNodeId(Id++);
+
+ return Id;
+}
+
+
+
+//===----------------------------------------------------------------------===//
+// SDNode Class
+//===----------------------------------------------------------------------===//
+
+// Out-of-line virtual method to give class a home.
+void SDNode::ANCHOR() {}
+void UnarySDNode::ANCHOR() {}
+void BinarySDNode::ANCHOR() {}
+void TernarySDNode::ANCHOR() {}
+void HandleSDNode::ANCHOR() {}
+void StringSDNode::ANCHOR() {}
+void ConstantSDNode::ANCHOR() {}
+void ConstantFPSDNode::ANCHOR() {}
+void GlobalAddressSDNode::ANCHOR() {}
+void FrameIndexSDNode::ANCHOR() {}
+void JumpTableSDNode::ANCHOR() {}
+void ConstantPoolSDNode::ANCHOR() {}
+void BasicBlockSDNode::ANCHOR() {}
+void SrcValueSDNode::ANCHOR() {}
+void RegisterSDNode::ANCHOR() {}
+void ExternalSymbolSDNode::ANCHOR() {}
+void CondCodeSDNode::ANCHOR() {}
+void VTSDNode::ANCHOR() {}
+void LoadSDNode::ANCHOR() {}
+void StoreSDNode::ANCHOR() {}
+
+HandleSDNode::~HandleSDNode() {
+ SDVTList VTs = { 0, 0 };
+ MorphNodeTo(ISD::HANDLENODE, VTs, 0, 0); // Drops operand uses.
+}
+
+GlobalAddressSDNode::GlobalAddressSDNode(bool isTarget, const GlobalValue *GA,
+ MVT::ValueType VT, int o)
+ : SDNode(isa<GlobalVariable>(GA) &&
+ dyn_cast<GlobalVariable>(GA)->isThreadLocal() ?
+ // Thread Local
+ (isTarget ? ISD::TargetGlobalTLSAddress : ISD::GlobalTLSAddress) :
+ // Non Thread Local
+ (isTarget ? ISD::TargetGlobalAddress : ISD::GlobalAddress),
+ getSDVTList(VT)), Offset(o) {
+ TheGlobal = const_cast<GlobalValue*>(GA);
+}
+
+/// Profile - Gather unique data for the node.
+///
+void SDNode::Profile(FoldingSetNodeID &ID) {
+ AddNodeIDNode(ID, this);
+}
+
+/// getValueTypeList - Return a pointer to the specified value type.
+///
+MVT::ValueType *SDNode::getValueTypeList(MVT::ValueType VT) {
+ static MVT::ValueType VTs[MVT::LAST_VALUETYPE];
+ VTs[VT] = VT;
+ return &VTs[VT];
+}
+
+/// hasNUsesOfValue - Return true if there are exactly NUSES uses of the
+/// indicated value. This method ignores uses of other values defined by this
+/// operation.
+bool SDNode::hasNUsesOfValue(unsigned NUses, unsigned Value) const {
+ assert(Value < getNumValues() && "Bad value!");
+
+ // If there is only one value, this is easy.
+ if (getNumValues() == 1)
+ return use_size() == NUses;
+ if (Uses.size() < NUses) return false;
+
+ SDOperand TheValue(const_cast<SDNode *>(this), Value);
+
+ SmallPtrSet<SDNode*, 32> UsersHandled;
+
+ for (SDNode::use_iterator UI = Uses.begin(), E = Uses.end(); UI != E; ++UI) {
+ SDNode *User = *UI;
+ if (User->getNumOperands() == 1 ||
+ UsersHandled.insert(User)) // First time we've seen this?
+ for (unsigned i = 0, e = User->getNumOperands(); i != e; ++i)
+ if (User->getOperand(i) == TheValue) {
+ if (NUses == 0)
+ return false; // too many uses
+ --NUses;
+ }
+ }
+
+ // Found exactly the right number of uses?
+ return NUses == 0;
+}
+
+
+/// isOnlyUse - Return true if this node is the only use of N.
+///
+bool SDNode::isOnlyUse(SDNode *N) const {
+ bool Seen = false;
+ for (SDNode::use_iterator I = N->use_begin(), E = N->use_end(); I != E; ++I) {
+ SDNode *User = *I;
+ if (User == this)
+ Seen = true;
+ else
+ return false;
+ }
+
+ return Seen;
+}
+
+/// isOperand - Return true if this node is an operand of N.
+///
+bool SDOperand::isOperand(SDNode *N) const {
+ for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i)
+ if (*this == N->getOperand(i))
+ return true;
+ return false;
+}
+
+bool SDNode::isOperand(SDNode *N) const {
+ for (unsigned i = 0, e = N->NumOperands; i != e; ++i)
+ if (this == N->OperandList[i].Val)
+ return true;
+ return false;
+}
+
+static void findPredecessor(SDNode *N, const SDNode *P, bool &found,
+ SmallPtrSet<SDNode *, 32> &Visited) {
+ if (found || !Visited.insert(N))
+ return;
+
+ for (unsigned i = 0, e = N->getNumOperands(); !found && i != e; ++i) {
+ SDNode *Op = N->getOperand(i).Val;
+ if (Op == P) {
+ found = true;
+ return;
+ }
+ findPredecessor(Op, P, found, Visited);
+ }
+}
+
+/// isPredecessor - Return true if this node is a predecessor of N. This node
+/// is either an operand of N or it can be reached by recursively traversing
+/// up the operands.
+/// NOTE: this is an expensive method. Use it carefully.
+bool SDNode::isPredecessor(SDNode *N) const {
+ SmallPtrSet<SDNode *, 32> Visited;
+ bool found = false;
+ findPredecessor(N, this, found, Visited);
+ return found;
+}
+
+uint64_t SDNode::getConstantOperandVal(unsigned Num) const {
+ assert(Num < NumOperands && "Invalid child # of SDNode!");
+ return cast<ConstantSDNode>(OperandList[Num])->getValue();
+}
+
+std::string SDNode::getOperationName(const SelectionDAG *G) const {
+ switch (getOpcode()) {
+ default:
+ if (getOpcode() < ISD::BUILTIN_OP_END)
+ return "<<Unknown DAG Node>>";
+ else {
+ if (G) {
+ if (const TargetInstrInfo *TII = G->getTarget().getInstrInfo())
+ if (getOpcode()-ISD::BUILTIN_OP_END < TII->getNumOpcodes())
+ return TII->getName(getOpcode()-ISD::BUILTIN_OP_END);
+
+ TargetLowering &TLI = G->getTargetLoweringInfo();
+ const char *Name =
+ TLI.getTargetNodeName(getOpcode());
+ if (Name) return Name;
+ }
+
+ return "<<Unknown Target Node>>";
+ }
+
+ case ISD::PCMARKER: return "PCMarker";
+ case ISD::READCYCLECOUNTER: return "ReadCycleCounter";
+ case ISD::SRCVALUE: return "SrcValue";
+ case ISD::EntryToken: return "EntryToken";
+ case ISD::TokenFactor: return "TokenFactor";
+ case ISD::AssertSext: return "AssertSext";
+ case ISD::AssertZext: return "AssertZext";
+
+ case ISD::STRING: return "String";
+ case ISD::BasicBlock: return "BasicBlock";
+ case ISD::VALUETYPE: return "ValueType";
+ case ISD::Register: return "Register";
+
+ case ISD::Constant: return "Constant";
+ case ISD::ConstantFP: return "ConstantFP";
+ case ISD::GlobalAddress: return "GlobalAddress";
+ case ISD::GlobalTLSAddress: return "GlobalTLSAddress";
+ case ISD::FrameIndex: return "FrameIndex";
+ case ISD::JumpTable: return "JumpTable";
+ case ISD::GLOBAL_OFFSET_TABLE: return "GLOBAL_OFFSET_TABLE";
+ case ISD::RETURNADDR: return "RETURNADDR";
+ case ISD::FRAMEADDR: return "FRAMEADDR";
+ case ISD::FRAME_TO_ARGS_OFFSET: return "FRAME_TO_ARGS_OFFSET";
+ case ISD::EXCEPTIONADDR: return "EXCEPTIONADDR";
+ case ISD::EHSELECTION: return "EHSELECTION";
+ case ISD::EH_RETURN: return "EH_RETURN";
+ case ISD::ConstantPool: return "ConstantPool";
+ case ISD::ExternalSymbol: return "ExternalSymbol";
+ case ISD::INTRINSIC_WO_CHAIN: {
+ unsigned IID = cast<ConstantSDNode>(getOperand(0))->getValue();
+ return Intrinsic::getName((Intrinsic::ID)IID);
+ }
+ case ISD::INTRINSIC_VOID:
+ case ISD::INTRINSIC_W_CHAIN: {
+ unsigned IID = cast<ConstantSDNode>(getOperand(1))->getValue();
+ return Intrinsic::getName((Intrinsic::ID)IID);
+ }
+
+ case ISD::BUILD_VECTOR: return "BUILD_VECTOR";
+ case ISD::TargetConstant: return "TargetConstant";
+ case ISD::TargetConstantFP:return "TargetConstantFP";
+ case ISD::TargetGlobalAddress: return "TargetGlobalAddress";
+ case ISD::TargetGlobalTLSAddress: return "TargetGlobalTLSAddress";
+ case ISD::TargetFrameIndex: return "TargetFrameIndex";
+ case ISD::TargetJumpTable: return "TargetJumpTable";
+ case ISD::TargetConstantPool: return "TargetConstantPool";
+ case ISD::TargetExternalSymbol: return "TargetExternalSymbol";
+
+ case ISD::CopyToReg: return "CopyToReg";
+ case ISD::CopyFromReg: return "CopyFromReg";
+ case ISD::UNDEF: return "undef";
+ case ISD::MERGE_VALUES: return "merge_values";
+ case ISD::INLINEASM: return "inlineasm";
+ case ISD::LABEL: return "label";
+ case ISD::HANDLENODE: return "handlenode";
+ case ISD::FORMAL_ARGUMENTS: return "formal_arguments";
+ case ISD::CALL: return "call";
+
+ // Unary operators
+ case ISD::FABS: return "fabs";
+ case ISD::FNEG: return "fneg";
+ case ISD::FSQRT: return "fsqrt";
+ case ISD::FSIN: return "fsin";
+ case ISD::FCOS: return "fcos";
+ case ISD::FPOWI: return "fpowi";
+
+ // Binary operators
+ case ISD::ADD: return "add";
+ case ISD::SUB: return "sub";
+ case ISD::MUL: return "mul";
+ case ISD::MULHU: return "mulhu";
+ case ISD::MULHS: return "mulhs";
+ case ISD::SDIV: return "sdiv";
+ case ISD::UDIV: return "udiv";
+ case ISD::SREM: return "srem";
+ case ISD::UREM: return "urem";
+ case ISD::AND: return "and";
+ case ISD::OR: return "or";
+ case ISD::XOR: return "xor";
+ case ISD::SHL: return "shl";
+ case ISD::SRA: return "sra";
+ case ISD::SRL: return "srl";
+ case ISD::ROTL: return "rotl";
+ case ISD::ROTR: return "rotr";
+ case ISD::FADD: return "fadd";
+ case ISD::FSUB: return "fsub";
+ case ISD::FMUL: return "fmul";
+ case ISD::FDIV: return "fdiv";
+ case ISD::FREM: return "frem";
+ case ISD::FCOPYSIGN: return "fcopysign";
+
+ case ISD::SETCC: return "setcc";
+ case ISD::SELECT: return "select";
+ case ISD::SELECT_CC: return "select_cc";
+ case ISD::INSERT_VECTOR_ELT: return "insert_vector_elt";
+ case ISD::EXTRACT_VECTOR_ELT: return "extract_vector_elt";
+ case ISD::CONCAT_VECTORS: return "concat_vectors";
+ case ISD::EXTRACT_SUBVECTOR: return "extract_subvector";
+ case ISD::SCALAR_TO_VECTOR: return "scalar_to_vector";
+ case ISD::VECTOR_SHUFFLE: return "vector_shuffle";
+ case ISD::CARRY_FALSE: return "carry_false";
+ case ISD::ADDC: return "addc";
+ case ISD::ADDE: return "adde";
+ case ISD::SUBC: return "subc";
+ case ISD::SUBE: return "sube";
+ case ISD::SHL_PARTS: return "shl_parts";
+ case ISD::SRA_PARTS: return "sra_parts";
+ case ISD::SRL_PARTS: return "srl_parts";
+
+ // Conversion operators.
+ case ISD::SIGN_EXTEND: return "sign_extend";
+ case ISD::ZERO_EXTEND: return "zero_extend";
+ case ISD::ANY_EXTEND: return "any_extend";
+ case ISD::SIGN_EXTEND_INREG: return "sign_extend_inreg";
+ case ISD::TRUNCATE: return "truncate";
+ case ISD::FP_ROUND: return "fp_round";
+ case ISD::FP_ROUND_INREG: return "fp_round_inreg";
+ case ISD::FP_EXTEND: return "fp_extend";
+
+ case ISD::SINT_TO_FP: return "sint_to_fp";
+ case ISD::UINT_TO_FP: return "uint_to_fp";
+ case ISD::FP_TO_SINT: return "fp_to_sint";
+ case ISD::FP_TO_UINT: return "fp_to_uint";
+ case ISD::BIT_CONVERT: return "bit_convert";
+
+ // Control flow instructions
+ case ISD::BR: return "br";
+ case ISD::BRIND: return "brind";
+ case ISD::BR_JT: return "br_jt";
+ case ISD::BRCOND: return "brcond";
+ case ISD::BR_CC: return "br_cc";
+ case ISD::RET: return "ret";
+ case ISD::CALLSEQ_START: return "callseq_start";
+ case ISD::CALLSEQ_END: return "callseq_end";
+
+ // Other operators
+ case ISD::LOAD: return "load";
+ case ISD::STORE: return "store";
+ case ISD::VAARG: return "vaarg";
+ case ISD::VACOPY: return "vacopy";
+ case ISD::VAEND: return "vaend";
+ case ISD::VASTART: return "vastart";
+ case ISD::DYNAMIC_STACKALLOC: return "dynamic_stackalloc";
+ case ISD::EXTRACT_ELEMENT: return "extract_element";
+ case ISD::BUILD_PAIR: return "build_pair";
+ case ISD::STACKSAVE: return "stacksave";
+ case ISD::STACKRESTORE: return "stackrestore";
+
+ // Block memory operations.
+ case ISD::MEMSET: return "memset";
+ case ISD::MEMCPY: return "memcpy";
+ case ISD::MEMMOVE: return "memmove";
+
+ // Bit manipulation
+ case ISD::BSWAP: return "bswap";
+ case ISD::CTPOP: return "ctpop";
+ case ISD::CTTZ: return "cttz";
+ case ISD::CTLZ: return "ctlz";
+
+ // Debug info
+ case ISD::LOCATION: return "location";
+ case ISD::DEBUG_LOC: return "debug_loc";
+
+ case ISD::CONDCODE:
+ switch (cast<CondCodeSDNode>(this)->get()) {
+ default: assert(0 && "Unknown setcc condition!");
+ case ISD::SETOEQ: return "setoeq";
+ case ISD::SETOGT: return "setogt";
+ case ISD::SETOGE: return "setoge";
+ case ISD::SETOLT: return "setolt";
+ case ISD::SETOLE: return "setole";
+ case ISD::SETONE: return "setone";
+
+ case ISD::SETO: return "seto";
+ case ISD::SETUO: return "setuo";
+ case ISD::SETUEQ: return "setue";
+ case ISD::SETUGT: return "setugt";
+ case ISD::SETUGE: return "setuge";
+ case ISD::SETULT: return "setult";
+ case ISD::SETULE: return "setule";
+ case ISD::SETUNE: return "setune";
+
+ case ISD::SETEQ: return "seteq";
+ case ISD::SETGT: return "setgt";
+ case ISD::SETGE: return "setge";
+ case ISD::SETLT: return "setlt";
+ case ISD::SETLE: return "setle";
+ case ISD::SETNE: return "setne";
+ }
+ }
+}
+
+const char *SDNode::getIndexedModeName(ISD::MemIndexedMode AM) {
+ switch (AM) {
+ default:
+ return "";
+ case ISD::PRE_INC:
+ return "<pre-inc>";
+ case ISD::PRE_DEC:
+ return "<pre-dec>";
+ case ISD::POST_INC:
+ return "<post-inc>";
+ case ISD::POST_DEC:
+ return "<post-dec>";
+ }
+}
+
+void SDNode::dump() const { dump(0); }
+void SDNode::dump(const SelectionDAG *G) const {
+ cerr << (void*)this << ": ";
+
+ for (unsigned i = 0, e = getNumValues(); i != e; ++i) {
+ if (i) cerr << ",";
+ if (getValueType(i) == MVT::Other)
+ cerr << "ch";
+ else
+ cerr << MVT::getValueTypeString(getValueType(i));
+ }
+ cerr << " = " << getOperationName(G);
+
+ cerr << " ";
+ for (unsigned i = 0, e = getNumOperands(); i != e; ++i) {
+ if (i) cerr << ", ";
+ cerr << (void*)getOperand(i).Val;
+ if (unsigned RN = getOperand(i).ResNo)
+ cerr << ":" << RN;
+ }
+
+ if (const ConstantSDNode *CSDN = dyn_cast<ConstantSDNode>(this)) {
+ cerr << "<" << CSDN->getValue() << ">";
+ } else if (const ConstantFPSDNode *CSDN = dyn_cast<ConstantFPSDNode>(this)) {
+ cerr << "<" << CSDN->getValue() << ">";
+ } else if (const GlobalAddressSDNode *GADN =
+ dyn_cast<GlobalAddressSDNode>(this)) {
+ int offset = GADN->getOffset();
+ cerr << "<";
+ WriteAsOperand(*cerr.stream(), GADN->getGlobal()) << ">";
+ if (offset > 0)
+ cerr << " + " << offset;
+ else
+ cerr << " " << offset;
+ } else if (const FrameIndexSDNode *FIDN = dyn_cast<FrameIndexSDNode>(this)) {
+ cerr << "<" << FIDN->getIndex() << ">";
+ } else if (const JumpTableSDNode *JTDN = dyn_cast<JumpTableSDNode>(this)) {
+ cerr << "<" << JTDN->getIndex() << ">";
+ } else if (const ConstantPoolSDNode *CP = dyn_cast<ConstantPoolSDNode>(this)){
+ int offset = CP->getOffset();
+ if (CP->isMachineConstantPoolEntry())
+ cerr << "<" << *CP->getMachineCPVal() << ">";
+ else
+ cerr << "<" << *CP->getConstVal() << ">";
+ if (offset > 0)
+ cerr << " + " << offset;
+ else
+ cerr << " " << offset;
+ } else if (const BasicBlockSDNode *BBDN = dyn_cast<BasicBlockSDNode>(this)) {
+ cerr << "<";
+ const Value *LBB = (const Value*)BBDN->getBasicBlock()->getBasicBlock();
+ if (LBB)
+ cerr << LBB->getName() << " ";
+ cerr << (const void*)BBDN->getBasicBlock() << ">";
+ } else if (const RegisterSDNode *R = dyn_cast<RegisterSDNode>(this)) {
+ if (G && R->getReg() && MRegisterInfo::isPhysicalRegister(R->getReg())) {
+ cerr << " " <<G->getTarget().getRegisterInfo()->getName(R->getReg());
+ } else {
+ cerr << " #" << R->getReg();
+ }
+ } else if (const ExternalSymbolSDNode *ES =
+ dyn_cast<ExternalSymbolSDNode>(this)) {
+ cerr << "'" << ES->getSymbol() << "'";
+ } else if (const SrcValueSDNode *M = dyn_cast<SrcValueSDNode>(this)) {
+ if (M->getValue())
+ cerr << "<" << M->getValue() << ":" << M->getOffset() << ">";
+ else
+ cerr << "<null:" << M->getOffset() << ">";
+ } else if (const VTSDNode *N = dyn_cast<VTSDNode>(this)) {
+ cerr << ":" << MVT::getValueTypeString(N->getVT());
+ } else if (const LoadSDNode *LD = dyn_cast<LoadSDNode>(this)) {
+ bool doExt = true;
+ switch (LD->getExtensionType()) {
+ default: doExt = false; break;
+ case ISD::EXTLOAD:
+ cerr << " <anyext ";
+ break;
+ case ISD::SEXTLOAD:
+ cerr << " <sext ";
+ break;
+ case ISD::ZEXTLOAD:
+ cerr << " <zext ";
+ break;
+ }
+ if (doExt)
+ cerr << MVT::getValueTypeString(LD->getLoadedVT()) << ">";
+
+ const char *AM = getIndexedModeName(LD->getAddressingMode());
+ if (AM != "")
+ cerr << " " << AM;
+ } else if (const StoreSDNode *ST = dyn_cast<StoreSDNode>(this)) {
+ if (ST->isTruncatingStore())
+ cerr << " <trunc "
+ << MVT::getValueTypeString(ST->getStoredVT()) << ">";
+
+ const char *AM = getIndexedModeName(ST->getAddressingMode());
+ if (AM != "")
+ cerr << " " << AM;
+ }
+}
+
+static void DumpNodes(const SDNode *N, unsigned indent, const SelectionDAG *G) {
+ for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i)
+ if (N->getOperand(i).Val->hasOneUse())
+ DumpNodes(N->getOperand(i).Val, indent+2, G);
+ else
+ cerr << "\n" << std::string(indent+2, ' ')
+ << (void*)N->getOperand(i).Val << ": <multiple use>";
+
+
+ cerr << "\n" << std::string(indent, ' ');
+ N->dump(G);
+}
+
+void SelectionDAG::dump() const {
+ cerr << "SelectionDAG has " << AllNodes.size() << " nodes:";
+ std::vector<const SDNode*> Nodes;
+ for (allnodes_const_iterator I = allnodes_begin(), E = allnodes_end();
+ I != E; ++I)
+ Nodes.push_back(I);
+
+ std::sort(Nodes.begin(), Nodes.end());
+
+ for (unsigned i = 0, e = Nodes.size(); i != e; ++i) {
+ if (!Nodes[i]->hasOneUse() && Nodes[i] != getRoot().Val)
+ DumpNodes(Nodes[i], 2, this);
+ }
+
+ if (getRoot().Val) DumpNodes(getRoot().Val, 2, this);
+
+ cerr << "\n\n";
+}
+
+const Type *ConstantPoolSDNode::getType() const {
+ if (isMachineConstantPoolEntry())
+ return Val.MachineCPVal->getType();
+ return Val.ConstVal->getType();
+}
diff --git a/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp b/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
new file mode 100644
index 0000000..9256faa
--- /dev/null
+++ b/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
@@ -0,0 +1,4904 @@
+//===-- SelectionDAGISel.cpp - Implement the SelectionDAGISel class -------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file was developed by the LLVM research group and is distributed under
+// the University of Illinois Open Source License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This implements the SelectionDAGISel class.
+//
+//===----------------------------------------------------------------------===//
+
+#define DEBUG_TYPE "isel"
+#include "llvm/ADT/BitVector.h"
+#include "llvm/Analysis/AliasAnalysis.h"
+#include "llvm/CodeGen/SelectionDAGISel.h"
+#include "llvm/CodeGen/ScheduleDAG.h"
+#include "llvm/Constants.h"
+#include "llvm/CallingConv.h"
+#include "llvm/DerivedTypes.h"
+#include "llvm/Function.h"
+#include "llvm/GlobalVariable.h"
+#include "llvm/InlineAsm.h"
+#include "llvm/Instructions.h"
+#include "llvm/Intrinsics.h"
+#include "llvm/IntrinsicInst.h"
+#include "llvm/ParameterAttributes.h"
+#include "llvm/CodeGen/MachineModuleInfo.h"
+#include "llvm/CodeGen/MachineFunction.h"
+#include "llvm/CodeGen/MachineFrameInfo.h"
+#include "llvm/CodeGen/MachineJumpTableInfo.h"
+#include "llvm/CodeGen/MachineInstrBuilder.h"
+#include "llvm/CodeGen/SchedulerRegistry.h"
+#include "llvm/CodeGen/SelectionDAG.h"
+#include "llvm/CodeGen/SSARegMap.h"
+#include "llvm/Target/MRegisterInfo.h"
+#include "llvm/Target/TargetData.h"
+#include "llvm/Target/TargetFrameInfo.h"
+#include "llvm/Target/TargetInstrInfo.h"
+#include "llvm/Target/TargetLowering.h"
+#include "llvm/Target/TargetMachine.h"
+#include "llvm/Target/TargetOptions.h"
+#include "llvm/Support/MathExtras.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/Compiler.h"
+#include <algorithm>
+using namespace llvm;
+
+#ifndef NDEBUG
+static cl::opt<bool>
+ViewISelDAGs("view-isel-dags", cl::Hidden,
+ cl::desc("Pop up a window to show isel dags as they are selected"));
+static cl::opt<bool>
+ViewSchedDAGs("view-sched-dags", cl::Hidden,
+ cl::desc("Pop up a window to show sched dags as they are processed"));
+#else
+static const bool ViewISelDAGs = 0, ViewSchedDAGs = 0;
+#endif
+
+//===---------------------------------------------------------------------===//
+///
+/// RegisterScheduler class - Track the registration of instruction schedulers.
+///
+//===---------------------------------------------------------------------===//
+MachinePassRegistry RegisterScheduler::Registry;
+
+//===---------------------------------------------------------------------===//
+///
+/// ISHeuristic command line option for instruction schedulers.
+///
+//===---------------------------------------------------------------------===//
+namespace {
+ cl::opt<RegisterScheduler::FunctionPassCtor, false,
+ RegisterPassParser<RegisterScheduler> >
+ ISHeuristic("pre-RA-sched",
+ cl::init(&createDefaultScheduler),
+ cl::desc("Instruction schedulers available (before register allocation):"));
+
+ static RegisterScheduler
+ defaultListDAGScheduler("default", " Best scheduler for the target",
+ createDefaultScheduler);
+} // namespace
+
+namespace { struct AsmOperandInfo; }
+
+namespace {
+ /// RegsForValue - This struct represents the physical registers that a
+ /// particular value is assigned and the type information about the value.
+ /// This is needed because values can be promoted into larger registers and
+ /// expanded into multiple smaller registers than the value.
+ struct VISIBILITY_HIDDEN RegsForValue {
+ /// Regs - This list holds the register (for legal and promoted values)
+ /// or register set (for expanded values) that the value should be assigned
+ /// to.
+ std::vector<unsigned> Regs;
+
+ /// RegVT - The value type of each register.
+ ///
+ MVT::ValueType RegVT;
+
+ /// ValueVT - The value type of the LLVM value, which may be promoted from
+ /// RegVT or made from merging the two expanded parts.
+ MVT::ValueType ValueVT;
+
+ RegsForValue() : RegVT(MVT::Other), ValueVT(MVT::Other) {}
+
+ RegsForValue(unsigned Reg, MVT::ValueType regvt, MVT::ValueType valuevt)
+ : RegVT(regvt), ValueVT(valuevt) {
+ Regs.push_back(Reg);
+ }
+ RegsForValue(const std::vector<unsigned> ®s,
+ MVT::ValueType regvt, MVT::ValueType valuevt)
+ : Regs(regs), RegVT(regvt), ValueVT(valuevt) {
+ }
+
+ /// getCopyFromRegs - Emit a series of CopyFromReg nodes that copies from
+ /// this value and returns the result as a ValueVT value. This uses
+ /// Chain/Flag as the input and updates them for the output Chain/Flag.
+ /// If the Flag pointer is NULL, no flag is used.
+ SDOperand getCopyFromRegs(SelectionDAG &DAG,
+ SDOperand &Chain, SDOperand *Flag) const;
+
+ /// getCopyToRegs - Emit a series of CopyToReg nodes that copies the
+ /// specified value into the registers specified by this object. This uses
+ /// Chain/Flag as the input and updates them for the output Chain/Flag.
+ /// If the Flag pointer is NULL, no flag is used.
+ void getCopyToRegs(SDOperand Val, SelectionDAG &DAG,
+ SDOperand &Chain, SDOperand *Flag) const;
+
+ /// AddInlineAsmOperands - Add this value to the specified inlineasm node
+ /// operand list. This adds the code marker and includes the number of
+ /// values added into it.
+ void AddInlineAsmOperands(unsigned Code, SelectionDAG &DAG,
+ std::vector<SDOperand> &Ops) const;
+ };
+}
+
+namespace llvm {
+ //===--------------------------------------------------------------------===//
+ /// createDefaultScheduler - This creates an instruction scheduler appropriate
+ /// for the target.
+ ScheduleDAG* createDefaultScheduler(SelectionDAGISel *IS,
+ SelectionDAG *DAG,
+ MachineBasicBlock *BB) {
+ TargetLowering &TLI = IS->getTargetLowering();
+
+ if (TLI.getSchedulingPreference() == TargetLowering::SchedulingForLatency) {
+ return createTDListDAGScheduler(IS, DAG, BB);
+ } else {
+ assert(TLI.getSchedulingPreference() ==
+ TargetLowering::SchedulingForRegPressure && "Unknown sched type!");
+ return createBURRListDAGScheduler(IS, DAG, BB);
+ }
+ }
+
+
+ //===--------------------------------------------------------------------===//
+ /// FunctionLoweringInfo - This contains information that is global to a
+ /// function that is used when lowering a region of the function.
+ class FunctionLoweringInfo {
+ public:
+ TargetLowering &TLI;
+ Function &Fn;
+ MachineFunction &MF;
+ SSARegMap *RegMap;
+
+ FunctionLoweringInfo(TargetLowering &TLI, Function &Fn,MachineFunction &MF);
+
+ /// MBBMap - A mapping from LLVM basic blocks to their machine code entry.
+ std::map<const BasicBlock*, MachineBasicBlock *> MBBMap;
+
+ /// ValueMap - Since we emit code for the function a basic block at a time,
+ /// we must remember which virtual registers hold the values for
+ /// cross-basic-block values.
+ DenseMap<const Value*, unsigned> ValueMap;
+
+ /// StaticAllocaMap - Keep track of frame indices for fixed sized allocas in
+ /// the entry block. This allows the allocas to be efficiently referenced
+ /// anywhere in the function.
+ std::map<const AllocaInst*, int> StaticAllocaMap;
+
+#ifndef NDEBUG
+ SmallSet<Instruction*, 8> CatchInfoLost;
+ SmallSet<Instruction*, 8> CatchInfoFound;
+#endif
+
+ unsigned MakeReg(MVT::ValueType VT) {
+ return RegMap->createVirtualRegister(TLI.getRegClassFor(VT));
+ }
+
+ /// isExportedInst - Return true if the specified value is an instruction
+ /// exported from its block.
+ bool isExportedInst(const Value *V) {
+ return ValueMap.count(V);
+ }
+
+ unsigned CreateRegForValue(const Value *V);
+
+ unsigned InitializeRegForValue(const Value *V) {
+ unsigned &R = ValueMap[V];
+ assert(R == 0 && "Already initialized this value register!");
+ return R = CreateRegForValue(V);
+ }
+ };
+}
+
+/// isSelector - Return true if this instruction is a call to the
+/// eh.selector intrinsic.
+static bool isSelector(Instruction *I) {
+ if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(I))
+ return II->getIntrinsicID() == Intrinsic::eh_selector;
+ return false;
+}
+
+/// isUsedOutsideOfDefiningBlock - Return true if this instruction is used by
+/// PHI nodes or outside of the basic block that defines it, or used by a
+/// switch instruction, which may expand to multiple basic blocks.
+static bool isUsedOutsideOfDefiningBlock(Instruction *I) {
+ if (isa<PHINode>(I)) return true;
+ BasicBlock *BB = I->getParent();
+ for (Value::use_iterator UI = I->use_begin(), E = I->use_end(); UI != E; ++UI)
+ if (cast<Instruction>(*UI)->getParent() != BB || isa<PHINode>(*UI) ||
+ // FIXME: Remove switchinst special case.
+ isa<SwitchInst>(*UI))
+ return true;
+ return false;
+}
+
+/// isOnlyUsedInEntryBlock - If the specified argument is only used in the
+/// entry block, return true. This includes arguments used by switches, since
+/// the switch may expand into multiple basic blocks.
+static bool isOnlyUsedInEntryBlock(Argument *A) {
+ BasicBlock *Entry = A->getParent()->begin();
+ for (Value::use_iterator UI = A->use_begin(), E = A->use_end(); UI != E; ++UI)
+ if (cast<Instruction>(*UI)->getParent() != Entry || isa<SwitchInst>(*UI))
+ return false; // Use not in entry block.
+ return true;
+}
+
+FunctionLoweringInfo::FunctionLoweringInfo(TargetLowering &tli,
+ Function &fn, MachineFunction &mf)
+ : TLI(tli), Fn(fn), MF(mf), RegMap(MF.getSSARegMap()) {
+
+ // Create a vreg for each argument register that is not dead and is used
+ // outside of the entry block for the function.
+ for (Function::arg_iterator AI = Fn.arg_begin(), E = Fn.arg_end();
+ AI != E; ++AI)
+ if (!isOnlyUsedInEntryBlock(AI))
+ InitializeRegForValue(AI);
+
+ // Initialize the mapping of values to registers. This is only set up for
+ // instruction values that are used outside of the block that defines
+ // them.
+ Function::iterator BB = Fn.begin(), EB = Fn.end();
+ for (BasicBlock::iterator I = BB->begin(), E = BB->end(); I != E; ++I)
+ if (AllocaInst *AI = dyn_cast<AllocaInst>(I))
+ if (ConstantInt *CUI = dyn_cast<ConstantInt>(AI->getArraySize())) {
+ const Type *Ty = AI->getAllocatedType();
+ uint64_t TySize = TLI.getTargetData()->getTypeSize(Ty);
+ unsigned Align =
+ std::max((unsigned)TLI.getTargetData()->getPrefTypeAlignment(Ty),
+ AI->getAlignment());
+
+ TySize *= CUI->getZExtValue(); // Get total allocated size.
+ if (TySize == 0) TySize = 1; // Don't create zero-sized stack objects.
+ StaticAllocaMap[AI] =
+ MF.getFrameInfo()->CreateStackObject(TySize, Align);
+ }
+
+ for (; BB != EB; ++BB)
+ for (BasicBlock::iterator I = BB->begin(), E = BB->end(); I != E; ++I)
+ if (!I->use_empty() && isUsedOutsideOfDefiningBlock(I))
+ if (!isa<AllocaInst>(I) ||
+ !StaticAllocaMap.count(cast<AllocaInst>(I)))
+ InitializeRegForValue(I);
+
+ // Create an initial MachineBasicBlock for each LLVM BasicBlock in F. This
+ // also creates the initial PHI MachineInstrs, though none of the input
+ // operands are populated.
+ for (BB = Fn.begin(), EB = Fn.end(); BB != EB; ++BB) {
+ MachineBasicBlock *MBB = new MachineBasicBlock(BB);
+ MBBMap[BB] = MBB;
+ MF.getBasicBlockList().push_back(MBB);
+
+ // Create Machine PHI nodes for LLVM PHI nodes, lowering them as
+ // appropriate.
+ PHINode *PN;
+ for (BasicBlock::iterator I = BB->begin();(PN = dyn_cast<PHINode>(I)); ++I){
+ if (PN->use_empty()) continue;
+
+ MVT::ValueType VT = TLI.getValueType(PN->getType());
+ unsigned NumRegisters = TLI.getNumRegisters(VT);
+ unsigned PHIReg = ValueMap[PN];
+ assert(PHIReg && "PHI node does not have an assigned virtual register!");
+ const TargetInstrInfo *TII = TLI.getTargetMachine().getInstrInfo();
+ for (unsigned i = 0; i != NumRegisters; ++i)
+ BuildMI(MBB, TII->get(TargetInstrInfo::PHI), PHIReg+i);
+ }
+ }
+}
+
+/// CreateRegForValue - Allocate the appropriate number of virtual registers of
+/// the correctly promoted or expanded types. Assign these registers
+/// consecutive vreg numbers and return the first assigned number.
+unsigned FunctionLoweringInfo::CreateRegForValue(const Value *V) {
+ MVT::ValueType VT = TLI.getValueType(V->getType());
+
+ unsigned NumRegisters = TLI.getNumRegisters(VT);
+ MVT::ValueType RegisterVT = TLI.getRegisterType(VT);
+
+ unsigned R = MakeReg(RegisterVT);
+ for (unsigned i = 1; i != NumRegisters; ++i)
+ MakeReg(RegisterVT);
+
+ return R;
+}
+
+//===----------------------------------------------------------------------===//
+/// SelectionDAGLowering - This is the common target-independent lowering
+/// implementation that is parameterized by a TargetLowering object.
+/// Also, targets can overload any lowering method.
+///
+namespace llvm {
+class SelectionDAGLowering {
+ MachineBasicBlock *CurMBB;
+
+ DenseMap<const Value*, SDOperand> NodeMap;
+
+ /// PendingLoads - Loads are not emitted to the program immediately. We bunch
+ /// them up and then emit token factor nodes when possible. This allows us to
+ /// get simple disambiguation between loads without worrying about alias
+ /// analysis.
+ std::vector<SDOperand> PendingLoads;
+
+ /// Case - A struct to record the Value for a switch case, and the
+ /// case's target basic block.
+ struct Case {
+ Constant* Low;
+ Constant* High;
+ MachineBasicBlock* BB;
+
+ Case() : Low(0), High(0), BB(0) { }
+ Case(Constant* low, Constant* high, MachineBasicBlock* bb) :
+ Low(low), High(high), BB(bb) { }
+ uint64_t size() const {
+ uint64_t rHigh = cast<ConstantInt>(High)->getSExtValue();
+ uint64_t rLow = cast<ConstantInt>(Low)->getSExtValue();
+ return (rHigh - rLow + 1ULL);
+ }
+ };
+
+ struct CaseBits {
+ uint64_t Mask;
+ MachineBasicBlock* BB;
+ unsigned Bits;
+
+ CaseBits(uint64_t mask, MachineBasicBlock* bb, unsigned bits):
+ Mask(mask), BB(bb), Bits(bits) { }
+ };
+
+ typedef std::vector<Case> CaseVector;
+ typedef std::vector<CaseBits> CaseBitsVector;
+ typedef CaseVector::iterator CaseItr;
+ typedef std::pair<CaseItr, CaseItr> CaseRange;
+
+ /// CaseRec - A struct with ctor used in lowering switches to a binary tree
+ /// of conditional branches.
+ struct CaseRec {
+ CaseRec(MachineBasicBlock *bb, Constant *lt, Constant *ge, CaseRange r) :
+ CaseBB(bb), LT(lt), GE(ge), Range(r) {}
+
+ /// CaseBB - The MBB in which to emit the compare and branch
+ MachineBasicBlock *CaseBB;
+ /// LT, GE - If nonzero, we know the current case value must be less-than or
+ /// greater-than-or-equal-to these Constants.
+ Constant *LT;
+ Constant *GE;
+ /// Range - A pair of iterators representing the range of case values to be
+ /// processed at this point in the binary search tree.
+ CaseRange Range;
+ };
+
+ typedef std::vector<CaseRec> CaseRecVector;
+
+ /// The comparison function for sorting the switch case values in the vector.
+ /// WARNING: Case ranges should be disjoint!
+ struct CaseCmp {
+ bool operator () (const Case& C1, const Case& C2) {
+ assert(isa<ConstantInt>(C1.Low) && isa<ConstantInt>(C2.High));
+ const ConstantInt* CI1 = cast<const ConstantInt>(C1.Low);
+ const ConstantInt* CI2 = cast<const ConstantInt>(C2.High);
+ return CI1->getValue().slt(CI2->getValue());
+ }
+ };
+
+ struct CaseBitsCmp {
+ bool operator () (const CaseBits& C1, const CaseBits& C2) {
+ return C1.Bits > C2.Bits;
+ }
+ };
+
+ unsigned Clusterify(CaseVector& Cases, const SwitchInst &SI);
+
+public:
+ // TLI - This is information that describes the available target features we
+ // need for lowering. This indicates when operations are unavailable,
+ // implemented with a libcall, etc.
+ TargetLowering &TLI;
+ SelectionDAG &DAG;
+ const TargetData *TD;
+
+ /// SwitchCases - Vector of CaseBlock structures used to communicate
+ /// SwitchInst code generation information.
+ std::vector<SelectionDAGISel::CaseBlock> SwitchCases;
+ /// JTCases - Vector of JumpTable structures used to communicate
+ /// SwitchInst code generation information.
+ std::vector<SelectionDAGISel::JumpTableBlock> JTCases;
+ std::vector<SelectionDAGISel::BitTestBlock> BitTestCases;
+
+ /// FuncInfo - Information about the function as a whole.
+ ///
+ FunctionLoweringInfo &FuncInfo;
+
+ SelectionDAGLowering(SelectionDAG &dag, TargetLowering &tli,
+ FunctionLoweringInfo &funcinfo)
+ : TLI(tli), DAG(dag), TD(DAG.getTarget().getTargetData()),
+ FuncInfo(funcinfo) {
+ }
+
+ /// getRoot - Return the current virtual root of the Selection DAG.
+ ///
+ SDOperand getRoot() {
+ if (PendingLoads.empty())
+ return DAG.getRoot();
+
+ if (PendingLoads.size() == 1) {
+ SDOperand Root = PendingLoads[0];
+ DAG.setRoot(Root);
+ PendingLoads.clear();
+ return Root;
+ }
+
+ // Otherwise, we have to make a token factor node.
+ SDOperand Root = DAG.getNode(ISD::TokenFactor, MVT::Other,
+ &PendingLoads[0], PendingLoads.size());
+ PendingLoads.clear();
+ DAG.setRoot(Root);
+ return Root;
+ }
+
+ SDOperand CopyValueToVirtualRegister(Value *V, unsigned Reg);
+
+ void visit(Instruction &I) { visit(I.getOpcode(), I); }
+
+ void visit(unsigned Opcode, User &I) {
+ // Note: this doesn't use InstVisitor, because it has to work with
+ // ConstantExpr's in addition to instructions.
+ switch (Opcode) {
+ default: assert(0 && "Unknown instruction type encountered!");
+ abort();
+ // Build the switch statement using the Instruction.def file.
+#define HANDLE_INST(NUM, OPCODE, CLASS) \
+ case Instruction::OPCODE:return visit##OPCODE((CLASS&)I);
+#include "llvm/Instruction.def"
+ }
+ }
+
+ void setCurrentBasicBlock(MachineBasicBlock *MBB) { CurMBB = MBB; }
+
+ SDOperand getLoadFrom(const Type *Ty, SDOperand Ptr,
+ const Value *SV, SDOperand Root,
+ bool isVolatile, unsigned Alignment);
+
+ SDOperand getIntPtrConstant(uint64_t Val) {
+ return DAG.getConstant(Val, TLI.getPointerTy());
+ }
+
+ SDOperand getValue(const Value *V);
+
+ void setValue(const Value *V, SDOperand NewN) {
+ SDOperand &N = NodeMap[V];
+ assert(N.Val == 0 && "Already set a value for this node!");
+ N = NewN;
+ }
+
+ void GetRegistersForValue(AsmOperandInfo &OpInfo, bool HasEarlyClobber,
+ std::set<unsigned> &OutputRegs,
+ std::set<unsigned> &InputRegs);
+
+ void FindMergedConditions(Value *Cond, MachineBasicBlock *TBB,
+ MachineBasicBlock *FBB, MachineBasicBlock *CurBB,
+ unsigned Opc);
+ bool isExportableFromCurrentBlock(Value *V, const BasicBlock *FromBB);
+ void ExportFromCurrentBlock(Value *V);
+ void LowerCallTo(Instruction &I,
+ const Type *CalledValueTy, unsigned CallingConv,
+ bool IsTailCall, SDOperand Callee, unsigned OpIdx,
+ MachineBasicBlock *LandingPad = NULL);
+
+ // Terminator instructions.
+ void visitRet(ReturnInst &I);
+ void visitBr(BranchInst &I);
+ void visitSwitch(SwitchInst &I);
+ void visitUnreachable(UnreachableInst &I) { /* noop */ }
+
+ // Helpers for visitSwitch
+ bool handleSmallSwitchRange(CaseRec& CR,
+ CaseRecVector& WorkList,
+ Value* SV,
+ MachineBasicBlock* Default);
+ bool handleJTSwitchCase(CaseRec& CR,
+ CaseRecVector& WorkList,
+ Value* SV,
+ MachineBasicBlock* Default);
+ bool handleBTSplitSwitchCase(CaseRec& CR,
+ CaseRecVector& WorkList,
+ Value* SV,
+ MachineBasicBlock* Default);
+ bool handleBitTestsSwitchCase(CaseRec& CR,
+ CaseRecVector& WorkList,
+ Value* SV,
+ MachineBasicBlock* Default);
+ void visitSwitchCase(SelectionDAGISel::CaseBlock &CB);
+ void visitBitTestHeader(SelectionDAGISel::BitTestBlock &B);
+ void visitBitTestCase(MachineBasicBlock* NextMBB,
+ unsigned Reg,
+ SelectionDAGISel::BitTestCase &B);
+ void visitJumpTable(SelectionDAGISel::JumpTable &JT);
+ void visitJumpTableHeader(SelectionDAGISel::JumpTable &JT,
+ SelectionDAGISel::JumpTableHeader &JTH);
+
+ // These all get lowered before this pass.
+ void visitInvoke(InvokeInst &I);
+ void visitUnwind(UnwindInst &I);
+
+ void visitBinary(User &I, unsigned OpCode);
+ void visitShift(User &I, unsigned Opcode);
+ void visitAdd(User &I) {
+ if (I.getType()->isFPOrFPVector())
+ visitBinary(I, ISD::FADD);
+ else
+ visitBinary(I, ISD::ADD);
+ }
+ void visitSub(User &I);
+ void visitMul(User &I) {
+ if (I.getType()->isFPOrFPVector())
+ visitBinary(I, ISD::FMUL);
+ else
+ visitBinary(I, ISD::MUL);
+ }
+ void visitURem(User &I) { visitBinary(I, ISD::UREM); }
+ void visitSRem(User &I) { visitBinary(I, ISD::SREM); }
+ void visitFRem(User &I) { visitBinary(I, ISD::FREM); }
+ void visitUDiv(User &I) { visitBinary(I, ISD::UDIV); }
+ void visitSDiv(User &I) { visitBinary(I, ISD::SDIV); }
+ void visitFDiv(User &I) { visitBinary(I, ISD::FDIV); }
+ void visitAnd (User &I) { visitBinary(I, ISD::AND); }
+ void visitOr (User &I) { visitBinary(I, ISD::OR); }
+ void visitXor (User &I) { visitBinary(I, ISD::XOR); }
+ void visitShl (User &I) { visitShift(I, ISD::SHL); }
+ void visitLShr(User &I) { visitShift(I, ISD::SRL); }
+ void visitAShr(User &I) { visitShift(I, ISD::SRA); }
+ void visitICmp(User &I);
+ void visitFCmp(User &I);
+ // Visit the conversion instructions
+ void visitTrunc(User &I);
+ void visitZExt(User &I);
+ void visitSExt(User &I);
+ void visitFPTrunc(User &I);
+ void visitFPExt(User &I);
+ void visitFPToUI(User &I);
+ void visitFPToSI(User &I);
+ void visitUIToFP(User &I);
+ void visitSIToFP(User &I);
+ void visitPtrToInt(User &I);
+ void visitIntToPtr(User &I);
+ void visitBitCast(User &I);
+
+ void visitExtractElement(User &I);
+ void visitInsertElement(User &I);
+ void visitShuffleVector(User &I);
+
+ void visitGetElementPtr(User &I);
+ void visitSelect(User &I);
+
+ void visitMalloc(MallocInst &I);
+ void visitFree(FreeInst &I);
+ void visitAlloca(AllocaInst &I);
+ void visitLoad(LoadInst &I);
+ void visitStore(StoreInst &I);
+ void visitPHI(PHINode &I) { } // PHI nodes are handled specially.
+ void visitCall(CallInst &I);
+ void visitInlineAsm(CallInst &I);
+ const char *visitIntrinsicCall(CallInst &I, unsigned Intrinsic);
+ void visitTargetIntrinsic(CallInst &I, unsigned Intrinsic);
+
+ void visitVAStart(CallInst &I);
+ void visitVAArg(VAArgInst &I);
+ void visitVAEnd(CallInst &I);
+ void visitVACopy(CallInst &I);
+
+ void visitMemIntrinsic(CallInst &I, unsigned Op);
+
+ void visitUserOp1(Instruction &I) {
+ assert(0 && "UserOp1 should not exist at instruction selection time!");
+ abort();
+ }
+ void visitUserOp2(Instruction &I) {
+ assert(0 && "UserOp2 should not exist at instruction selection time!");
+ abort();
+ }
+};
+} // end namespace llvm
+
+
+/// getCopyFromParts - Create a value that contains the
+/// specified legal parts combined into the value they represent.
+static SDOperand getCopyFromParts(SelectionDAG &DAG,
+ const SDOperand *Parts,
+ unsigned NumParts,
+ MVT::ValueType PartVT,
+ MVT::ValueType ValueVT,
+ ISD::NodeType AssertOp = ISD::DELETED_NODE) {
+ if (!MVT::isVector(ValueVT) || NumParts == 1) {
+ SDOperand Val = Parts[0];
+
+ // If the value was expanded, copy from the top part.
+ if (NumParts > 1) {
+ assert(NumParts == 2 &&
+ "Cannot expand to more than 2 elts yet!");
+ SDOperand Hi = Parts[1];
+ if (!DAG.getTargetLoweringInfo().isLittleEndian())
+ std::swap(Val, Hi);
+ return DAG.getNode(ISD::BUILD_PAIR, ValueVT, Val, Hi);
+ }
+
+ // Otherwise, if the value was promoted or extended, truncate it to the
+ // appropriate type.
+ if (PartVT == ValueVT)
+ return Val;
+
+ if (MVT::isVector(PartVT)) {
+ assert(MVT::isVector(ValueVT) && "Unknown vector conversion!");
+ return DAG.getNode(ISD::BIT_CONVERT, PartVT, Val);
+ }
+
+ if (MVT::isInteger(PartVT) &&
+ MVT::isInteger(ValueVT)) {
+ if (ValueVT < PartVT) {
+ // For a truncate, see if we have any information to
+ // indicate whether the truncated bits will always be
+ // zero or sign-extension.
+ if (AssertOp != ISD::DELETED_NODE)
+ Val = DAG.getNode(AssertOp, PartVT, Val,
+ DAG.getValueType(ValueVT));
+ return DAG.getNode(ISD::TRUNCATE, ValueVT, Val);
+ } else {
+ return DAG.getNode(ISD::ANY_EXTEND, ValueVT, Val);
+ }
+ }
+
+ if (MVT::isFloatingPoint(PartVT) &&
+ MVT::isFloatingPoint(ValueVT))
+ return DAG.getNode(ISD::FP_ROUND, ValueVT, Val);
+
+ if (MVT::getSizeInBits(PartVT) ==
+ MVT::getSizeInBits(ValueVT))
+ return DAG.getNode(ISD::BIT_CONVERT, ValueVT, Val);
+
+ assert(0 && "Unknown mismatch!");
+ }
+
+ // Handle a multi-element vector.
+ MVT::ValueType IntermediateVT, RegisterVT;
+ unsigned NumIntermediates;
+ unsigned NumRegs =
+ DAG.getTargetLoweringInfo()
+ .getVectorTypeBreakdown(ValueVT, IntermediateVT, NumIntermediates,
+ RegisterVT);
+
+ assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!");
+ assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!");
+ assert(RegisterVT == Parts[0].getValueType() &&
+ "Part type doesn't match part!");
+
+ // Assemble the parts into intermediate operands.
+ SmallVector<SDOperand, 8> Ops(NumIntermediates);
+ if (NumIntermediates == NumParts) {
+ // If the register was not expanded, truncate or copy the value,
+ // as appropriate.
+ for (unsigned i = 0; i != NumParts; ++i)
+ Ops[i] = getCopyFromParts(DAG, &Parts[i], 1,
+ PartVT, IntermediateVT);
+ } else if (NumParts > 0) {
+ // If the intermediate type was expanded, build the intermediate operands
+ // from the parts.
+ assert(NumIntermediates % NumParts == 0 &&
+ "Must expand into a divisible number of parts!");
+ unsigned Factor = NumIntermediates / NumParts;
+ for (unsigned i = 0; i != NumIntermediates; ++i)
+ Ops[i] = getCopyFromParts(DAG, &Parts[i * Factor], Factor,
+ PartVT, IntermediateVT);
+ }
+
+ // Build a vector with BUILD_VECTOR or CONCAT_VECTORS from the intermediate
+ // operands.
+ return DAG.getNode(MVT::isVector(IntermediateVT) ?
+ ISD::CONCAT_VECTORS :
+ ISD::BUILD_VECTOR,
+ ValueVT, &Ops[0], NumParts);
+}
+
+/// getCopyToParts - Create a series of nodes that contain the
+/// specified value split into legal parts.
+static void getCopyToParts(SelectionDAG &DAG,
+ SDOperand Val,
+ SDOperand *Parts,
+ unsigned NumParts,
+ MVT::ValueType PartVT) {
+ MVT::ValueType ValueVT = Val.getValueType();
+
+ if (!MVT::isVector(ValueVT) || NumParts == 1) {
+ // If the value was expanded, copy from the parts.
+ if (NumParts > 1) {
+ for (unsigned i = 0; i != NumParts; ++i)
+ Parts[i] = DAG.getNode(ISD::EXTRACT_ELEMENT, PartVT, Val,
+ DAG.getConstant(i, MVT::i32));
+ if (!DAG.getTargetLoweringInfo().isLittleEndian())
+ std::reverse(Parts, Parts + NumParts);
+ return;
+ }
+
+ // If there is a single part and the types differ, this must be
+ // a promotion.
+ if (PartVT != ValueVT) {
+ if (MVT::isVector(PartVT)) {
+ assert(MVT::isVector(ValueVT) &&
+ "Not a vector-vector cast?");
+ Val = DAG.getNode(ISD::BIT_CONVERT, PartVT, Val);
+ } else if (MVT::isInteger(PartVT) && MVT::isInteger(ValueVT)) {
+ if (PartVT < ValueVT)
+ Val = DAG.getNode(ISD::TRUNCATE, PartVT, Val);
+ else
+ Val = DAG.getNode(ISD::ANY_EXTEND, PartVT, Val);
+ } else if (MVT::isFloatingPoint(PartVT) &&
+ MVT::isFloatingPoint(ValueVT)) {
+ Val = DAG.getNode(ISD::FP_EXTEND, PartVT, Val);
+ } else if (MVT::getSizeInBits(PartVT) ==
+ MVT::getSizeInBits(ValueVT)) {
+ Val = DAG.getNode(ISD::BIT_CONVERT, PartVT, Val);
+ } else {
+ assert(0 && "Unknown mismatch!");
+ }
+ }
+ Parts[0] = Val;
+ return;
+ }
+
+ // Handle a multi-element vector.
+ MVT::ValueType IntermediateVT, RegisterVT;
+ unsigned NumIntermediates;
+ unsigned NumRegs =
+ DAG.getTargetLoweringInfo()
+ .getVectorTypeBreakdown(ValueVT, IntermediateVT, NumIntermediates,
+ RegisterVT);
+ unsigned NumElements = MVT::getVectorNumElements(ValueVT);
+
+ assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!");
+ assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!");
+
+ // Split the vector into intermediate operands.
+ SmallVector<SDOperand, 8> Ops(NumIntermediates);
+ for (unsigned i = 0; i != NumIntermediates; ++i)
+ if (MVT::isVector(IntermediateVT))
+ Ops[i] = DAG.getNode(ISD::EXTRACT_SUBVECTOR,
+ IntermediateVT, Val,
+ DAG.getConstant(i * (NumElements / NumIntermediates),
+ MVT::i32));
+ else
+ Ops[i] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT,
+ IntermediateVT, Val,
+ DAG.getConstant(i, MVT::i32));
+
+ // Split the intermediate operands into legal parts.
+ if (NumParts == NumIntermediates) {
+ // If the register was not expanded, promote or copy the value,
+ // as appropriate.
+ for (unsigned i = 0; i != NumParts; ++i)
+ getCopyToParts(DAG, Ops[i], &Parts[i], 1, PartVT);
+ } else if (NumParts > 0) {
+ // If the intermediate type was expanded, split each the value into
+ // legal parts.
+ assert(NumParts % NumIntermediates == 0 &&
+ "Must expand into a divisible number of parts!");
+ unsigned Factor = NumParts / NumIntermediates;
+ for (unsigned i = 0; i != NumIntermediates; ++i)
+ getCopyToParts(DAG, Ops[i], &Parts[i * Factor], Factor, PartVT);
+ }
+}
+
+
+SDOperand SelectionDAGLowering::getValue(const Value *V) {
+ SDOperand &N = NodeMap[V];
+ if (N.Val) return N;
+
+ const Type *VTy = V->getType();
+ MVT::ValueType VT = TLI.getValueType(VTy);
+ if (Constant *C = const_cast<Constant*>(dyn_cast<Constant>(V))) {
+ if (ConstantExpr *CE = dyn_cast<ConstantExpr>(C)) {
+ visit(CE->getOpcode(), *CE);
+ SDOperand N1 = NodeMap[V];
+ assert(N1.Val && "visit didn't populate the ValueMap!");
+ return N1;
+ } else if (GlobalValue *GV = dyn_cast<GlobalValue>(C)) {
+ return N = DAG.getGlobalAddress(GV, VT);
+ } else if (isa<ConstantPointerNull>(C)) {
+ return N = DAG.getConstant(0, TLI.getPointerTy());
+ } else if (isa<UndefValue>(C)) {
+ if (!isa<VectorType>(VTy))
+ return N = DAG.getNode(ISD::UNDEF, VT);
+
+ // Create a BUILD_VECTOR of undef nodes.
+ const VectorType *PTy = cast<VectorType>(VTy);
+ unsigned NumElements = PTy->getNumElements();
+ MVT::ValueType PVT = TLI.getValueType(PTy->getElementType());
+
+ SmallVector<SDOperand, 8> Ops;
+ Ops.assign(NumElements, DAG.getNode(ISD::UNDEF, PVT));
+
+ // Create a VConstant node with generic Vector type.
+ MVT::ValueType VT = MVT::getVectorType(PVT, NumElements);
+ return N = DAG.getNode(ISD::BUILD_VECTOR, VT,
+ &Ops[0], Ops.size());
+ } else if (ConstantFP *CFP = dyn_cast<ConstantFP>(C)) {
+ return N = DAG.getConstantFP(CFP->getValue(), VT);
+ } else if (const VectorType *PTy = dyn_cast<VectorType>(VTy)) {
+ unsigned NumElements = PTy->getNumElements();
+ MVT::ValueType PVT = TLI.getValueType(PTy->getElementType());
+
+ // Now that we know the number and type of the elements, push a
+ // Constant or ConstantFP node onto the ops list for each element of
+ // the vector constant.
+ SmallVector<SDOperand, 8> Ops;
+ if (ConstantVector *CP = dyn_cast<ConstantVector>(C)) {
+ for (unsigned i = 0; i != NumElements; ++i)
+ Ops.push_back(getValue(CP->getOperand(i)));
+ } else {
+ assert(isa<ConstantAggregateZero>(C) && "Unknown vector constant!");
+ SDOperand Op;
+ if (MVT::isFloatingPoint(PVT))
+ Op = DAG.getConstantFP(0, PVT);
+ else
+ Op = DAG.getConstant(0, PVT);
+ Ops.assign(NumElements, Op);
+ }
+
+ // Create a BUILD_VECTOR node.
+ MVT::ValueType VT = MVT::getVectorType(PVT, NumElements);
+ return NodeMap[V] = DAG.getNode(ISD::BUILD_VECTOR, VT, &Ops[0],
+ Ops.size());
+ } else {
+ // Canonicalize all constant ints to be unsigned.
+ return N = DAG.getConstant(cast<ConstantInt>(C)->getZExtValue(),VT);
+ }
+ }
+
+ if (const AllocaInst *AI = dyn_cast<AllocaInst>(V)) {
+ std::map<const AllocaInst*, int>::iterator SI =
+ FuncInfo.StaticAllocaMap.find(AI);
+ if (SI != FuncInfo.StaticAllocaMap.end())
+ return DAG.getFrameIndex(SI->second, TLI.getPointerTy());
+ }
+
+ unsigned InReg = FuncInfo.ValueMap[V];
+ assert(InReg && "Value not in map!");
+
+ MVT::ValueType RegisterVT = TLI.getRegisterType(VT);
+ unsigned NumRegs = TLI.getNumRegisters(VT);
+
+ std::vector<unsigned> Regs(NumRegs);
+ for (unsigned i = 0; i != NumRegs; ++i)
+ Regs[i] = InReg + i;
+
+ RegsForValue RFV(Regs, RegisterVT, VT);
+ SDOperand Chain = DAG.getEntryNode();
+
+ return RFV.getCopyFromRegs(DAG, Chain, NULL);
+}
+
+
+void SelectionDAGLowering::visitRet(ReturnInst &I) {
+ if (I.getNumOperands() == 0) {
+ DAG.setRoot(DAG.getNode(ISD::RET, MVT::Other, getRoot()));
+ return;
+ }
+ SmallVector<SDOperand, 8> NewValues;
+ NewValues.push_back(getRoot());
+ for (unsigned i = 0, e = I.getNumOperands(); i != e; ++i) {
+ SDOperand RetOp = getValue(I.getOperand(i));
+
+ // If this is an integer return value, we need to promote it ourselves to
+ // the full width of a register, since getCopyToParts and Legalize will use
+ // ANY_EXTEND rather than sign/zero.
+ // FIXME: C calling convention requires the return type to be promoted to
+ // at least 32-bit. But this is not necessary for non-C calling conventions.
+ if (MVT::isInteger(RetOp.getValueType()) &&
+ RetOp.getValueType() < MVT::i64) {
+ MVT::ValueType TmpVT;
+ if (TLI.getTypeAction(MVT::i32) == TargetLowering::Promote)
+ TmpVT = TLI.getTypeToTransformTo(MVT::i32);
+ else
+ TmpVT = MVT::i32;
+ const FunctionType *FTy = I.getParent()->getParent()->getFunctionType();
+ const ParamAttrsList *Attrs = FTy->getParamAttrs();
+ ISD::NodeType ExtendKind = ISD::ANY_EXTEND;
+ if (Attrs && Attrs->paramHasAttr(0, ParamAttr::SExt))
+ ExtendKind = ISD::SIGN_EXTEND;
+ if (Attrs && Attrs->paramHasAttr(0, ParamAttr::ZExt))
+ ExtendKind = ISD::ZERO_EXTEND;
+ RetOp = DAG.getNode(ExtendKind, TmpVT, RetOp);
+ NewValues.push_back(RetOp);
+ NewValues.push_back(DAG.getConstant(false, MVT::i32));
+ } else {
+ MVT::ValueType VT = RetOp.getValueType();
+ unsigned NumParts = TLI.getNumRegisters(VT);
+ MVT::ValueType PartVT = TLI.getRegisterType(VT);
+ SmallVector<SDOperand, 4> Parts(NumParts);
+ getCopyToParts(DAG, RetOp, &Parts[0], NumParts, PartVT);
+ for (unsigned i = 0; i < NumParts; ++i) {
+ NewValues.push_back(Parts[i]);
+ NewValues.push_back(DAG.getConstant(false, MVT::i32));
+ }
+ }
+ }
+ DAG.setRoot(DAG.getNode(ISD::RET, MVT::Other,
+ &NewValues[0], NewValues.size()));
+}
+
+/// ExportFromCurrentBlock - If this condition isn't known to be exported from
+/// the current basic block, add it to ValueMap now so that we'll get a
+/// CopyTo/FromReg.
+void SelectionDAGLowering::ExportFromCurrentBlock(Value *V) {
+ // No need to export constants.
+ if (!isa<Instruction>(V) && !isa<Argument>(V)) return;
+
+ // Already exported?
+ if (FuncInfo.isExportedInst(V)) return;
+
+ unsigned Reg = FuncInfo.InitializeRegForValue(V);
+ PendingLoads.push_back(CopyValueToVirtualRegister(V, Reg));
+}
+
+bool SelectionDAGLowering::isExportableFromCurrentBlock(Value *V,
+ const BasicBlock *FromBB) {
+ // The operands of the setcc have to be in this block. We don't know
+ // how to export them from some other block.
+ if (Instruction *VI = dyn_cast<Instruction>(V)) {
+ // Can export from current BB.
+ if (VI->getParent() == FromBB)
+ return true;
+
+ // Is already exported, noop.
+ return FuncInfo.isExportedInst(V);
+ }
+
+ // If this is an argument, we can export it if the BB is the entry block or
+ // if it is already exported.
+ if (isa<Argument>(V)) {
+ if (FromBB == &FromBB->getParent()->getEntryBlock())
+ return true;
+
+ // Otherwise, can only export this if it is already exported.
+ return FuncInfo.isExportedInst(V);
+ }
+
+ // Otherwise, constants can always be exported.
+ return true;
+}
+
+static bool InBlock(const Value *V, const BasicBlock *BB) {
+ if (const Instruction *I = dyn_cast<Instruction>(V))
+ return I->getParent() == BB;
+ return true;
+}
+
+/// FindMergedConditions - If Cond is an expression like
+void SelectionDAGLowering::FindMergedConditions(Value *Cond,
+ MachineBasicBlock *TBB,
+ MachineBasicBlock *FBB,
+ MachineBasicBlock *CurBB,
+ unsigned Opc) {
+ // If this node is not part of the or/and tree, emit it as a branch.
+ Instruction *BOp = dyn_cast<Instruction>(Cond);
+
+ if (!BOp || !(isa<BinaryOperator>(BOp) || isa<CmpInst>(BOp)) ||
+ (unsigned)BOp->getOpcode() != Opc || !BOp->hasOneUse() ||
+ BOp->getParent() != CurBB->getBasicBlock() ||
+ !InBlock(BOp->getOperand(0), CurBB->getBasicBlock()) ||
+ !InBlock(BOp->getOperand(1), CurBB->getBasicBlock())) {
+ const BasicBlock *BB = CurBB->getBasicBlock();
+
+ // If the leaf of the tree is a comparison, merge the condition into
+ // the caseblock.
+ if ((isa<ICmpInst>(Cond) || isa<FCmpInst>(Cond)) &&
+ // The operands of the cmp have to be in this block. We don't know
+ // how to export them from some other block. If this is the first block
+ // of the sequence, no exporting is needed.
+ (CurBB == CurMBB ||
+ (isExportableFromCurrentBlock(BOp->getOperand(0), BB) &&
+ isExportableFromCurrentBlock(BOp->getOperand(1), BB)))) {
+ BOp = cast<Instruction>(Cond);
+ ISD::CondCode Condition;
+ if (ICmpInst *IC = dyn_cast<ICmpInst>(Cond)) {
+ switch (IC->getPredicate()) {
+ default: assert(0 && "Unknown icmp predicate opcode!");
+ case ICmpInst::ICMP_EQ: Condition = ISD::SETEQ; break;
+ case ICmpInst::ICMP_NE: Condition = ISD::SETNE; break;
+ case ICmpInst::ICMP_SLE: Condition = ISD::SETLE; break;
+ case ICmpInst::ICMP_ULE: Condition = ISD::SETULE; break;
+ case ICmpInst::ICMP_SGE: Condition = ISD::SETGE; break;
+ case ICmpInst::ICMP_UGE: Condition = ISD::SETUGE; break;
+ case ICmpInst::ICMP_SLT: Condition = ISD::SETLT; break;
+ case ICmpInst::ICMP_ULT: Condition = ISD::SETULT; break;
+ case ICmpInst::ICMP_SGT: Condition = ISD::SETGT; break;
+ case ICmpInst::ICMP_UGT: Condition = ISD::SETUGT; break;
+ }
+ } else if (FCmpInst *FC = dyn_cast<FCmpInst>(Cond)) {
+ ISD::CondCode FPC, FOC;
+ switch (FC->getPredicate()) {
+ default: assert(0 && "Unknown fcmp predicate opcode!");
+ case FCmpInst::FCMP_FALSE: FOC = FPC = ISD::SETFALSE; break;
+ case FCmpInst::FCMP_OEQ: FOC = ISD::SETEQ; FPC = ISD::SETOEQ; break;
+ case FCmpInst::FCMP_OGT: FOC = ISD::SETGT; FPC = ISD::SETOGT; break;
+ case FCmpInst::FCMP_OGE: FOC = ISD::SETGE; FPC = ISD::SETOGE; break;
+ case FCmpInst::FCMP_OLT: FOC = ISD::SETLT; FPC = ISD::SETOLT; break;
+ case FCmpInst::FCMP_OLE: FOC = ISD::SETLE; FPC = ISD::SETOLE; break;
+ case FCmpInst::FCMP_ONE: FOC = ISD::SETNE; FPC = ISD::SETONE; break;
+ case FCmpInst::FCMP_ORD: FOC = ISD::SETEQ; FPC = ISD::SETO; break;
+ case FCmpInst::FCMP_UNO: FOC = ISD::SETNE; FPC = ISD::SETUO; break;
+ case FCmpInst::FCMP_UEQ: FOC = ISD::SETEQ; FPC = ISD::SETUEQ; break;
+ case FCmpInst::FCMP_UGT: FOC = ISD::SETGT; FPC = ISD::SETUGT; break;
+ case FCmpInst::FCMP_UGE: FOC = ISD::SETGE; FPC = ISD::SETUGE; break;
+ case FCmpInst::FCMP_ULT: FOC = ISD::SETLT; FPC = ISD::SETULT; break;
+ case FCmpInst::FCMP_ULE: FOC = ISD::SETLE; FPC = ISD::SETULE; break;
+ case FCmpInst::FCMP_UNE: FOC = ISD::SETNE; FPC = ISD::SETUNE; break;
+ case FCmpInst::FCMP_TRUE: FOC = FPC = ISD::SETTRUE; break;
+ }
+ if (FiniteOnlyFPMath())
+ Condition = FOC;
+ else
+ Condition = FPC;
+ } else {
+ Condition = ISD::SETEQ; // silence warning.
+ assert(0 && "Unknown compare instruction");
+ }
+
+ SelectionDAGISel::CaseBlock CB(Condition, BOp->getOperand(0),
+ BOp->getOperand(1), NULL, TBB, FBB, CurBB);
+ SwitchCases.push_back(CB);
+ return;
+ }
+
+ // Create a CaseBlock record representing this branch.
+ SelectionDAGISel::CaseBlock CB(ISD::SETEQ, Cond, ConstantInt::getTrue(),
+ NULL, TBB, FBB, CurBB);
+ SwitchCases.push_back(CB);
+ return;
+ }
+
+
+ // Create TmpBB after CurBB.
+ MachineFunction::iterator BBI = CurBB;
+ MachineBasicBlock *TmpBB = new MachineBasicBlock(CurBB->getBasicBlock());
+ CurBB->getParent()->getBasicBlockList().insert(++BBI, TmpBB);
+
+ if (Opc == Instruction::Or) {
+ // Codegen X | Y as:
+ // jmp_if_X TBB
+ // jmp TmpBB
+ // TmpBB:
+ // jmp_if_Y TBB
+ // jmp FBB
+ //
+
+ // Emit the LHS condition.
+ FindMergedConditions(BOp->getOperand(0), TBB, TmpBB, CurBB, Opc);
+
+ // Emit the RHS condition into TmpBB.
+ FindMergedConditions(BOp->getOperand(1), TBB, FBB, TmpBB, Opc);
+ } else {
+ assert(Opc == Instruction::And && "Unknown merge op!");
+ // Codegen X & Y as:
+ // jmp_if_X TmpBB
+ // jmp FBB
+ // TmpBB:
+ // jmp_if_Y TBB
+ // jmp FBB
+ //
+ // This requires creation of TmpBB after CurBB.
+
+ // Emit the LHS condition.
+ FindMergedConditions(BOp->getOperand(0), TmpBB, FBB, CurBB, Opc);
+
+ // Emit the RHS condition into TmpBB.
+ FindMergedConditions(BOp->getOperand(1), TBB, FBB, TmpBB, Opc);
+ }
+}
+
+/// If the set of cases should be emitted as a series of branches, return true.
+/// If we should emit this as a bunch of and/or'd together conditions, return
+/// false.
+static bool
+ShouldEmitAsBranches(const std::vector<SelectionDAGISel::CaseBlock> &Cases) {
+ if (Cases.size() != 2) return true;
+
+ // If this is two comparisons of the same values or'd or and'd together, they
+ // will get folded into a single comparison, so don't emit two blocks.
+ if ((Cases[0].CmpLHS == Cases[1].CmpLHS &&
+ Cases[0].CmpRHS == Cases[1].CmpRHS) ||
+ (Cases[0].CmpRHS == Cases[1].CmpLHS &&
+ Cases[0].CmpLHS == Cases[1].CmpRHS)) {
+ return false;
+ }
+
+ return true;
+}
+
+void SelectionDAGLowering::visitBr(BranchInst &I) {
+ // Update machine-CFG edges.
+ MachineBasicBlock *Succ0MBB = FuncInfo.MBBMap[I.getSuccessor(0)];
+
+ // Figure out which block is immediately after the current one.
+ MachineBasicBlock *NextBlock = 0;
+ MachineFunction::iterator BBI = CurMBB;
+ if (++BBI != CurMBB->getParent()->end())
+ NextBlock = BBI;
+
+ if (I.isUnconditional()) {
+ // If this is not a fall-through branch, emit the branch.
+ if (Succ0MBB != NextBlock)
+ DAG.setRoot(DAG.getNode(ISD::BR, MVT::Other, getRoot(),
+ DAG.getBasicBlock(Succ0MBB)));
+
+ // Update machine-CFG edges.
+ CurMBB->addSuccessor(Succ0MBB);
+
+ return;
+ }
+
+ // If this condition is one of the special cases we handle, do special stuff
+ // now.
+ Value *CondVal = I.getCondition();
+ MachineBasicBlock *Succ1MBB = FuncInfo.MBBMap[I.getSuccessor(1)];
+
+ // If this is a series of conditions that are or'd or and'd together, emit
+ // this as a sequence of branches instead of setcc's with and/or operations.
+ // For example, instead of something like:
+ // cmp A, B
+ // C = seteq
+ // cmp D, E
+ // F = setle
+ // or C, F
+ // jnz foo
+ // Emit:
+ // cmp A, B
+ // je foo
+ // cmp D, E
+ // jle foo
+ //
+ if (BinaryOperator *BOp = dyn_cast<BinaryOperator>(CondVal)) {
+ if (BOp->hasOneUse() &&
+ (BOp->getOpcode() == Instruction::And ||
+ BOp->getOpcode() == Instruction::Or)) {
+ FindMergedConditions(BOp, Succ0MBB, Succ1MBB, CurMBB, BOp->getOpcode());
+ // If the compares in later blocks need to use values not currently
+ // exported from this block, export them now. This block should always
+ // be the first entry.
+ assert(SwitchCases[0].ThisBB == CurMBB && "Unexpected lowering!");
+
+ // Allow some cases to be rejected.
+ if (ShouldEmitAsBranches(SwitchCases)) {
+ for (unsigned i = 1, e = SwitchCases.size(); i != e; ++i) {
+ ExportFromCurrentBlock(SwitchCases[i].CmpLHS);
+ ExportFromCurrentBlock(SwitchCases[i].CmpRHS);
+ }
+
+ // Emit the branch for this block.
+ visitSwitchCase(SwitchCases[0]);
+ SwitchCases.erase(SwitchCases.begin());
+ return;
+ }
+
+ // Okay, we decided not to do this, remove any inserted MBB's and clear
+ // SwitchCases.
+ for (unsigned i = 1, e = SwitchCases.size(); i != e; ++i)
+ CurMBB->getParent()->getBasicBlockList().erase(SwitchCases[i].ThisBB);
+
+ SwitchCases.clear();
+ }
+ }
+
+ // Create a CaseBlock record representing this branch.
+ SelectionDAGISel::CaseBlock CB(ISD::SETEQ, CondVal, ConstantInt::getTrue(),
+ NULL, Succ0MBB, Succ1MBB, CurMBB);
+ // Use visitSwitchCase to actually insert the fast branch sequence for this
+ // cond branch.
+ visitSwitchCase(CB);
+}
+
+/// visitSwitchCase - Emits the necessary code to represent a single node in
+/// the binary search tree resulting from lowering a switch instruction.
+void SelectionDAGLowering::visitSwitchCase(SelectionDAGISel::CaseBlock &CB) {
+ SDOperand Cond;
+ SDOperand CondLHS = getValue(CB.CmpLHS);
+
+ // Build the setcc now.
+ if (CB.CmpMHS == NULL) {
+ // Fold "(X == true)" to X and "(X == false)" to !X to
+ // handle common cases produced by branch lowering.
+ if (CB.CmpRHS == ConstantInt::getTrue() && CB.CC == ISD::SETEQ)
+ Cond = CondLHS;
+ else if (CB.CmpRHS == ConstantInt::getFalse() && CB.CC == ISD::SETEQ) {
+ SDOperand True = DAG.getConstant(1, CondLHS.getValueType());
+ Cond = DAG.getNode(ISD::XOR, CondLHS.getValueType(), CondLHS, True);
+ } else
+ Cond = DAG.getSetCC(MVT::i1, CondLHS, getValue(CB.CmpRHS), CB.CC);
+ } else {
+ assert(CB.CC == ISD::SETLE && "Can handle only LE ranges now");
+
+ uint64_t Low = cast<ConstantInt>(CB.CmpLHS)->getSExtValue();
+ uint64_t High = cast<ConstantInt>(CB.CmpRHS)->getSExtValue();
+
+ SDOperand CmpOp = getValue(CB.CmpMHS);
+ MVT::ValueType VT = CmpOp.getValueType();
+
+ if (cast<ConstantInt>(CB.CmpLHS)->isMinValue(true)) {
+ Cond = DAG.getSetCC(MVT::i1, CmpOp, DAG.getConstant(High, VT), ISD::SETLE);
+ } else {
+ SDOperand SUB = DAG.getNode(ISD::SUB, VT, CmpOp, DAG.getConstant(Low, VT));
+ Cond = DAG.getSetCC(MVT::i1, SUB,
+ DAG.getConstant(High-Low, VT), ISD::SETULE);
+ }
+
+ }
+
+ // Set NextBlock to be the MBB immediately after the current one, if any.
+ // This is used to avoid emitting unnecessary branches to the next block.
+ MachineBasicBlock *NextBlock = 0;
+ MachineFunction::iterator BBI = CurMBB;
+ if (++BBI != CurMBB->getParent()->end())
+ NextBlock = BBI;
+
+ // If the lhs block is the next block, invert the condition so that we can
+ // fall through to the lhs instead of the rhs block.
+ if (CB.TrueBB == NextBlock) {
+ std::swap(CB.TrueBB, CB.FalseBB);
+ SDOperand True = DAG.getConstant(1, Cond.getValueType());
+ Cond = DAG.getNode(ISD::XOR, Cond.getValueType(), Cond, True);
+ }
+ SDOperand BrCond = DAG.getNode(ISD::BRCOND, MVT::Other, getRoot(), Cond,
+ DAG.getBasicBlock(CB.TrueBB));
+ if (CB.FalseBB == NextBlock)
+ DAG.setRoot(BrCond);
+ else
+ DAG.setRoot(DAG.getNode(ISD::BR, MVT::Other, BrCond,
+ DAG.getBasicBlock(CB.FalseBB)));
+ // Update successor info
+ CurMBB->addSuccessor(CB.TrueBB);
+ CurMBB->addSuccessor(CB.FalseBB);
+}
+
+/// visitJumpTable - Emit JumpTable node in the current MBB
+void SelectionDAGLowering::visitJumpTable(SelectionDAGISel::JumpTable &JT) {
+ // Emit the code for the jump table
+ assert(JT.Reg != -1U && "Should lower JT Header first!");
+ MVT::ValueType PTy = TLI.getPointerTy();
+ SDOperand Index = DAG.getCopyFromReg(getRoot(), JT.Reg, PTy);
+ SDOperand Table = DAG.getJumpTable(JT.JTI, PTy);
+ DAG.setRoot(DAG.getNode(ISD::BR_JT, MVT::Other, Index.getValue(1),
+ Table, Index));
+ return;
+}
+
+/// visitJumpTableHeader - This function emits necessary code to produce index
+/// in the JumpTable from switch case.
+void SelectionDAGLowering::visitJumpTableHeader(SelectionDAGISel::JumpTable &JT,
+ SelectionDAGISel::JumpTableHeader &JTH) {
+ // Subtract the lowest switch case value from the value being switched on
+ // and conditional branch to default mbb if the result is greater than the
+ // difference between smallest and largest cases.
+ SDOperand SwitchOp = getValue(JTH.SValue);
+ MVT::ValueType VT = SwitchOp.getValueType();
+ SDOperand SUB = DAG.getNode(ISD::SUB, VT, SwitchOp,
+ DAG.getConstant(JTH.First, VT));
+
+ // The SDNode we just created, which holds the value being switched on
+ // minus the the smallest case value, needs to be copied to a virtual
+ // register so it can be used as an index into the jump table in a
+ // subsequent basic block. This value may be smaller or larger than the
+ // target's pointer type, and therefore require extension or truncating.
+ if (MVT::getSizeInBits(VT) > MVT::getSizeInBits(TLI.getPointerTy()))
+ SwitchOp = DAG.getNode(ISD::TRUNCATE, TLI.getPointerTy(), SUB);
+ else
+ SwitchOp = DAG.getNode(ISD::ZERO_EXTEND, TLI.getPointerTy(), SUB);
+
+ unsigned JumpTableReg = FuncInfo.MakeReg(TLI.getPointerTy());
+ SDOperand CopyTo = DAG.getCopyToReg(getRoot(), JumpTableReg, SwitchOp);
+ JT.Reg = JumpTableReg;
+
+ // Emit the range check for the jump table, and branch to the default
+ // block for the switch statement if the value being switched on exceeds
+ // the largest case in the switch.
+ SDOperand CMP = DAG.getSetCC(TLI.getSetCCResultTy(), SUB,
+ DAG.getConstant(JTH.Last-JTH.First,VT),
+ ISD::SETUGT);
+
+ // Set NextBlock to be the MBB immediately after the current one, if any.
+ // This is used to avoid emitting unnecessary branches to the next block.
+ MachineBasicBlock *NextBlock = 0;
+ MachineFunction::iterator BBI = CurMBB;
+ if (++BBI != CurMBB->getParent()->end())
+ NextBlock = BBI;
+
+ SDOperand BrCond = DAG.getNode(ISD::BRCOND, MVT::Other, CopyTo, CMP,
+ DAG.getBasicBlock(JT.Default));
+
+ if (JT.MBB == NextBlock)
+ DAG.setRoot(BrCond);
+ else
+ DAG.setRoot(DAG.getNode(ISD::BR, MVT::Other, BrCond,
+ DAG.getBasicBlock(JT.MBB)));
+
+ return;
+}
+
+/// visitBitTestHeader - This function emits necessary code to produce value
+/// suitable for "bit tests"
+void SelectionDAGLowering::visitBitTestHeader(SelectionDAGISel::BitTestBlock &B) {
+ // Subtract the minimum value
+ SDOperand SwitchOp = getValue(B.SValue);
+ MVT::ValueType VT = SwitchOp.getValueType();
+ SDOperand SUB = DAG.getNode(ISD::SUB, VT, SwitchOp,
+ DAG.getConstant(B.First, VT));
+
+ // Check range
+ SDOperand RangeCmp = DAG.getSetCC(TLI.getSetCCResultTy(), SUB,
+ DAG.getConstant(B.Range, VT),
+ ISD::SETUGT);
+
+ SDOperand ShiftOp;
+ if (MVT::getSizeInBits(VT) > MVT::getSizeInBits(TLI.getShiftAmountTy()))
+ ShiftOp = DAG.getNode(ISD::TRUNCATE, TLI.getShiftAmountTy(), SUB);
+ else
+ ShiftOp = DAG.getNode(ISD::ZERO_EXTEND, TLI.getShiftAmountTy(), SUB);
+
+ // Make desired shift
+ SDOperand SwitchVal = DAG.getNode(ISD::SHL, TLI.getPointerTy(),
+ DAG.getConstant(1, TLI.getPointerTy()),
+ ShiftOp);
+
+ unsigned SwitchReg = FuncInfo.MakeReg(TLI.getPointerTy());
+ SDOperand CopyTo = DAG.getCopyToReg(getRoot(), SwitchReg, SwitchVal);
+ B.Reg = SwitchReg;
+
+ SDOperand BrRange = DAG.getNode(ISD::BRCOND, MVT::Other, CopyTo, RangeCmp,
+ DAG.getBasicBlock(B.Default));
+
+ // Set NextBlock to be the MBB immediately after the current one, if any.
+ // This is used to avoid emitting unnecessary branches to the next block.
+ MachineBasicBlock *NextBlock = 0;
+ MachineFunction::iterator BBI = CurMBB;
+ if (++BBI != CurMBB->getParent()->end())
+ NextBlock = BBI;
+
+ MachineBasicBlock* MBB = B.Cases[0].ThisBB;
+ if (MBB == NextBlock)
+ DAG.setRoot(BrRange);
+ else
+ DAG.setRoot(DAG.getNode(ISD::BR, MVT::Other, CopyTo,
+ DAG.getBasicBlock(MBB)));
+
+ CurMBB->addSuccessor(B.Default);
+ CurMBB->addSuccessor(MBB);
+
+ return;
+}
+
+/// visitBitTestCase - this function produces one "bit test"
+void SelectionDAGLowering::visitBitTestCase(MachineBasicBlock* NextMBB,
+ unsigned Reg,
+ SelectionDAGISel::BitTestCase &B) {
+ // Emit bit tests and jumps
+ SDOperand SwitchVal = DAG.getCopyFromReg(getRoot(), Reg, TLI.getPointerTy());
+
+ SDOperand AndOp = DAG.getNode(ISD::AND, TLI.getPointerTy(),
+ SwitchVal,
+ DAG.getConstant(B.Mask,
+ TLI.getPointerTy()));
+ SDOperand AndCmp = DAG.getSetCC(TLI.getSetCCResultTy(), AndOp,
+ DAG.getConstant(0, TLI.getPointerTy()),
+ ISD::SETNE);
+ SDOperand BrAnd = DAG.getNode(ISD::BRCOND, MVT::Other, getRoot(),
+ AndCmp, DAG.getBasicBlock(B.TargetBB));
+
+ // Set NextBlock to be the MBB immediately after the current one, if any.
+ // This is used to avoid emitting unnecessary branches to the next block.
+ MachineBasicBlock *NextBlock = 0;
+ MachineFunction::iterator BBI = CurMBB;
+ if (++BBI != CurMBB->getParent()->end())
+ NextBlock = BBI;
+
+ if (NextMBB == NextBlock)
+ DAG.setRoot(BrAnd);
+ else
+ DAG.setRoot(DAG.getNode(ISD::BR, MVT::Other, BrAnd,
+ DAG.getBasicBlock(NextMBB)));
+
+ CurMBB->addSuccessor(B.TargetBB);
+ CurMBB->addSuccessor(NextMBB);
+
+ return;
+}
+
+void SelectionDAGLowering::visitInvoke(InvokeInst &I) {
+ // Retrieve successors.
+ MachineBasicBlock *Return = FuncInfo.MBBMap[I.getSuccessor(0)];
+ MachineBasicBlock *LandingPad = FuncInfo.MBBMap[I.getSuccessor(1)];
+
+ LowerCallTo(I, I.getCalledValue()->getType(),
+ I.getCallingConv(),
+ false,
+ getValue(I.getOperand(0)),
+ 3, LandingPad);
+
+ // If the value of the invoke is used outside of its defining block, make it
+ // available as a virtual register.
+ if (!I.use_empty()) {
+ DenseMap<const Value*, unsigned>::iterator VMI = FuncInfo.ValueMap.find(&I);
+ if (VMI != FuncInfo.ValueMap.end())
+ DAG.setRoot(CopyValueToVirtualRegister(&I, VMI->second));
+ }
+
+ // Drop into normal successor.
+ DAG.setRoot(DAG.getNode(ISD::BR, MVT::Other, getRoot(),
+ DAG.getBasicBlock(Return)));
+
+ // Update successor info
+ CurMBB->addSuccessor(Return);
+ CurMBB->addSuccessor(LandingPad);
+}
+
+void SelectionDAGLowering::visitUnwind(UnwindInst &I) {
+}
+
+/// handleSmallSwitchCaseRange - Emit a series of specific tests (suitable for
+/// small case ranges).
+bool SelectionDAGLowering::handleSmallSwitchRange(CaseRec& CR,
+ CaseRecVector& WorkList,
+ Value* SV,
+ MachineBasicBlock* Default) {
+ Case& BackCase = *(CR.Range.second-1);
+
+ // Size is the number of Cases represented by this range.
+ unsigned Size = CR.Range.second - CR.Range.first;
+ if (Size > 3)
+ return false;
+
+ // Get the MachineFunction which holds the current MBB. This is used when
+ // inserting any additional MBBs necessary to represent the switch.
+ MachineFunction *CurMF = CurMBB->getParent();
+
+ // Figure out which block is immediately after the current one.
+ MachineBasicBlock *NextBlock = 0;
+ MachineFunction::iterator BBI = CR.CaseBB;
+
+ if (++BBI != CurMBB->getParent()->end())
+ NextBlock = BBI;
+
+ // TODO: If any two of the cases has the same destination, and if one value
+ // is the same as the other, but has one bit unset that the other has set,
+ // use bit manipulation to do two compares at once. For example:
+ // "if (X == 6 || X == 4)" -> "if ((X|2) == 6)"
+
+ // Rearrange the case blocks so that the last one falls through if possible.
+ if (NextBlock && Default != NextBlock && BackCase.BB != NextBlock) {
+ // The last case block won't fall through into 'NextBlock' if we emit the
+ // branches in this order. See if rearranging a case value would help.
+ for (CaseItr I = CR.Range.first, E = CR.Range.second-1; I != E; ++I) {
+ if (I->BB == NextBlock) {
+ std::swap(*I, BackCase);
+ break;
+ }
+ }
+ }
+
+ // Create a CaseBlock record representing a conditional branch to
+ // the Case's target mbb if the value being switched on SV is equal
+ // to C.
+ MachineBasicBlock *CurBlock = CR.CaseBB;
+ for (CaseItr I = CR.Range.first, E = CR.Range.second; I != E; ++I) {
+ MachineBasicBlock *FallThrough;
+ if (I != E-1) {
+ FallThrough = new MachineBasicBlock(CurBlock->getBasicBlock());
+ CurMF->getBasicBlockList().insert(BBI, FallThrough);
+ } else {
+ // If the last case doesn't match, go to the default block.
+ FallThrough = Default;
+ }
+
+ Value *RHS, *LHS, *MHS;
+ ISD::CondCode CC;
+ if (I->High == I->Low) {
+ // This is just small small case range :) containing exactly 1 case
+ CC = ISD::SETEQ;
+ LHS = SV; RHS = I->High; MHS = NULL;
+ } else {
+ CC = ISD::SETLE;
+ LHS = I->Low; MHS = SV; RHS = I->High;
+ }
+ SelectionDAGISel::CaseBlock CB(CC, LHS, RHS, MHS,
+ I->BB, FallThrough, CurBlock);
+
+ // If emitting the first comparison, just call visitSwitchCase to emit the
+ // code into the current block. Otherwise, push the CaseBlock onto the
+ // vector to be later processed by SDISel, and insert the node's MBB
+ // before the next MBB.
+ if (CurBlock == CurMBB)
+ visitSwitchCase(CB);
+ else
+ SwitchCases.push_back(CB);
+
+ CurBlock = FallThrough;
+ }
+
+ return true;
+}
+
+static inline bool areJTsAllowed(const TargetLowering &TLI) {
+ return (TLI.isOperationLegal(ISD::BR_JT, MVT::Other) ||
+ TLI.isOperationLegal(ISD::BRIND, MVT::Other));
+}
+
+/// handleJTSwitchCase - Emit jumptable for current switch case range
+bool SelectionDAGLowering::handleJTSwitchCase(CaseRec& CR,
+ CaseRecVector& WorkList,
+ Value* SV,
+ MachineBasicBlock* Default) {
+ Case& FrontCase = *CR.Range.first;
+ Case& BackCase = *(CR.Range.second-1);
+
+ int64_t First = cast<ConstantInt>(FrontCase.Low)->getSExtValue();
+ int64_t Last = cast<ConstantInt>(BackCase.High)->getSExtValue();
+
+ uint64_t TSize = 0;
+ for (CaseItr I = CR.Range.first, E = CR.Range.second;
+ I!=E; ++I)
+ TSize += I->size();
+
+ if (!areJTsAllowed(TLI) || TSize <= 3)
+ return false;
+
+ double Density = (double)TSize / (double)((Last - First) + 1ULL);
+ if (Density < 0.4)
+ return false;
+
+ DOUT << "Lowering jump table\n"
+ << "First entry: " << First << ". Last entry: " << Last << "\n"
+ << "Size: " << TSize << ". Density: " << Density << "\n\n";
+
+ // Get the MachineFunction which holds the current MBB. This is used when
+ // inserting any additional MBBs necessary to represent the switch.
+ MachineFunction *CurMF = CurMBB->getParent();
+
+ // Figure out which block is immediately after the current one.
+ MachineBasicBlock *NextBlock = 0;
+ MachineFunction::iterator BBI = CR.CaseBB;
+
+ if (++BBI != CurMBB->getParent()->end())
+ NextBlock = BBI;
+
+ const BasicBlock *LLVMBB = CR.CaseBB->getBasicBlock();
+
+ // Create a new basic block to hold the code for loading the address
+ // of the jump table, and jumping to it. Update successor information;
+ // we will either branch to the default case for the switch, or the jump
+ // table.
+ MachineBasicBlock *JumpTableBB = new MachineBasicBlock(LLVMBB);
+ CurMF->getBasicBlockList().insert(BBI, JumpTableBB);
+ CR.CaseBB->addSuccessor(Default);
+ CR.CaseBB->addSuccessor(JumpTableBB);
+
+ // Build a vector of destination BBs, corresponding to each target
+ // of the jump table. If the value of the jump table slot corresponds to
+ // a case statement, push the case's BB onto the vector, otherwise, push
+ // the default BB.
+ std::vector<MachineBasicBlock*> DestBBs;
+ int64_t TEI = First;
+ for (CaseItr I = CR.Range.first, E = CR.Range.second; I != E; ++TEI) {
+ int64_t Low = cast<ConstantInt>(I->Low)->getSExtValue();
+ int64_t High = cast<ConstantInt>(I->High)->getSExtValue();
+
+ if ((Low <= TEI) && (TEI <= High)) {
+ DestBBs.push_back(I->BB);
+ if (TEI==High)
+ ++I;
+ } else {
+ DestBBs.push_back(Default);
+ }
+ }
+
+ // Update successor info. Add one edge to each unique successor.
+ BitVector SuccsHandled(CR.CaseBB->getParent()->getNumBlockIDs());
+ for (std::vector<MachineBasicBlock*>::iterator I = DestBBs.begin(),
+ E = DestBBs.end(); I != E; ++I) {
+ if (!SuccsHandled[(*I)->getNumber()]) {
+ SuccsHandled[(*I)->getNumber()] = true;
+ JumpTableBB->addSuccessor(*I);
+ }
+ }
+
+ // Create a jump table index for this jump table, or return an existing
+ // one.
+ unsigned JTI = CurMF->getJumpTableInfo()->getJumpTableIndex(DestBBs);
+
+ // Set the jump table information so that we can codegen it as a second
+ // MachineBasicBlock
+ SelectionDAGISel::JumpTable JT(-1U, JTI, JumpTableBB, Default);
+ SelectionDAGISel::JumpTableHeader JTH(First, Last, SV, CR.CaseBB,
+ (CR.CaseBB == CurMBB));
+ if (CR.CaseBB == CurMBB)
+ visitJumpTableHeader(JT, JTH);
+
+ JTCases.push_back(SelectionDAGISel::JumpTableBlock(JTH, JT));
+
+ return true;
+}
+
+/// handleBTSplitSwitchCase - emit comparison and split binary search tree into
+/// 2 subtrees.
+bool SelectionDAGLowering::handleBTSplitSwitchCase(CaseRec& CR,
+ CaseRecVector& WorkList,
+ Value* SV,
+ MachineBasicBlock* Default) {
+ // Get the MachineFunction which holds the current MBB. This is used when
+ // inserting any additional MBBs necessary to represent the switch.
+ MachineFunction *CurMF = CurMBB->getParent();
+
+ // Figure out which block is immediately after the current one.
+ MachineBasicBlock *NextBlock = 0;
+ MachineFunction::iterator BBI = CR.CaseBB;
+
+ if (++BBI != CurMBB->getParent()->end())
+ NextBlock = BBI;
+
+ Case& FrontCase = *CR.Range.first;
+ Case& BackCase = *(CR.Range.second-1);
+ const BasicBlock *LLVMBB = CR.CaseBB->getBasicBlock();
+
+ // Size is the number of Cases represented by this range.
+ unsigned Size = CR.Range.second - CR.Range.first;
+
+ int64_t First = cast<ConstantInt>(FrontCase.Low)->getSExtValue();
+ int64_t Last = cast<ConstantInt>(BackCase.High)->getSExtValue();
+ double FMetric = 0;
+ CaseItr Pivot = CR.Range.first + Size/2;
+
+ // Select optimal pivot, maximizing sum density of LHS and RHS. This will
+ // (heuristically) allow us to emit JumpTable's later.
+ uint64_t TSize = 0;
+ for (CaseItr I = CR.Range.first, E = CR.Range.second;
+ I!=E; ++I)
+ TSize += I->size();
+
+ uint64_t LSize = FrontCase.size();
+ uint64_t RSize = TSize-LSize;
+ DOUT << "Selecting best pivot: \n"
+ << "First: " << First << ", Last: " << Last <<"\n"
+ << "LSize: " << LSize << ", RSize: " << RSize << "\n";
+ for (CaseItr I = CR.Range.first, J=I+1, E = CR.Range.second;
+ J!=E; ++I, ++J) {
+ int64_t LEnd = cast<ConstantInt>(I->High)->getSExtValue();
+ int64_t RBegin = cast<ConstantInt>(J->Low)->getSExtValue();
+ assert((RBegin-LEnd>=1) && "Invalid case distance");
+ double LDensity = (double)LSize / (double)((LEnd - First) + 1ULL);
+ double RDensity = (double)RSize / (double)((Last - RBegin) + 1ULL);
+ double Metric = Log2_64(RBegin-LEnd)*(LDensity+RDensity);
+ // Should always split in some non-trivial place
+ DOUT <<"=>Step\n"
+ << "LEnd: " << LEnd << ", RBegin: " << RBegin << "\n"
+ << "LDensity: " << LDensity << ", RDensity: " << RDensity << "\n"
+ << "Metric: " << Metric << "\n";
+ if (FMetric < Metric) {
+ Pivot = J;
+ FMetric = Metric;
+ DOUT << "Current metric set to: " << FMetric << "\n";
+ }
+
+ LSize += J->size();
+ RSize -= J->size();
+ }
+ if (areJTsAllowed(TLI)) {
+ // If our case is dense we *really* should handle it earlier!
+ assert((FMetric > 0) && "Should handle dense range earlier!");
+ } else {
+ Pivot = CR.Range.first + Size/2;
+ }
+
+ CaseRange LHSR(CR.Range.first, Pivot);
+ CaseRange RHSR(Pivot, CR.Range.second);
+ Constant *C = Pivot->Low;
+ MachineBasicBlock *FalseBB = 0, *TrueBB = 0;
+
+ // We know that we branch to the LHS if the Value being switched on is
+ // less than the Pivot value, C. We use this to optimize our binary
+ // tree a bit, by recognizing that if SV is greater than or equal to the
+ // LHS's Case Value, and that Case Value is exactly one less than the
+ // Pivot's Value, then we can branch directly to the LHS's Target,
+ // rather than creating a leaf node for it.
+ if ((LHSR.second - LHSR.first) == 1 &&
+ LHSR.first->High == CR.GE &&
+ cast<ConstantInt>(C)->getSExtValue() ==
+ (cast<ConstantInt>(CR.GE)->getSExtValue() + 1LL)) {
+ TrueBB = LHSR.first->BB;
+ } else {
+ TrueBB = new MachineBasicBlock(LLVMBB);
+ CurMF->getBasicBlockList().insert(BBI, TrueBB);
+ WorkList.push_back(CaseRec(TrueBB, C, CR.GE, LHSR));
+ }
+
+ // Similar to the optimization above, if the Value being switched on is
+ // known to be less than the Constant CR.LT, and the current Case Value
+ // is CR.LT - 1, then we can branch directly to the target block for
+ // the current Case Value, rather than emitting a RHS leaf node for it.
+ if ((RHSR.second - RHSR.first) == 1 && CR.LT &&
+ cast<ConstantInt>(RHSR.first->Low)->getSExtValue() ==
+ (cast<ConstantInt>(CR.LT)->getSExtValue() - 1LL)) {
+ FalseBB = RHSR.first->BB;
+ } else {
+ FalseBB = new MachineBasicBlock(LLVMBB);
+ CurMF->getBasicBlockList().insert(BBI, FalseBB);
+ WorkList.push_back(CaseRec(FalseBB,CR.LT,C,RHSR));
+ }
+
+ // Create a CaseBlock record representing a conditional branch to
+ // the LHS node if the value being switched on SV is less than C.
+ // Otherwise, branch to LHS.
+ SelectionDAGISel::CaseBlock CB(ISD::SETLT, SV, C, NULL,
+ TrueBB, FalseBB, CR.CaseBB);
+
+ if (CR.CaseBB == CurMBB)
+ visitSwitchCase(CB);
+ else
+ SwitchCases.push_back(CB);
+
+ return true;
+}
+
+/// handleBitTestsSwitchCase - if current case range has few destination and
+/// range span less, than machine word bitwidth, encode case range into series
+/// of masks and emit bit tests with these masks.
+bool SelectionDAGLowering::handleBitTestsSwitchCase(CaseRec& CR,
+ CaseRecVector& WorkList,
+ Value* SV,
+ MachineBasicBlock* Default){
+ unsigned IntPtrBits = MVT::getSizeInBits(TLI.getPointerTy());
+
+ Case& FrontCase = *CR.Range.first;
+ Case& BackCase = *(CR.Range.second-1);
+
+ // Get the MachineFunction which holds the current MBB. This is used when
+ // inserting any additional MBBs necessary to represent the switch.
+ MachineFunction *CurMF = CurMBB->getParent();
+
+ unsigned numCmps = 0;
+ for (CaseItr I = CR.Range.first, E = CR.Range.second;
+ I!=E; ++I) {
+ // Single case counts one, case range - two.
+ if (I->Low == I->High)
+ numCmps +=1;
+ else
+ numCmps +=2;
+ }
+
+ // Count unique destinations
+ SmallSet<MachineBasicBlock*, 4> Dests;
+ for (CaseItr I = CR.Range.first, E = CR.Range.second; I!=E; ++I) {
+ Dests.insert(I->BB);
+ if (Dests.size() > 3)
+ // Don't bother the code below, if there are too much unique destinations
+ return false;
+ }
+ DOUT << "Total number of unique destinations: " << Dests.size() << "\n"
+ << "Total number of comparisons: " << numCmps << "\n";
+
+ // Compute span of values.
+ Constant* minValue = FrontCase.Low;
+ Constant* maxValue = BackCase.High;
+ uint64_t range = cast<ConstantInt>(maxValue)->getSExtValue() -
+ cast<ConstantInt>(minValue)->getSExtValue();
+ DOUT << "Compare range: " << range << "\n"
+ << "Low bound: " << cast<ConstantInt>(minValue)->getSExtValue() << "\n"
+ << "High bound: " << cast<ConstantInt>(maxValue)->getSExtValue() << "\n";
+
+ if (range>=IntPtrBits ||
+ (!(Dests.size() == 1 && numCmps >= 3) &&
+ !(Dests.size() == 2 && numCmps >= 5) &&
+ !(Dests.size() >= 3 && numCmps >= 6)))
+ return false;
+
+ DOUT << "Emitting bit tests\n";
+ int64_t lowBound = 0;
+
+ // Optimize the case where all the case values fit in a
+ // word without having to subtract minValue. In this case,
+ // we can optimize away the subtraction.
+ if (cast<ConstantInt>(minValue)->getSExtValue() >= 0 &&
+ cast<ConstantInt>(maxValue)->getSExtValue() < IntPtrBits) {
+ range = cast<ConstantInt>(maxValue)->getSExtValue();
+ } else {
+ lowBound = cast<ConstantInt>(minValue)->getSExtValue();
+ }
+
+ CaseBitsVector CasesBits;
+ unsigned i, count = 0;
+
+ for (CaseItr I = CR.Range.first, E = CR.Range.second; I!=E; ++I) {
+ MachineBasicBlock* Dest = I->BB;
+ for (i = 0; i < count; ++i)
+ if (Dest == CasesBits[i].BB)
+ break;
+
+ if (i == count) {
+ assert((count < 3) && "Too much destinations to test!");
+ CasesBits.push_back(CaseBits(0, Dest, 0));
+ count++;
+ }
+
+ uint64_t lo = cast<ConstantInt>(I->Low)->getSExtValue() - lowBound;
+ uint64_t hi = cast<ConstantInt>(I->High)->getSExtValue() - lowBound;
+
+ for (uint64_t j = lo; j <= hi; j++) {
+ CasesBits[i].Mask |= 1ULL << j;
+ CasesBits[i].Bits++;
+ }
+
+ }
+ std::sort(CasesBits.begin(), CasesBits.end(), CaseBitsCmp());
+
+ SelectionDAGISel::BitTestInfo BTC;
+
+ // Figure out which block is immediately after the current one.
+ MachineFunction::iterator BBI = CR.CaseBB;
+ ++BBI;
+
+ const BasicBlock *LLVMBB = CR.CaseBB->getBasicBlock();
+
+ DOUT << "Cases:\n";
+ for (unsigned i = 0, e = CasesBits.size(); i!=e; ++i) {
+ DOUT << "Mask: " << CasesBits[i].Mask << ", Bits: " << CasesBits[i].Bits
+ << ", BB: " << CasesBits[i].BB << "\n";
+
+ MachineBasicBlock *CaseBB = new MachineBasicBlock(LLVMBB);
+ CurMF->getBasicBlockList().insert(BBI, CaseBB);
+ BTC.push_back(SelectionDAGISel::BitTestCase(CasesBits[i].Mask,
+ CaseBB,
+ CasesBits[i].BB));
+ }
+
+ SelectionDAGISel::BitTestBlock BTB(lowBound, range, SV,
+ -1U, (CR.CaseBB == CurMBB),
+ CR.CaseBB, Default, BTC);
+
+ if (CR.CaseBB == CurMBB)
+ visitBitTestHeader(BTB);
+
+ BitTestCases.push_back(BTB);
+
+ return true;
+}
+
+
+// Clusterify - Transform simple list of Cases into list of CaseRange's
+unsigned SelectionDAGLowering::Clusterify(CaseVector& Cases,
+ const SwitchInst& SI) {
+ unsigned numCmps = 0;
+
+ // Start with "simple" cases
+ for (unsigned i = 1; i < SI.getNumSuccessors(); ++i) {
+ MachineBasicBlock *SMBB = FuncInfo.MBBMap[SI.getSuccessor(i)];
+ Cases.push_back(Case(SI.getSuccessorValue(i),
+ SI.getSuccessorValue(i),
+ SMBB));
+ }
+ sort(Cases.begin(), Cases.end(), CaseCmp());
+
+ // Merge case into clusters
+ if (Cases.size()>=2)
+ // Must recompute end() each iteration because it may be
+ // invalidated by erase if we hold on to it
+ for (CaseItr I=Cases.begin(), J=++(Cases.begin()); J!=Cases.end(); ) {
+ int64_t nextValue = cast<ConstantInt>(J->Low)->getSExtValue();
+ int64_t currentValue = cast<ConstantInt>(I->High)->getSExtValue();
+ MachineBasicBlock* nextBB = J->BB;
+ MachineBasicBlock* currentBB = I->BB;
+
+ // If the two neighboring cases go to the same destination, merge them
+ // into a single case.
+ if ((nextValue-currentValue==1) && (currentBB == nextBB)) {
+ I->High = J->High;
+ J = Cases.erase(J);
+ } else {
+ I = J++;
+ }
+ }
+
+ for (CaseItr I=Cases.begin(), E=Cases.end(); I!=E; ++I, ++numCmps) {
+ if (I->Low != I->High)
+ // A range counts double, since it requires two compares.
+ ++numCmps;
+ }
+
+ return numCmps;
+}
+
+void SelectionDAGLowering::visitSwitch(SwitchInst &SI) {
+ // Figure out which block is immediately after the current one.
+ MachineBasicBlock *NextBlock = 0;
+ MachineFunction::iterator BBI = CurMBB;
+
+ MachineBasicBlock *Default = FuncInfo.MBBMap[SI.getDefaultDest()];
+
+ // If there is only the default destination, branch to it if it is not the
+ // next basic block. Otherwise, just fall through.
+ if (SI.getNumOperands() == 2) {
+ // Update machine-CFG edges.
+
+ // If this is not a fall-through branch, emit the branch.
+ if (Default != NextBlock)
+ DAG.setRoot(DAG.getNode(ISD::BR, MVT::Other, getRoot(),
+ DAG.getBasicBlock(Default)));
+
+ CurMBB->addSuccessor(Default);
+ return;
+ }
+
+ // If there are any non-default case statements, create a vector of Cases
+ // representing each one, and sort the vector so that we can efficiently
+ // create a binary search tree from them.
+ CaseVector Cases;
+ unsigned numCmps = Clusterify(Cases, SI);
+ DOUT << "Clusterify finished. Total clusters: " << Cases.size()
+ << ". Total compares: " << numCmps << "\n";
+
+ // Get the Value to be switched on and default basic blocks, which will be
+ // inserted into CaseBlock records, representing basic blocks in the binary
+ // search tree.
+ Value *SV = SI.getOperand(0);
+
+ // Push the initial CaseRec onto the worklist
+ CaseRecVector WorkList;
+ WorkList.push_back(CaseRec(CurMBB,0,0,CaseRange(Cases.begin(),Cases.end())));
+
+ while (!WorkList.empty()) {
+ // Grab a record representing a case range to process off the worklist
+ CaseRec CR = WorkList.back();
+ WorkList.pop_back();
+
+ if (handleBitTestsSwitchCase(CR, WorkList, SV, Default))
+ continue;
+
+ // If the range has few cases (two or less) emit a series of specific
+ // tests.
+ if (handleSmallSwitchRange(CR, WorkList, SV, Default))
+ continue;
+
+ // If the switch has more than 5 blocks, and at least 40% dense, and the
+ // target supports indirect branches, then emit a jump table rather than
+ // lowering the switch to a binary tree of conditional branches.
+ if (handleJTSwitchCase(CR, WorkList, SV, Default))
+ continue;
+
+ // Emit binary tree. We need to pick a pivot, and push left and right ranges
+ // onto the worklist. Leafs are handled via handleSmallSwitchRange() call.
+ handleBTSplitSwitchCase(CR, WorkList, SV, Default);
+ }
+}
+
+
+void SelectionDAGLowering::visitSub(User &I) {
+ // -0.0 - X --> fneg
+ const Type *Ty = I.getType();
+ if (isa<VectorType>(Ty)) {
+ if (ConstantVector *CV = dyn_cast<ConstantVector>(I.getOperand(0))) {
+ const VectorType *DestTy = cast<VectorType>(I.getType());
+ const Type *ElTy = DestTy->getElementType();
+ if (ElTy->isFloatingPoint()) {
+ unsigned VL = DestTy->getNumElements();
+ std::vector<Constant*> NZ(VL, ConstantFP::get(ElTy, -0.0));
+ Constant *CNZ = ConstantVector::get(&NZ[0], NZ.size());
+ if (CV == CNZ) {
+ SDOperand Op2 = getValue(I.getOperand(1));
+ setValue(&I, DAG.getNode(ISD::FNEG, Op2.getValueType(), Op2));
+ return;
+ }
+ }
+ }
+ }
+ if (Ty->isFloatingPoint()) {
+ if (ConstantFP *CFP = dyn_cast<ConstantFP>(I.getOperand(0)))
+ if (CFP->isExactlyValue(-0.0)) {
+ SDOperand Op2 = getValue(I.getOperand(1));
+ setValue(&I, DAG.getNode(ISD::FNEG, Op2.getValueType(), Op2));
+ return;
+ }
+ }
+
+ visitBinary(I, Ty->isFPOrFPVector() ? ISD::FSUB : ISD::SUB);
+}
+
+void SelectionDAGLowering::visitBinary(User &I, unsigned OpCode) {
+ SDOperand Op1 = getValue(I.getOperand(0));
+ SDOperand Op2 = getValue(I.getOperand(1));
+
+ setValue(&I, DAG.getNode(OpCode, Op1.getValueType(), Op1, Op2));
+}
+
+void SelectionDAGLowering::visitShift(User &I, unsigned Opcode) {
+ SDOperand Op1 = getValue(I.getOperand(0));
+ SDOperand Op2 = getValue(I.getOperand(1));
+
+ if (MVT::getSizeInBits(TLI.getShiftAmountTy()) <
+ MVT::getSizeInBits(Op2.getValueType()))
+ Op2 = DAG.getNode(ISD::TRUNCATE, TLI.getShiftAmountTy(), Op2);
+ else if (TLI.getShiftAmountTy() > Op2.getValueType())
+ Op2 = DAG.getNode(ISD::ANY_EXTEND, TLI.getShiftAmountTy(), Op2);
+
+ setValue(&I, DAG.getNode(Opcode, Op1.getValueType(), Op1, Op2));
+}
+
+void SelectionDAGLowering::visitICmp(User &I) {
+ ICmpInst::Predicate predicate = ICmpInst::BAD_ICMP_PREDICATE;
+ if (ICmpInst *IC = dyn_cast<ICmpInst>(&I))
+ predicate = IC->getPredicate();
+ else if (ConstantExpr *IC = dyn_cast<ConstantExpr>(&I))
+ predicate = ICmpInst::Predicate(IC->getPredicate());
+ SDOperand Op1 = getValue(I.getOperand(0));
+ SDOperand Op2 = getValue(I.getOperand(1));
+ ISD::CondCode Opcode;
+ switch (predicate) {
+ case ICmpInst::ICMP_EQ : Opcode = ISD::SETEQ; break;
+ case ICmpInst::ICMP_NE : Opcode = ISD::SETNE; break;
+ case ICmpInst::ICMP_UGT : Opcode = ISD::SETUGT; break;
+ case ICmpInst::ICMP_UGE : Opcode = ISD::SETUGE; break;
+ case ICmpInst::ICMP_ULT : Opcode = ISD::SETULT; break;
+ case ICmpInst::ICMP_ULE : Opcode = ISD::SETULE; break;
+ case ICmpInst::ICMP_SGT : Opcode = ISD::SETGT; break;
+ case ICmpInst::ICMP_SGE : Opcode = ISD::SETGE; break;
+ case ICmpInst::ICMP_SLT : Opcode = ISD::SETLT; break;
+ case ICmpInst::ICMP_SLE : Opcode = ISD::SETLE; break;
+ default:
+ assert(!"Invalid ICmp predicate value");
+ Opcode = ISD::SETEQ;
+ break;
+ }
+ setValue(&I, DAG.getSetCC(MVT::i1, Op1, Op2, Opcode));
+}
+
+void SelectionDAGLowering::visitFCmp(User &I) {
+ FCmpInst::Predicate predicate = FCmpInst::BAD_FCMP_PREDICATE;
+ if (FCmpInst *FC = dyn_cast<FCmpInst>(&I))
+ predicate = FC->getPredicate();
+ else if (ConstantExpr *FC = dyn_cast<ConstantExpr>(&I))
+ predicate = FCmpInst::Predicate(FC->getPredicate());
+ SDOperand Op1 = getValue(I.getOperand(0));
+ SDOperand Op2 = getValue(I.getOperand(1));
+ ISD::CondCode Condition, FOC, FPC;
+ switch (predicate) {
+ case FCmpInst::FCMP_FALSE: FOC = FPC = ISD::SETFALSE; break;
+ case FCmpInst::FCMP_OEQ: FOC = ISD::SETEQ; FPC = ISD::SETOEQ; break;
+ case FCmpInst::FCMP_OGT: FOC = ISD::SETGT; FPC = ISD::SETOGT; break;
+ case FCmpInst::FCMP_OGE: FOC = ISD::SETGE; FPC = ISD::SETOGE; break;
+ case FCmpInst::FCMP_OLT: FOC = ISD::SETLT; FPC = ISD::SETOLT; break;
+ case FCmpInst::FCMP_OLE: FOC = ISD::SETLE; FPC = ISD::SETOLE; break;
+ case FCmpInst::FCMP_ONE: FOC = ISD::SETNE; FPC = ISD::SETONE; break;
+ case FCmpInst::FCMP_ORD: FOC = ISD::SETEQ; FPC = ISD::SETO; break;
+ case FCmpInst::FCMP_UNO: FOC = ISD::SETNE; FPC = ISD::SETUO; break;
+ case FCmpInst::FCMP_UEQ: FOC = ISD::SETEQ; FPC = ISD::SETUEQ; break;
+ case FCmpInst::FCMP_UGT: FOC = ISD::SETGT; FPC = ISD::SETUGT; break;
+ case FCmpInst::FCMP_UGE: FOC = ISD::SETGE; FPC = ISD::SETUGE; break;
+ case FCmpInst::FCMP_ULT: FOC = ISD::SETLT; FPC = ISD::SETULT; break;
+ case FCmpInst::FCMP_ULE: FOC = ISD::SETLE; FPC = ISD::SETULE; break;
+ case FCmpInst::FCMP_UNE: FOC = ISD::SETNE; FPC = ISD::SETUNE; break;
+ case FCmpInst::FCMP_TRUE: FOC = FPC = ISD::SETTRUE; break;
+ default:
+ assert(!"Invalid FCmp predicate value");
+ FOC = FPC = ISD::SETFALSE;
+ break;
+ }
+ if (FiniteOnlyFPMath())
+ Condition = FOC;
+ else
+ Condition = FPC;
+ setValue(&I, DAG.getSetCC(MVT::i1, Op1, Op2, Condition));
+}
+
+void SelectionDAGLowering::visitSelect(User &I) {
+ SDOperand Cond = getValue(I.getOperand(0));
+ SDOperand TrueVal = getValue(I.getOperand(1));
+ SDOperand FalseVal = getValue(I.getOperand(2));
+ setValue(&I, DAG.getNode(ISD::SELECT, TrueVal.getValueType(), Cond,
+ TrueVal, FalseVal));
+}
+
+
+void SelectionDAGLowering::visitTrunc(User &I) {
+ // TruncInst cannot be a no-op cast because sizeof(src) > sizeof(dest).
+ SDOperand N = getValue(I.getOperand(0));
+ MVT::ValueType DestVT = TLI.getValueType(I.getType());
+ setValue(&I, DAG.getNode(ISD::TRUNCATE, DestVT, N));
+}
+
+void SelectionDAGLowering::visitZExt(User &I) {
+ // ZExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
+ // ZExt also can't be a cast to bool for same reason. So, nothing much to do
+ SDOperand N = getValue(I.getOperand(0));
+ MVT::ValueType DestVT = TLI.getValueType(I.getType());
+ setValue(&I, DAG.getNode(ISD::ZERO_EXTEND, DestVT, N));
+}
+
+void SelectionDAGLowering::visitSExt(User &I) {
+ // SExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
+ // SExt also can't be a cast to bool for same reason. So, nothing much to do
+ SDOperand N = getValue(I.getOperand(0));
+ MVT::ValueType DestVT = TLI.getValueType(I.getType());
+ setValue(&I, DAG.getNode(ISD::SIGN_EXTEND, DestVT, N));
+}
+
+void SelectionDAGLowering::visitFPTrunc(User &I) {
+ // FPTrunc is never a no-op cast, no need to check
+ SDOperand N = getValue(I.getOperand(0));
+ MVT::ValueType DestVT = TLI.getValueType(I.getType());
+ setValue(&I, DAG.getNode(ISD::FP_ROUND, DestVT, N));
+}
+
+void SelectionDAGLowering::visitFPExt(User &I){
+ // FPTrunc is never a no-op cast, no need to check
+ SDOperand N = getValue(I.getOperand(0));
+ MVT::ValueType DestVT = TLI.getValueType(I.getType());
+ setValue(&I, DAG.getNode(ISD::FP_EXTEND, DestVT, N));
+}
+
+void SelectionDAGLowering::visitFPToUI(User &I) {
+ // FPToUI is never a no-op cast, no need to check
+ SDOperand N = getValue(I.getOperand(0));
+ MVT::ValueType DestVT = TLI.getValueType(I.getType());
+ setValue(&I, DAG.getNode(ISD::FP_TO_UINT, DestVT, N));
+}
+
+void SelectionDAGLowering::visitFPToSI(User &I) {
+ // FPToSI is never a no-op cast, no need to check
+ SDOperand N = getValue(I.getOperand(0));
+ MVT::ValueType DestVT = TLI.getValueType(I.getType());
+ setValue(&I, DAG.getNode(ISD::FP_TO_SINT, DestVT, N));
+}
+
+void SelectionDAGLowering::visitUIToFP(User &I) {
+ // UIToFP is never a no-op cast, no need to check
+ SDOperand N = getValue(I.getOperand(0));
+ MVT::ValueType DestVT = TLI.getValueType(I.getType());
+ setValue(&I, DAG.getNode(ISD::UINT_TO_FP, DestVT, N));
+}
+
+void SelectionDAGLowering::visitSIToFP(User &I){
+ // UIToFP is never a no-op cast, no need to check
+ SDOperand N = getValue(I.getOperand(0));
+ MVT::ValueType DestVT = TLI.getValueType(I.getType());
+ setValue(&I, DAG.getNode(ISD::SINT_TO_FP, DestVT, N));
+}
+
+void SelectionDAGLowering::visitPtrToInt(User &I) {
+ // What to do depends on the size of the integer and the size of the pointer.
+ // We can either truncate, zero extend, or no-op, accordingly.
+ SDOperand N = getValue(I.getOperand(0));
+ MVT::ValueType SrcVT = N.getValueType();
+ MVT::ValueType DestVT = TLI.getValueType(I.getType());
+ SDOperand Result;
+ if (MVT::getSizeInBits(DestVT) < MVT::getSizeInBits(SrcVT))
+ Result = DAG.getNode(ISD::TRUNCATE, DestVT, N);
+ else
+ // Note: ZERO_EXTEND can handle cases where the sizes are equal too
+ Result = DAG.getNode(ISD::ZERO_EXTEND, DestVT, N);
+ setValue(&I, Result);
+}
+
+void SelectionDAGLowering::visitIntToPtr(User &I) {
+ // What to do depends on the size of the integer and the size of the pointer.
+ // We can either truncate, zero extend, or no-op, accordingly.
+ SDOperand N = getValue(I.getOperand(0));
+ MVT::ValueType SrcVT = N.getValueType();
+ MVT::ValueType DestVT = TLI.getValueType(I.getType());
+ if (MVT::getSizeInBits(DestVT) < MVT::getSizeInBits(SrcVT))
+ setValue(&I, DAG.getNode(ISD::TRUNCATE, DestVT, N));
+ else
+ // Note: ZERO_EXTEND can handle cases where the sizes are equal too
+ setValue(&I, DAG.getNode(ISD::ZERO_EXTEND, DestVT, N));
+}
+
+void SelectionDAGLowering::visitBitCast(User &I) {
+ SDOperand N = getValue(I.getOperand(0));
+ MVT::ValueType DestVT = TLI.getValueType(I.getType());
+
+ // BitCast assures us that source and destination are the same size so this
+ // is either a BIT_CONVERT or a no-op.
+ if (DestVT != N.getValueType())
+ setValue(&I, DAG.getNode(ISD::BIT_CONVERT, DestVT, N)); // convert types
+ else
+ setValue(&I, N); // noop cast.
+}
+
+void SelectionDAGLowering::visitInsertElement(User &I) {
+ SDOperand InVec = getValue(I.getOperand(0));
+ SDOperand InVal = getValue(I.getOperand(1));
+ SDOperand InIdx = DAG.getNode(ISD::ZERO_EXTEND, TLI.getPointerTy(),
+ getValue(I.getOperand(2)));
+
+ setValue(&I, DAG.getNode(ISD::INSERT_VECTOR_ELT,
+ TLI.getValueType(I.getType()),
+ InVec, InVal, InIdx));
+}
+
+void SelectionDAGLowering::visitExtractElement(User &I) {
+ SDOperand InVec = getValue(I.getOperand(0));
+ SDOperand InIdx = DAG.getNode(ISD::ZERO_EXTEND, TLI.getPointerTy(),
+ getValue(I.getOperand(1)));
+ setValue(&I, DAG.getNode(ISD::EXTRACT_VECTOR_ELT,
+ TLI.getValueType(I.getType()), InVec, InIdx));
+}
+
+void SelectionDAGLowering::visitShuffleVector(User &I) {
+ SDOperand V1 = getValue(I.getOperand(0));
+ SDOperand V2 = getValue(I.getOperand(1));
+ SDOperand Mask = getValue(I.getOperand(2));
+
+ setValue(&I, DAG.getNode(ISD::VECTOR_SHUFFLE,
+ TLI.getValueType(I.getType()),
+ V1, V2, Mask));
+}
+
+
+void SelectionDAGLowering::visitGetElementPtr(User &I) {
+ SDOperand N = getValue(I.getOperand(0));
+ const Type *Ty = I.getOperand(0)->getType();
+
+ for (GetElementPtrInst::op_iterator OI = I.op_begin()+1, E = I.op_end();
+ OI != E; ++OI) {
+ Value *Idx = *OI;
+ if (const StructType *StTy = dyn_cast<StructType>(Ty)) {
+ unsigned Field = cast<ConstantInt>(Idx)->getZExtValue();
+ if (Field) {
+ // N = N + Offset
+ uint64_t Offset = TD->getStructLayout(StTy)->getElementOffset(Field);
+ N = DAG.getNode(ISD::ADD, N.getValueType(), N,
+ getIntPtrConstant(Offset));
+ }
+ Ty = StTy->getElementType(Field);
+ } else {
+ Ty = cast<SequentialType>(Ty)->getElementType();
+
+ // If this is a constant subscript, handle it quickly.
+ if (ConstantInt *CI = dyn_cast<ConstantInt>(Idx)) {
+ if (CI->getZExtValue() == 0) continue;
+ uint64_t Offs =
+ TD->getTypeSize(Ty)*cast<ConstantInt>(CI)->getSExtValue();
+ N = DAG.getNode(ISD::ADD, N.getValueType(), N, getIntPtrConstant(Offs));
+ continue;
+ }
+
+ // N = N + Idx * ElementSize;
+ uint64_t ElementSize = TD->getTypeSize(Ty);
+ SDOperand IdxN = getValue(Idx);
+
+ // If the index is smaller or larger than intptr_t, truncate or extend
+ // it.
+ if (IdxN.getValueType() < N.getValueType()) {
+ IdxN = DAG.getNode(ISD::SIGN_EXTEND, N.getValueType(), IdxN);
+ } else if (IdxN.getValueType() > N.getValueType())
+ IdxN = DAG.getNode(ISD::TRUNCATE, N.getValueType(), IdxN);
+
+ // If this is a multiply by a power of two, turn it into a shl
+ // immediately. This is a very common case.
+ if (isPowerOf2_64(ElementSize)) {
+ unsigned Amt = Log2_64(ElementSize);
+ IdxN = DAG.getNode(ISD::SHL, N.getValueType(), IdxN,
+ DAG.getConstant(Amt, TLI.getShiftAmountTy()));
+ N = DAG.getNode(ISD::ADD, N.getValueType(), N, IdxN);
+ continue;
+ }
+
+ SDOperand Scale = getIntPtrConstant(ElementSize);
+ IdxN = DAG.getNode(ISD::MUL, N.getValueType(), IdxN, Scale);
+ N = DAG.getNode(ISD::ADD, N.getValueType(), N, IdxN);
+ }
+ }
+ setValue(&I, N);
+}
+
+void SelectionDAGLowering::visitAlloca(AllocaInst &I) {
+ // If this is a fixed sized alloca in the entry block of the function,
+ // allocate it statically on the stack.
+ if (FuncInfo.StaticAllocaMap.count(&I))
+ return; // getValue will auto-populate this.
+
+ const Type *Ty = I.getAllocatedType();
+ uint64_t TySize = TLI.getTargetData()->getTypeSize(Ty);
+ unsigned Align =
+ std::max((unsigned)TLI.getTargetData()->getPrefTypeAlignment(Ty),
+ I.getAlignment());
+
+ SDOperand AllocSize = getValue(I.getArraySize());
+ MVT::ValueType IntPtr = TLI.getPointerTy();
+ if (IntPtr < AllocSize.getValueType())
+ AllocSize = DAG.getNode(ISD::TRUNCATE, IntPtr, AllocSize);
+ else if (IntPtr > AllocSize.getValueType())
+ AllocSize = DAG.getNode(ISD::ZERO_EXTEND, IntPtr, AllocSize);
+
+ AllocSize = DAG.getNode(ISD::MUL, IntPtr, AllocSize,
+ getIntPtrConstant(TySize));
+
+ // Handle alignment. If the requested alignment is less than the stack
+ // alignment, ignore it and round the size of the allocation up to the stack
+ // alignment size. If the size is greater than or equal to the stack
+ // alignment, we note this in the DYNAMIC_STACKALLOC node.
+ unsigned StackAlign =
+ TLI.getTargetMachine().getFrameInfo()->getStackAlignment();
+ if (Align < StackAlign) {
+ Align = 0;
+ // Add SA-1 to the size.
+ AllocSize = DAG.getNode(ISD::ADD, AllocSize.getValueType(), AllocSize,
+ getIntPtrConstant(StackAlign-1));
+ // Mask out the low bits for alignment purposes.
+ AllocSize = DAG.getNode(ISD::AND, AllocSize.getValueType(), AllocSize,
+ getIntPtrConstant(~(uint64_t)(StackAlign-1)));
+ }
+
+ SDOperand Ops[] = { getRoot(), AllocSize, getIntPtrConstant(Align) };
+ const MVT::ValueType *VTs = DAG.getNodeValueTypes(AllocSize.getValueType(),
+ MVT::Other);
+ SDOperand DSA = DAG.getNode(ISD::DYNAMIC_STACKALLOC, VTs, 2, Ops, 3);
+ setValue(&I, DSA);
+ DAG.setRoot(DSA.getValue(1));
+
+ // Inform the Frame Information that we have just allocated a variable-sized
+ // object.
+ CurMBB->getParent()->getFrameInfo()->CreateVariableSizedObject();
+}
+
+void SelectionDAGLowering::visitLoad(LoadInst &I) {
+ SDOperand Ptr = getValue(I.getOperand(0));
+
+ SDOperand Root;
+ if (I.isVolatile())
+ Root = getRoot();
+ else {
+ // Do not serialize non-volatile loads against each other.
+ Root = DAG.getRoot();
+ }
+
+ setValue(&I, getLoadFrom(I.getType(), Ptr, I.getOperand(0),
+ Root, I.isVolatile(), I.getAlignment()));
+}
+
+SDOperand SelectionDAGLowering::getLoadFrom(const Type *Ty, SDOperand Ptr,
+ const Value *SV, SDOperand Root,
+ bool isVolatile,
+ unsigned Alignment) {
+ SDOperand L =
+ DAG.getLoad(TLI.getValueType(Ty), Root, Ptr, SV, 0,
+ isVolatile, Alignment);
+
+ if (isVolatile)
+ DAG.setRoot(L.getValue(1));
+ else
+ PendingLoads.push_back(L.getValue(1));
+
+ return L;
+}
+
+
+void SelectionDAGLowering::visitStore(StoreInst &I) {
+ Value *SrcV = I.getOperand(0);
+ SDOperand Src = getValue(SrcV);
+ SDOperand Ptr = getValue(I.getOperand(1));
+ DAG.setRoot(DAG.getStore(getRoot(), Src, Ptr, I.getOperand(1), 0,
+ I.isVolatile(), I.getAlignment()));
+}
+
+/// IntrinsicCannotAccessMemory - Return true if the specified intrinsic cannot
+/// access memory and has no other side effects at all.
+static bool IntrinsicCannotAccessMemory(unsigned IntrinsicID) {
+#define GET_NO_MEMORY_INTRINSICS
+#include "llvm/Intrinsics.gen"
+#undef GET_NO_MEMORY_INTRINSICS
+ return false;
+}
+
+// IntrinsicOnlyReadsMemory - Return true if the specified intrinsic doesn't
+// have any side-effects or if it only reads memory.
+static bool IntrinsicOnlyReadsMemory(unsigned IntrinsicID) {
+#define GET_SIDE_EFFECT_INFO
+#include "llvm/Intrinsics.gen"
+#undef GET_SIDE_EFFECT_INFO
+ return false;
+}
+
+/// visitTargetIntrinsic - Lower a call of a target intrinsic to an INTRINSIC
+/// node.
+void SelectionDAGLowering::visitTargetIntrinsic(CallInst &I,
+ unsigned Intrinsic) {
+ bool HasChain = !IntrinsicCannotAccessMemory(Intrinsic);
+ bool OnlyLoad = HasChain && IntrinsicOnlyReadsMemory(Intrinsic);
+
+ // Build the operand list.
+ SmallVector<SDOperand, 8> Ops;
+ if (HasChain) { // If this intrinsic has side-effects, chainify it.
+ if (OnlyLoad) {
+ // We don't need to serialize loads against other loads.
+ Ops.push_back(DAG.getRoot());
+ } else {
+ Ops.push_back(getRoot());
+ }
+ }
+
+ // Add the intrinsic ID as an integer operand.
+ Ops.push_back(DAG.getConstant(Intrinsic, TLI.getPointerTy()));
+
+ // Add all operands of the call to the operand list.
+ for (unsigned i = 1, e = I.getNumOperands(); i != e; ++i) {
+ SDOperand Op = getValue(I.getOperand(i));
+ assert(TLI.isTypeLegal(Op.getValueType()) &&
+ "Intrinsic uses a non-legal type?");
+ Ops.push_back(Op);
+ }
+
+ std::vector<MVT::ValueType> VTs;
+ if (I.getType() != Type::VoidTy) {
+ MVT::ValueType VT = TLI.getValueType(I.getType());
+ if (MVT::isVector(VT)) {
+ const VectorType *DestTy = cast<VectorType>(I.getType());
+ MVT::ValueType EltVT = TLI.getValueType(DestTy->getElementType());
+
+ VT = MVT::getVectorType(EltVT, DestTy->getNumElements());
+ assert(VT != MVT::Other && "Intrinsic uses a non-legal type?");
+ }
+
+ assert(TLI.isTypeLegal(VT) && "Intrinsic uses a non-legal type?");
+ VTs.push_back(VT);
+ }
+ if (HasChain)
+ VTs.push_back(MVT::Other);
+
+ const MVT::ValueType *VTList = DAG.getNodeValueTypes(VTs);
+
+ // Create the node.
+ SDOperand Result;
+ if (!HasChain)
+ Result = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, VTList, VTs.size(),
+ &Ops[0], Ops.size());
+ else if (I.getType() != Type::VoidTy)
+ Result = DAG.getNode(ISD::INTRINSIC_W_CHAIN, VTList, VTs.size(),
+ &Ops[0], Ops.size());
+ else
+ Result = DAG.getNode(ISD::INTRINSIC_VOID, VTList, VTs.size(),
+ &Ops[0], Ops.size());
+
+ if (HasChain) {
+ SDOperand Chain = Result.getValue(Result.Val->getNumValues()-1);
+ if (OnlyLoad)
+ PendingLoads.push_back(Chain);
+ else
+ DAG.setRoot(Chain);
+ }
+ if (I.getType() != Type::VoidTy) {
+ if (const VectorType *PTy = dyn_cast<VectorType>(I.getType())) {
+ MVT::ValueType VT = TLI.getValueType(PTy);
+ Result = DAG.getNode(ISD::BIT_CONVERT, VT, Result);
+ }
+ setValue(&I, Result);
+ }
+}
+
+/// ExtractTypeInfo - Returns the type info, possibly bitcast, encoded in V.
+static GlobalVariable *ExtractTypeInfo (Value *V) {
+ V = IntrinsicInst::StripPointerCasts(V);
+ GlobalVariable *GV = dyn_cast<GlobalVariable>(V);
+ assert (GV || isa<ConstantPointerNull>(V) &&
+ "TypeInfo must be a global variable or NULL");
+ return GV;
+}
+
+/// addCatchInfo - Extract the personality and type infos from an eh.selector
+/// call, and add them to the specified machine basic block.
+static void addCatchInfo(CallInst &I, MachineModuleInfo *MMI,
+ MachineBasicBlock *MBB) {
+ // Inform the MachineModuleInfo of the personality for this landing pad.
+ ConstantExpr *CE = cast<ConstantExpr>(I.getOperand(2));
+ assert(CE->getOpcode() == Instruction::BitCast &&
+ isa<Function>(CE->getOperand(0)) &&
+ "Personality should be a function");
+ MMI->addPersonality(MBB, cast<Function>(CE->getOperand(0)));
+
+ // Gather all the type infos for this landing pad and pass them along to
+ // MachineModuleInfo.
+ std::vector<GlobalVariable *> TyInfo;
+ unsigned N = I.getNumOperands();
+
+ for (unsigned i = N - 1; i > 2; --i) {
+ if (ConstantInt *CI = dyn_cast<ConstantInt>(I.getOperand(i))) {
+ unsigned FilterLength = CI->getZExtValue();
+ unsigned FirstCatch = i + FilterLength + 1;
+ assert (FirstCatch <= N && "Invalid filter length");
+
+ if (FirstCatch < N) {
+ TyInfo.reserve(N - FirstCatch);
+ for (unsigned j = FirstCatch; j < N; ++j)
+ TyInfo.push_back(ExtractTypeInfo(I.getOperand(j)));
+ MMI->addCatchTypeInfo(MBB, TyInfo);
+ TyInfo.clear();
+ }
+
+ TyInfo.reserve(FilterLength);
+ for (unsigned j = i + 1; j < FirstCatch; ++j)
+ TyInfo.push_back(ExtractTypeInfo(I.getOperand(j)));
+ MMI->addFilterTypeInfo(MBB, TyInfo);
+ TyInfo.clear();
+
+ N = i;
+ }
+ }
+
+ if (N > 3) {
+ TyInfo.reserve(N - 3);
+ for (unsigned j = 3; j < N; ++j)
+ TyInfo.push_back(ExtractTypeInfo(I.getOperand(j)));
+ MMI->addCatchTypeInfo(MBB, TyInfo);
+ }
+}
+
+/// visitIntrinsicCall - Lower the call to the specified intrinsic function. If
+/// we want to emit this as a call to a named external function, return the name
+/// otherwise lower it and return null.
+const char *
+SelectionDAGLowering::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) {
+ switch (Intrinsic) {
+ default:
+ // By default, turn this into a target intrinsic node.
+ visitTargetIntrinsic(I, Intrinsic);
+ return 0;
+ case Intrinsic::vastart: visitVAStart(I); return 0;
+ case Intrinsic::vaend: visitVAEnd(I); return 0;
+ case Intrinsic::vacopy: visitVACopy(I); return 0;
+ case Intrinsic::returnaddress:
+ setValue(&I, DAG.getNode(ISD::RETURNADDR, TLI.getPointerTy(),
+ getValue(I.getOperand(1))));
+ return 0;
+ case Intrinsic::frameaddress:
+ setValue(&I, DAG.getNode(ISD::FRAMEADDR, TLI.getPointerTy(),
+ getValue(I.getOperand(1))));
+ return 0;
+ case Intrinsic::setjmp:
+ return "_setjmp"+!TLI.usesUnderscoreSetJmp();
+ break;
+ case Intrinsic::longjmp:
+ return "_longjmp"+!TLI.usesUnderscoreLongJmp();
+ break;
+ case Intrinsic::memcpy_i32:
+ case Intrinsic::memcpy_i64:
+ visitMemIntrinsic(I, ISD::MEMCPY);
+ return 0;
+ case Intrinsic::memset_i32:
+ case Intrinsic::memset_i64:
+ visitMemIntrinsic(I, ISD::MEMSET);
+ return 0;
+ case Intrinsic::memmove_i32:
+ case Intrinsic::memmove_i64:
+ visitMemIntrinsic(I, ISD::MEMMOVE);
+ return 0;
+
+ case Intrinsic::dbg_stoppoint: {
+ MachineModuleInfo *MMI = DAG.getMachineModuleInfo();
+ DbgStopPointInst &SPI = cast<DbgStopPointInst>(I);
+ if (MMI && SPI.getContext() && MMI->Verify(SPI.getContext())) {
+ SDOperand Ops[5];
+
+ Ops[0] = getRoot();
+ Ops[1] = getValue(SPI.getLineValue());
+ Ops[2] = getValue(SPI.getColumnValue());
+
+ DebugInfoDesc *DD = MMI->getDescFor(SPI.getContext());
+ assert(DD && "Not a debug information descriptor");
+ CompileUnitDesc *CompileUnit = cast<CompileUnitDesc>(DD);
+
+ Ops[3] = DAG.getString(CompileUnit->getFileName());
+ Ops[4] = DAG.getString(CompileUnit->getDirectory());
+
+ DAG.setRoot(DAG.getNode(ISD::LOCATION, MVT::Other, Ops, 5));
+ }
+
+ return 0;
+ }
+ case Intrinsic::dbg_region_start: {
+ MachineModuleInfo *MMI = DAG.getMachineModuleInfo();
+ DbgRegionStartInst &RSI = cast<DbgRegionStartInst>(I);
+ if (MMI && RSI.getContext() && MMI->Verify(RSI.getContext())) {
+ unsigned LabelID = MMI->RecordRegionStart(RSI.getContext());
+ DAG.setRoot(DAG.getNode(ISD::LABEL, MVT::Other, getRoot(),
+ DAG.getConstant(LabelID, MVT::i32)));
+ }
+
+ return 0;
+ }
+ case Intrinsic::dbg_region_end: {
+ MachineModuleInfo *MMI = DAG.getMachineModuleInfo();
+ DbgRegionEndInst &REI = cast<DbgRegionEndInst>(I);
+ if (MMI && REI.getContext() && MMI->Verify(REI.getContext())) {
+ unsigned LabelID = MMI->RecordRegionEnd(REI.getContext());
+ DAG.setRoot(DAG.getNode(ISD::LABEL, MVT::Other,
+ getRoot(), DAG.getConstant(LabelID, MVT::i32)));
+ }
+
+ return 0;
+ }
+ case Intrinsic::dbg_func_start: {
+ MachineModuleInfo *MMI = DAG.getMachineModuleInfo();
+ DbgFuncStartInst &FSI = cast<DbgFuncStartInst>(I);
+ if (MMI && FSI.getSubprogram() &&
+ MMI->Verify(FSI.getSubprogram())) {
+ unsigned LabelID = MMI->RecordRegionStart(FSI.getSubprogram());
+ DAG.setRoot(DAG.getNode(ISD::LABEL, MVT::Other,
+ getRoot(), DAG.getConstant(LabelID, MVT::i32)));
+ }
+
+ return 0;
+ }
+ case Intrinsic::dbg_declare: {
+ MachineModuleInfo *MMI = DAG.getMachineModuleInfo();
+ DbgDeclareInst &DI = cast<DbgDeclareInst>(I);
+ if (MMI && DI.getVariable() && MMI->Verify(DI.getVariable())) {
+ SDOperand AddressOp = getValue(DI.getAddress());
+ if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(AddressOp))
+ MMI->RecordVariable(DI.getVariable(), FI->getIndex());
+ }
+
+ return 0;
+ }
+
+ case Intrinsic::eh_exception: {
+ if (ExceptionHandling) {
+ if (!CurMBB->isLandingPad()) {
+ // FIXME: Mark exception register as live in. Hack for PR1508.
+ unsigned Reg = TLI.getExceptionAddressRegister();
+ if (Reg) CurMBB->addLiveIn(Reg);
+ }
+ // Insert the EXCEPTIONADDR instruction.
+ SDVTList VTs = DAG.getVTList(TLI.getPointerTy(), MVT::Other);
+ SDOperand Ops[1];
+ Ops[0] = DAG.getRoot();
+ SDOperand Op = DAG.getNode(ISD::EXCEPTIONADDR, VTs, Ops, 1);
+ setValue(&I, Op);
+ DAG.setRoot(Op.getValue(1));
+ } else {
+ setValue(&I, DAG.getConstant(0, TLI.getPointerTy()));
+ }
+ return 0;
+ }
+
+ case Intrinsic::eh_selector:{
+ MachineModuleInfo *MMI = DAG.getMachineModuleInfo();
+
+ if (ExceptionHandling && MMI) {
+ if (CurMBB->isLandingPad())
+ addCatchInfo(I, MMI, CurMBB);
+ else {
+#ifndef NDEBUG
+ FuncInfo.CatchInfoLost.insert(&I);
+#endif
+ // FIXME: Mark exception selector register as live in. Hack for PR1508.
+ unsigned Reg = TLI.getExceptionSelectorRegister();
+ if (Reg) CurMBB->addLiveIn(Reg);
+ }
+
+ // Insert the EHSELECTION instruction.
+ SDVTList VTs = DAG.getVTList(TLI.getPointerTy(), MVT::Other);
+ SDOperand Ops[2];
+ Ops[0] = getValue(I.getOperand(1));
+ Ops[1] = getRoot();
+ SDOperand Op = DAG.getNode(ISD::EHSELECTION, VTs, Ops, 2);
+ setValue(&I, Op);
+ DAG.setRoot(Op.getValue(1));
+ } else {
+ setValue(&I, DAG.getConstant(0, TLI.getPointerTy()));
+ }
+
+ return 0;
+ }
+
+ case Intrinsic::eh_typeid_for: {
+ MachineModuleInfo *MMI = DAG.getMachineModuleInfo();
+
+ if (MMI) {
+ // Find the type id for the given typeinfo.
+ GlobalVariable *GV = ExtractTypeInfo(I.getOperand(1));
+
+ unsigned TypeID = MMI->getTypeIDFor(GV);
+ setValue(&I, DAG.getConstant(TypeID, MVT::i32));
+ } else {
+ // Return something different to eh_selector.
+ setValue(&I, DAG.getConstant(1, MVT::i32));
+ }
+
+ return 0;
+ }
+
+ case Intrinsic::eh_return: {
+ MachineModuleInfo *MMI = DAG.getMachineModuleInfo();
+
+ if (MMI && ExceptionHandling) {
+ MMI->setCallsEHReturn(true);
+ DAG.setRoot(DAG.getNode(ISD::EH_RETURN,
+ MVT::Other,
+ getRoot(),
+ getValue(I.getOperand(1)),
+ getValue(I.getOperand(2))));
+ } else {
+ setValue(&I, DAG.getConstant(0, TLI.getPointerTy()));
+ }
+
+ return 0;
+ }
+
+ case Intrinsic::eh_unwind_init: {
+ if (MachineModuleInfo *MMI = DAG.getMachineModuleInfo()) {
+ MMI->setCallsUnwindInit(true);
+ }
+
+ return 0;
+ }
+
+ case Intrinsic::eh_dwarf_cfa: {
+ if (ExceptionHandling) {
+ MVT::ValueType VT = getValue(I.getOperand(1)).getValueType();
+ SDOperand Offset = DAG.getNode(ISD::ADD,
+ TLI.getPointerTy(),
+ DAG.getNode(ISD::FRAME_TO_ARGS_OFFSET,
+ VT),
+ getValue(I.getOperand(1)));
+ setValue(&I, DAG.getNode(ISD::ADD,
+ TLI.getPointerTy(),
+ DAG.getNode(ISD::FRAMEADDR,
+ TLI.getPointerTy(),
+ DAG.getConstant(0,
+ TLI.getPointerTy())),
+ Offset));
+ } else {
+ setValue(&I, DAG.getConstant(0, TLI.getPointerTy()));
+ }
+
+ return 0;
+ }
+
+ case Intrinsic::sqrt_f32:
+ case Intrinsic::sqrt_f64:
+ setValue(&I, DAG.getNode(ISD::FSQRT,
+ getValue(I.getOperand(1)).getValueType(),
+ getValue(I.getOperand(1))));
+ return 0;
+ case Intrinsic::powi_f32:
+ case Intrinsic::powi_f64:
+ setValue(&I, DAG.getNode(ISD::FPOWI,
+ getValue(I.getOperand(1)).getValueType(),
+ getValue(I.getOperand(1)),
+ getValue(I.getOperand(2))));
+ return 0;
+ case Intrinsic::pcmarker: {
+ SDOperand Tmp = getValue(I.getOperand(1));
+ DAG.setRoot(DAG.getNode(ISD::PCMARKER, MVT::Other, getRoot(), Tmp));
+ return 0;
+ }
+ case Intrinsic::readcyclecounter: {
+ SDOperand Op = getRoot();
+ SDOperand Tmp = DAG.getNode(ISD::READCYCLECOUNTER,
+ DAG.getNodeValueTypes(MVT::i64, MVT::Other), 2,
+ &Op, 1);
+ setValue(&I, Tmp);
+ DAG.setRoot(Tmp.getValue(1));
+ return 0;
+ }
+ case Intrinsic::part_select: {
+ // Currently not implemented: just abort
+ assert(0 && "part_select intrinsic not implemented");
+ abort();
+ }
+ case Intrinsic::part_set: {
+ // Currently not implemented: just abort
+ assert(0 && "part_set intrinsic not implemented");
+ abort();
+ }
+ case Intrinsic::bswap:
+ setValue(&I, DAG.getNode(ISD::BSWAP,
+ getValue(I.getOperand(1)).getValueType(),
+ getValue(I.getOperand(1))));
+ return 0;
+ case Intrinsic::cttz: {
+ SDOperand Arg = getValue(I.getOperand(1));
+ MVT::ValueType Ty = Arg.getValueType();
+ SDOperand result = DAG.getNode(ISD::CTTZ, Ty, Arg);
+ if (Ty < MVT::i32)
+ result = DAG.getNode(ISD::ZERO_EXTEND, MVT::i32, result);
+ else if (Ty > MVT::i32)
+ result = DAG.getNode(ISD::TRUNCATE, MVT::i32, result);
+ setValue(&I, result);
+ return 0;
+ }
+ case Intrinsic::ctlz: {
+ SDOperand Arg = getValue(I.getOperand(1));
+ MVT::ValueType Ty = Arg.getValueType();
+ SDOperand result = DAG.getNode(ISD::CTLZ, Ty, Arg);
+ if (Ty < MVT::i32)
+ result = DAG.getNode(ISD::ZERO_EXTEND, MVT::i32, result);
+ else if (Ty > MVT::i32)
+ result = DAG.getNode(ISD::TRUNCATE, MVT::i32, result);
+ setValue(&I, result);
+ return 0;
+ }
+ case Intrinsic::ctpop: {
+ SDOperand Arg = getValue(I.getOperand(1));
+ MVT::ValueType Ty = Arg.getValueType();
+ SDOperand result = DAG.getNode(ISD::CTPOP, Ty, Arg);
+ if (Ty < MVT::i32)
+ result = DAG.getNode(ISD::ZERO_EXTEND, MVT::i32, result);
+ else if (Ty > MVT::i32)
+ result = DAG.getNode(ISD::TRUNCATE, MVT::i32, result);
+ setValue(&I, result);
+ return 0;
+ }
+ case Intrinsic::stacksave: {
+ SDOperand Op = getRoot();
+ SDOperand Tmp = DAG.getNode(ISD::STACKSAVE,
+ DAG.getNodeValueTypes(TLI.getPointerTy(), MVT::Other), 2, &Op, 1);
+ setValue(&I, Tmp);
+ DAG.setRoot(Tmp.getValue(1));
+ return 0;
+ }
+ case Intrinsic::stackrestore: {
+ SDOperand Tmp = getValue(I.getOperand(1));
+ DAG.setRoot(DAG.getNode(ISD::STACKRESTORE, MVT::Other, getRoot(), Tmp));
+ return 0;
+ }
+ case Intrinsic::prefetch:
+ // FIXME: Currently discarding prefetches.
+ return 0;
+
+ case Intrinsic::var_annotation:
+ // Discard annotate attributes
+ return 0;
+ }
+}
+
+
+void SelectionDAGLowering::LowerCallTo(Instruction &I,
+ const Type *CalledValueTy,
+ unsigned CallingConv,
+ bool IsTailCall,
+ SDOperand Callee, unsigned OpIdx,
+ MachineBasicBlock *LandingPad) {
+ const PointerType *PT = cast<PointerType>(CalledValueTy);
+ const FunctionType *FTy = cast<FunctionType>(PT->getElementType());
+ const ParamAttrsList *Attrs = FTy->getParamAttrs();
+ MachineModuleInfo *MMI = DAG.getMachineModuleInfo();
+ unsigned BeginLabel = 0, EndLabel = 0;
+
+ TargetLowering::ArgListTy Args;
+ TargetLowering::ArgListEntry Entry;
+ Args.reserve(I.getNumOperands());
+ for (unsigned i = OpIdx, e = I.getNumOperands(); i != e; ++i) {
+ Value *Arg = I.getOperand(i);
+ SDOperand ArgNode = getValue(Arg);
+ Entry.Node = ArgNode; Entry.Ty = Arg->getType();
+
+ unsigned attrInd = i - OpIdx + 1;
+ Entry.isSExt = Attrs && Attrs->paramHasAttr(attrInd, ParamAttr::SExt);
+ Entry.isZExt = Attrs && Attrs->paramHasAttr(attrInd, ParamAttr::ZExt);
+ Entry.isInReg = Attrs && Attrs->paramHasAttr(attrInd, ParamAttr::InReg);
+ Entry.isSRet = Attrs && Attrs->paramHasAttr(attrInd, ParamAttr::StructRet);
+ Args.push_back(Entry);
+ }
+
+ if (ExceptionHandling && MMI) {
+ // Insert a label before the invoke call to mark the try range. This can be
+ // used to detect deletion of the invoke via the MachineModuleInfo.
+ BeginLabel = MMI->NextLabelID();
+ DAG.setRoot(DAG.getNode(ISD::LABEL, MVT::Other, getRoot(),
+ DAG.getConstant(BeginLabel, MVT::i32)));
+ }
+
+ std::pair<SDOperand,SDOperand> Result =
+ TLI.LowerCallTo(getRoot(), I.getType(),
+ Attrs && Attrs->paramHasAttr(0, ParamAttr::SExt),
+ FTy->isVarArg(), CallingConv, IsTailCall,
+ Callee, Args, DAG);
+ if (I.getType() != Type::VoidTy)
+ setValue(&I, Result.first);
+ DAG.setRoot(Result.second);
+
+ if (ExceptionHandling && MMI) {
+ // Insert a label at the end of the invoke call to mark the try range. This
+ // can be used to detect deletion of the invoke via the MachineModuleInfo.
+ EndLabel = MMI->NextLabelID();
+ DAG.setRoot(DAG.getNode(ISD::LABEL, MVT::Other, getRoot(),
+ DAG.getConstant(EndLabel, MVT::i32)));
+
+ // Inform MachineModuleInfo of range.
+ MMI->addInvoke(LandingPad, BeginLabel, EndLabel);
+ }
+}
+
+
+void SelectionDAGLowering::visitCall(CallInst &I) {
+ const char *RenameFn = 0;
+ if (Function *F = I.getCalledFunction()) {
+ if (F->isDeclaration())
+ if (unsigned IID = F->getIntrinsicID()) {
+ RenameFn = visitIntrinsicCall(I, IID);
+ if (!RenameFn)
+ return;
+ } else { // Not an LLVM intrinsic.
+ const std::string &Name = F->getName();
+ if (Name[0] == 'c' && (Name == "copysign" || Name == "copysignf")) {
+ if (I.getNumOperands() == 3 && // Basic sanity checks.
+ I.getOperand(1)->getType()->isFloatingPoint() &&
+ I.getType() == I.getOperand(1)->getType() &&
+ I.getType() == I.getOperand(2)->getType()) {
+ SDOperand LHS = getValue(I.getOperand(1));
+ SDOperand RHS = getValue(I.getOperand(2));
+ setValue(&I, DAG.getNode(ISD::FCOPYSIGN, LHS.getValueType(),
+ LHS, RHS));
+ return;
+ }
+ } else if (Name[0] == 'f' && (Name == "fabs" || Name == "fabsf")) {
+ if (I.getNumOperands() == 2 && // Basic sanity checks.
+ I.getOperand(1)->getType()->isFloatingPoint() &&
+ I.getType() == I.getOperand(1)->getType()) {
+ SDOperand Tmp = getValue(I.getOperand(1));
+ setValue(&I, DAG.getNode(ISD::FABS, Tmp.getValueType(), Tmp));
+ return;
+ }
+ } else if (Name[0] == 's' && (Name == "sin" || Name == "sinf")) {
+ if (I.getNumOperands() == 2 && // Basic sanity checks.
+ I.getOperand(1)->getType()->isFloatingPoint() &&
+ I.getType() == I.getOperand(1)->getType()) {
+ SDOperand Tmp = getValue(I.getOperand(1));
+ setValue(&I, DAG.getNode(ISD::FSIN, Tmp.getValueType(), Tmp));
+ return;
+ }
+ } else if (Name[0] == 'c' && (Name == "cos" || Name == "cosf")) {
+ if (I.getNumOperands() == 2 && // Basic sanity checks.
+ I.getOperand(1)->getType()->isFloatingPoint() &&
+ I.getType() == I.getOperand(1)->getType()) {
+ SDOperand Tmp = getValue(I.getOperand(1));
+ setValue(&I, DAG.getNode(ISD::FCOS, Tmp.getValueType(), Tmp));
+ return;
+ }
+ }
+ }
+ } else if (isa<InlineAsm>(I.getOperand(0))) {
+ visitInlineAsm(I);
+ return;
+ }
+
+ SDOperand Callee;
+ if (!RenameFn)
+ Callee = getValue(I.getOperand(0));
+ else
+ Callee = DAG.getExternalSymbol(RenameFn, TLI.getPointerTy());
+
+ LowerCallTo(I, I.getCalledValue()->getType(),
+ I.getCallingConv(),
+ I.isTailCall(),
+ Callee,
+ 1);
+}
+
+
+/// getCopyFromRegs - Emit a series of CopyFromReg nodes that copies from
+/// this value and returns the result as a ValueVT value. This uses
+/// Chain/Flag as the input and updates them for the output Chain/Flag.
+/// If the Flag pointer is NULL, no flag is used.
+SDOperand RegsForValue::getCopyFromRegs(SelectionDAG &DAG,
+ SDOperand &Chain, SDOperand *Flag)const{
+ // Copy the legal parts from the registers.
+ unsigned NumParts = Regs.size();
+ SmallVector<SDOperand, 8> Parts(NumParts);
+ for (unsigned i = 0; i != NumParts; ++i) {
+ SDOperand Part = Flag ?
+ DAG.getCopyFromReg(Chain, Regs[i], RegVT, *Flag) :
+ DAG.getCopyFromReg(Chain, Regs[i], RegVT);
+ Chain = Part.getValue(1);
+ if (Flag)
+ *Flag = Part.getValue(2);
+ Parts[i] = Part;
+ }
+
+ // Assemble the legal parts into the final value.
+ return getCopyFromParts(DAG, &Parts[0], NumParts, RegVT, ValueVT);
+}
+
+/// getCopyToRegs - Emit a series of CopyToReg nodes that copies the
+/// specified value into the registers specified by this object. This uses
+/// Chain/Flag as the input and updates them for the output Chain/Flag.
+/// If the Flag pointer is NULL, no flag is used.
+void RegsForValue::getCopyToRegs(SDOperand Val, SelectionDAG &DAG,
+ SDOperand &Chain, SDOperand *Flag) const {
+ // Get the list of the values's legal parts.
+ unsigned NumParts = Regs.size();
+ SmallVector<SDOperand, 8> Parts(NumParts);
+ getCopyToParts(DAG, Val, &Parts[0], NumParts, RegVT);
+
+ // Copy the parts into the registers.
+ for (unsigned i = 0; i != NumParts; ++i) {
+ SDOperand Part = Flag ?
+ DAG.getCopyToReg(Chain, Regs[i], Parts[i], *Flag) :
+ DAG.getCopyToReg(Chain, Regs[i], Parts[i]);
+ Chain = Part.getValue(0);
+ if (Flag)
+ *Flag = Part.getValue(1);
+ }
+}
+
+/// AddInlineAsmOperands - Add this value to the specified inlineasm node
+/// operand list. This adds the code marker and includes the number of
+/// values added into it.
+void RegsForValue::AddInlineAsmOperands(unsigned Code, SelectionDAG &DAG,
+ std::vector<SDOperand> &Ops) const {
+ MVT::ValueType IntPtrTy = DAG.getTargetLoweringInfo().getPointerTy();
+ Ops.push_back(DAG.getTargetConstant(Code | (Regs.size() << 3), IntPtrTy));
+ for (unsigned i = 0, e = Regs.size(); i != e; ++i)
+ Ops.push_back(DAG.getRegister(Regs[i], RegVT));
+}
+
+/// isAllocatableRegister - If the specified register is safe to allocate,
+/// i.e. it isn't a stack pointer or some other special register, return the
+/// register class for the register. Otherwise, return null.
+static const TargetRegisterClass *
+isAllocatableRegister(unsigned Reg, MachineFunction &MF,
+ const TargetLowering &TLI, const MRegisterInfo *MRI) {
+ MVT::ValueType FoundVT = MVT::Other;
+ const TargetRegisterClass *FoundRC = 0;
+ for (MRegisterInfo::regclass_iterator RCI = MRI->regclass_begin(),
+ E = MRI->regclass_end(); RCI != E; ++RCI) {
+ MVT::ValueType ThisVT = MVT::Other;
+
+ const TargetRegisterClass *RC = *RCI;
+ // If none of the the value types for this register class are valid, we
+ // can't use it. For example, 64-bit reg classes on 32-bit targets.
+ for (TargetRegisterClass::vt_iterator I = RC->vt_begin(), E = RC->vt_end();
+ I != E; ++I) {
+ if (TLI.isTypeLegal(*I)) {
+ // If we have already found this register in a different register class,
+ // choose the one with the largest VT specified. For example, on
+ // PowerPC, we favor f64 register classes over f32.
+ if (FoundVT == MVT::Other ||
+ MVT::getSizeInBits(FoundVT) < MVT::getSizeInBits(*I)) {
+ ThisVT = *I;
+ break;
+ }
+ }
+ }
+
+ if (ThisVT == MVT::Other) continue;
+
+ // NOTE: This isn't ideal. In particular, this might allocate the
+ // frame pointer in functions that need it (due to them not being taken
+ // out of allocation, because a variable sized allocation hasn't been seen
+ // yet). This is a slight code pessimization, but should still work.
+ for (TargetRegisterClass::iterator I = RC->allocation_order_begin(MF),
+ E = RC->allocation_order_end(MF); I != E; ++I)
+ if (*I == Reg) {
+ // We found a matching register class. Keep looking at others in case
+ // we find one with larger registers that this physreg is also in.
+ FoundRC = RC;
+ FoundVT = ThisVT;
+ break;
+ }
+ }
+ return FoundRC;
+}
+
+
+namespace {
+/// AsmOperandInfo - This contains information for each constraint that we are
+/// lowering.
+struct AsmOperandInfo : public InlineAsm::ConstraintInfo {
+ /// ConstraintCode - This contains the actual string for the code, like "m".
+ std::string ConstraintCode;
+
+ /// ConstraintType - Information about the constraint code, e.g. Register,
+ /// RegisterClass, Memory, Other, Unknown.
+ TargetLowering::ConstraintType ConstraintType;
+
+ /// CallOperand/CallOperandval - If this is the result output operand or a
+ /// clobber, this is null, otherwise it is the incoming operand to the
+ /// CallInst. This gets modified as the asm is processed.
+ SDOperand CallOperand;
+ Value *CallOperandVal;
+
+ /// ConstraintVT - The ValueType for the operand value.
+ MVT::ValueType ConstraintVT;
+
+ /// AssignedRegs - If this is a register or register class operand, this
+ /// contains the set of register corresponding to the operand.
+ RegsForValue AssignedRegs;
+
+ AsmOperandInfo(const InlineAsm::ConstraintInfo &info)
+ : InlineAsm::ConstraintInfo(info),
+ ConstraintType(TargetLowering::C_Unknown),
+ CallOperand(0,0), CallOperandVal(0), ConstraintVT(MVT::Other) {
+ }
+
+ void ComputeConstraintToUse(const TargetLowering &TLI);
+
+ /// MarkAllocatedRegs - Once AssignedRegs is set, mark the assigned registers
+ /// busy in OutputRegs/InputRegs.
+ void MarkAllocatedRegs(bool isOutReg, bool isInReg,
+ std::set<unsigned> &OutputRegs,
+ std::set<unsigned> &InputRegs) const {
+ if (isOutReg)
+ OutputRegs.insert(AssignedRegs.Regs.begin(), AssignedRegs.Regs.end());
+ if (isInReg)
+ InputRegs.insert(AssignedRegs.Regs.begin(), AssignedRegs.Regs.end());
+ }
+};
+} // end anon namespace.
+
+/// getConstraintGenerality - Return an integer indicating how general CT is.
+static unsigned getConstraintGenerality(TargetLowering::ConstraintType CT) {
+ switch (CT) {
+ default: assert(0 && "Unknown constraint type!");
+ case TargetLowering::C_Other:
+ case TargetLowering::C_Unknown:
+ return 0;
+ case TargetLowering::C_Register:
+ return 1;
+ case TargetLowering::C_RegisterClass:
+ return 2;
+ case TargetLowering::C_Memory:
+ return 3;
+ }
+}
+
+void AsmOperandInfo::ComputeConstraintToUse(const TargetLowering &TLI) {
+ assert(!Codes.empty() && "Must have at least one constraint");
+
+ std::string *Current = &Codes[0];
+ TargetLowering::ConstraintType CurType = TLI.getConstraintType(*Current);
+ if (Codes.size() == 1) { // Single-letter constraints ('r') are very common.
+ ConstraintCode = *Current;
+ ConstraintType = CurType;
+ return;
+ }
+
+ unsigned CurGenerality = getConstraintGenerality(CurType);
+
+ // If we have multiple constraints, try to pick the most general one ahead
+ // of time. This isn't a wonderful solution, but handles common cases.
+ for (unsigned j = 1, e = Codes.size(); j != e; ++j) {
+ TargetLowering::ConstraintType ThisType = TLI.getConstraintType(Codes[j]);
+ unsigned ThisGenerality = getConstraintGenerality(ThisType);
+ if (ThisGenerality > CurGenerality) {
+ // This constraint letter is more general than the previous one,
+ // use it.
+ CurType = ThisType;
+ Current = &Codes[j];
+ CurGenerality = ThisGenerality;
+ }
+ }
+
+ ConstraintCode = *Current;
+ ConstraintType = CurType;
+}
+
+
+void SelectionDAGLowering::
+GetRegistersForValue(AsmOperandInfo &OpInfo, bool HasEarlyClobber,
+ std::set<unsigned> &OutputRegs,
+ std::set<unsigned> &InputRegs) {
+ // Compute whether this value requires an input register, an output register,
+ // or both.
+ bool isOutReg = false;
+ bool isInReg = false;
+ switch (OpInfo.Type) {
+ case InlineAsm::isOutput:
+ isOutReg = true;
+
+ // If this is an early-clobber output, or if there is an input
+ // constraint that matches this, we need to reserve the input register
+ // so no other inputs allocate to it.
+ isInReg = OpInfo.isEarlyClobber || OpInfo.hasMatchingInput;
+ break;
+ case InlineAsm::isInput:
+ isInReg = true;
+ isOutReg = false;
+ break;
+ case InlineAsm::isClobber:
+ isOutReg = true;
+ isInReg = true;
+ break;
+ }
+
+
+ MachineFunction &MF = DAG.getMachineFunction();
+ std::vector<unsigned> Regs;
+
+ // If this is a constraint for a single physreg, or a constraint for a
+ // register class, find it.
+ std::pair<unsigned, const TargetRegisterClass*> PhysReg =
+ TLI.getRegForInlineAsmConstraint(OpInfo.ConstraintCode,
+ OpInfo.ConstraintVT);
+
+ unsigned NumRegs = 1;
+ if (OpInfo.ConstraintVT != MVT::Other)
+ NumRegs = TLI.getNumRegisters(OpInfo.ConstraintVT);
+ MVT::ValueType RegVT;
+ MVT::ValueType ValueVT = OpInfo.ConstraintVT;
+
+
+ // If this is a constraint for a specific physical register, like {r17},
+ // assign it now.
+ if (PhysReg.first) {
+ if (OpInfo.ConstraintVT == MVT::Other)
+ ValueVT = *PhysReg.second->vt_begin();
+
+ // Get the actual register value type. This is important, because the user
+ // may have asked for (e.g.) the AX register in i32 type. We need to
+ // remember that AX is actually i16 to get the right extension.
+ RegVT = *PhysReg.second->vt_begin();
+
+ // This is a explicit reference to a physical register.
+ Regs.push_back(PhysReg.first);
+
+ // If this is an expanded reference, add the rest of the regs to Regs.
+ if (NumRegs != 1) {
+ TargetRegisterClass::iterator I = PhysReg.second->begin();
+ TargetRegisterClass::iterator E = PhysReg.second->end();
+ for (; *I != PhysReg.first; ++I)
+ assert(I != E && "Didn't find reg!");
+
+ // Already added the first reg.
+ --NumRegs; ++I;
+ for (; NumRegs; --NumRegs, ++I) {
+ assert(I != E && "Ran out of registers to allocate!");
+ Regs.push_back(*I);
+ }
+ }
+ OpInfo.AssignedRegs = RegsForValue(Regs, RegVT, ValueVT);
+ OpInfo.MarkAllocatedRegs(isOutReg, isInReg, OutputRegs, InputRegs);
+ return;
+ }
+
+ // Otherwise, if this was a reference to an LLVM register class, create vregs
+ // for this reference.
+ std::vector<unsigned> RegClassRegs;
+ const TargetRegisterClass *RC = PhysReg.second;
+ if (RC) {
+ // If this is an early clobber or tied register, our regalloc doesn't know
+ // how to maintain the constraint. If it isn't, go ahead and create vreg
+ // and let the regalloc do the right thing.
+ if (!OpInfo.hasMatchingInput && !OpInfo.isEarlyClobber &&
+ // If there is some other early clobber and this is an input register,
+ // then we are forced to pre-allocate the input reg so it doesn't
+ // conflict with the earlyclobber.
+ !(OpInfo.Type == InlineAsm::isInput && HasEarlyClobber)) {
+ RegVT = *PhysReg.second->vt_begin();
+
+ if (OpInfo.ConstraintVT == MVT::Other)
+ ValueVT = RegVT;
+
+ // Create the appropriate number of virtual registers.
+ SSARegMap *RegMap = MF.getSSARegMap();
+ for (; NumRegs; --NumRegs)
+ Regs.push_back(RegMap->createVirtualRegister(PhysReg.second));
+
+ OpInfo.AssignedRegs = RegsForValue(Regs, RegVT, ValueVT);
+ OpInfo.MarkAllocatedRegs(isOutReg, isInReg, OutputRegs, InputRegs);
+ return;
+ }
+
+ // Otherwise, we can't allocate it. Let the code below figure out how to
+ // maintain these constraints.
+ RegClassRegs.assign(PhysReg.second->begin(), PhysReg.second->end());
+
+ } else {
+ // This is a reference to a register class that doesn't directly correspond
+ // to an LLVM register class. Allocate NumRegs consecutive, available,
+ // registers from the class.
+ RegClassRegs = TLI.getRegClassForInlineAsmConstraint(OpInfo.ConstraintCode,
+ OpInfo.ConstraintVT);
+ }
+
+ const MRegisterInfo *MRI = DAG.getTarget().getRegisterInfo();
+ unsigned NumAllocated = 0;
+ for (unsigned i = 0, e = RegClassRegs.size(); i != e; ++i) {
+ unsigned Reg = RegClassRegs[i];
+ // See if this register is available.
+ if ((isOutReg && OutputRegs.count(Reg)) || // Already used.
+ (isInReg && InputRegs.count(Reg))) { // Already used.
+ // Make sure we find consecutive registers.
+ NumAllocated = 0;
+ continue;
+ }
+
+ // Check to see if this register is allocatable (i.e. don't give out the
+ // stack pointer).
+ if (RC == 0) {
+ RC = isAllocatableRegister(Reg, MF, TLI, MRI);
+ if (!RC) { // Couldn't allocate this register.
+ // Reset NumAllocated to make sure we return consecutive registers.
+ NumAllocated = 0;
+ continue;
+ }
+ }
+
+ // Okay, this register is good, we can use it.
+ ++NumAllocated;
+
+ // If we allocated enough consecutive registers, succeed.
+ if (NumAllocated == NumRegs) {
+ unsigned RegStart = (i-NumAllocated)+1;
+ unsigned RegEnd = i+1;
+ // Mark all of the allocated registers used.
+ for (unsigned i = RegStart; i != RegEnd; ++i)
+ Regs.push_back(RegClassRegs[i]);
+
+ OpInfo.AssignedRegs = RegsForValue(Regs, *RC->vt_begin(),
+ OpInfo.ConstraintVT);
+ OpInfo.MarkAllocatedRegs(isOutReg, isInReg, OutputRegs, InputRegs);
+ return;
+ }
+ }
+
+ // Otherwise, we couldn't allocate enough registers for this.
+ return;
+}
+
+
+/// visitInlineAsm - Handle a call to an InlineAsm object.
+///
+void SelectionDAGLowering::visitInlineAsm(CallInst &I) {
+ InlineAsm *IA = cast<InlineAsm>(I.getOperand(0));
+
+ /// ConstraintOperands - Information about all of the constraints.
+ std::vector<AsmOperandInfo> ConstraintOperands;
+
+ SDOperand Chain = getRoot();
+ SDOperand Flag;
+
+ std::set<unsigned> OutputRegs, InputRegs;
+
+ // Do a prepass over the constraints, canonicalizing them, and building up the
+ // ConstraintOperands list.
+ std::vector<InlineAsm::ConstraintInfo>
+ ConstraintInfos = IA->ParseConstraints();
+
+ // SawEarlyClobber - Keep track of whether we saw an earlyclobber output
+ // constraint. If so, we can't let the register allocator allocate any input
+ // registers, because it will not know to avoid the earlyclobbered output reg.
+ bool SawEarlyClobber = false;
+
+ unsigned OpNo = 1; // OpNo - The operand of the CallInst.
+ for (unsigned i = 0, e = ConstraintInfos.size(); i != e; ++i) {
+ ConstraintOperands.push_back(AsmOperandInfo(ConstraintInfos[i]));
+ AsmOperandInfo &OpInfo = ConstraintOperands.back();
+
+ MVT::ValueType OpVT = MVT::Other;
+
+ // Compute the value type for each operand.
+ switch (OpInfo.Type) {
+ case InlineAsm::isOutput:
+ if (!OpInfo.isIndirect) {
+ // The return value of the call is this value. As such, there is no
+ // corresponding argument.
+ assert(I.getType() != Type::VoidTy && "Bad inline asm!");
+ OpVT = TLI.getValueType(I.getType());
+ } else {
+ OpInfo.CallOperandVal = I.getOperand(OpNo++);
+ }
+ break;
+ case InlineAsm::isInput:
+ OpInfo.CallOperandVal = I.getOperand(OpNo++);
+ break;
+ case InlineAsm::isClobber:
+ // Nothing to do.
+ break;
+ }
+
+ // If this is an input or an indirect output, process the call argument.
+ if (OpInfo.CallOperandVal) {
+ OpInfo.CallOperand = getValue(OpInfo.CallOperandVal);
+ const Type *OpTy = OpInfo.CallOperandVal->getType();
+ // If this is an indirect operand, the operand is a pointer to the
+ // accessed type.
+ if (OpInfo.isIndirect)
+ OpTy = cast<PointerType>(OpTy)->getElementType();
+
+ // If OpTy is not a first-class value, it may be a struct/union that we
+ // can tile with integers.
+ if (!OpTy->isFirstClassType() && OpTy->isSized()) {
+ unsigned BitSize = TD->getTypeSizeInBits(OpTy);
+ switch (BitSize) {
+ default: break;
+ case 1:
+ case 8:
+ case 16:
+ case 32:
+ case 64:
+ OpTy = IntegerType::get(BitSize);
+ break;
+ }
+ }
+
+ OpVT = TLI.getValueType(OpTy, true);
+ }
+
+ OpInfo.ConstraintVT = OpVT;
+
+ // Compute the constraint code and ConstraintType to use.
+ OpInfo.ComputeConstraintToUse(TLI);
+
+ // Keep track of whether we see an earlyclobber.
+ SawEarlyClobber |= OpInfo.isEarlyClobber;
+
+ // If this is a memory input, and if the operand is not indirect, do what we
+ // need to to provide an address for the memory input.
+ if (OpInfo.ConstraintType == TargetLowering::C_Memory &&
+ !OpInfo.isIndirect) {
+ assert(OpInfo.Type == InlineAsm::isInput &&
+ "Can only indirectify direct input operands!");
+
+ // Memory operands really want the address of the value. If we don't have
+ // an indirect input, put it in the constpool if we can, otherwise spill
+ // it to a stack slot.
+
+ // If the operand is a float, integer, or vector constant, spill to a
+ // constant pool entry to get its address.
+ Value *OpVal = OpInfo.CallOperandVal;
+ if (isa<ConstantFP>(OpVal) || isa<ConstantInt>(OpVal) ||
+ isa<ConstantVector>(OpVal)) {
+ OpInfo.CallOperand = DAG.getConstantPool(cast<Constant>(OpVal),
+ TLI.getPointerTy());
+ } else {
+ // Otherwise, create a stack slot and emit a store to it before the
+ // asm.
+ const Type *Ty = OpVal->getType();
+ uint64_t TySize = TLI.getTargetData()->getTypeSize(Ty);
+ unsigned Align = TLI.getTargetData()->getPrefTypeAlignment(Ty);
+ MachineFunction &MF = DAG.getMachineFunction();
+ int SSFI = MF.getFrameInfo()->CreateStackObject(TySize, Align);
+ SDOperand StackSlot = DAG.getFrameIndex(SSFI, TLI.getPointerTy());
+ Chain = DAG.getStore(Chain, OpInfo.CallOperand, StackSlot, NULL, 0);
+ OpInfo.CallOperand = StackSlot;
+ }
+
+ // There is no longer a Value* corresponding to this operand.
+ OpInfo.CallOperandVal = 0;
+ // It is now an indirect operand.
+ OpInfo.isIndirect = true;
+ }
+
+ // If this constraint is for a specific register, allocate it before
+ // anything else.
+ if (OpInfo.ConstraintType == TargetLowering::C_Register)
+ GetRegistersForValue(OpInfo, SawEarlyClobber, OutputRegs, InputRegs);
+ }
+ ConstraintInfos.clear();
+
+
+ // Second pass - Loop over all of the operands, assigning virtual or physregs
+ // to registerclass operands.
+ for (unsigned i = 0, e = ConstraintOperands.size(); i != e; ++i) {
+ AsmOperandInfo &OpInfo = ConstraintOperands[i];
+
+ // C_Register operands have already been allocated, Other/Memory don't need
+ // to be.
+ if (OpInfo.ConstraintType == TargetLowering::C_RegisterClass)
+ GetRegistersForValue(OpInfo, SawEarlyClobber, OutputRegs, InputRegs);
+ }
+
+ // AsmNodeOperands - The operands for the ISD::INLINEASM node.
+ std::vector<SDOperand> AsmNodeOperands;
+ AsmNodeOperands.push_back(SDOperand()); // reserve space for input chain
+ AsmNodeOperands.push_back(
+ DAG.getTargetExternalSymbol(IA->getAsmString().c_str(), MVT::Other));
+
+
+ // Loop over all of the inputs, copying the operand values into the
+ // appropriate registers and processing the output regs.
+ RegsForValue RetValRegs;
+
+ // IndirectStoresToEmit - The set of stores to emit after the inline asm node.
+ std::vector<std::pair<RegsForValue, Value*> > IndirectStoresToEmit;
+
+ for (unsigned i = 0, e = ConstraintOperands.size(); i != e; ++i) {
+ AsmOperandInfo &OpInfo = ConstraintOperands[i];
+
+ switch (OpInfo.Type) {
+ case InlineAsm::isOutput: {
+ if (OpInfo.ConstraintType != TargetLowering::C_RegisterClass &&
+ OpInfo.ConstraintType != TargetLowering::C_Register) {
+ // Memory output, or 'other' output (e.g. 'X' constraint).
+ assert(OpInfo.isIndirect && "Memory output must be indirect operand");
+
+ // Add information to the INLINEASM node to know about this output.
+ unsigned ResOpType = 4/*MEM*/ | (1 << 3);
+ AsmNodeOperands.push_back(DAG.getTargetConstant(ResOpType,
+ TLI.getPointerTy()));
+ AsmNodeOperands.push_back(OpInfo.CallOperand);
+ break;
+ }
+
+ // Otherwise, this is a register or register class output.
+
+ // Copy the output from the appropriate register. Find a register that
+ // we can use.
+ if (OpInfo.AssignedRegs.Regs.empty()) {
+ cerr << "Couldn't allocate output reg for contraint '"
+ << OpInfo.ConstraintCode << "'!\n";
+ exit(1);
+ }
+
+ if (!OpInfo.isIndirect) {
+ // This is the result value of the call.
+ assert(RetValRegs.Regs.empty() &&
+ "Cannot have multiple output constraints yet!");
+ assert(I.getType() != Type::VoidTy && "Bad inline asm!");
+ RetValRegs = OpInfo.AssignedRegs;
+ } else {
+ IndirectStoresToEmit.push_back(std::make_pair(OpInfo.AssignedRegs,
+ OpInfo.CallOperandVal));
+ }
+
+ // Add information to the INLINEASM node to know that this register is
+ // set.
+ OpInfo.AssignedRegs.AddInlineAsmOperands(2 /*REGDEF*/, DAG,
+ AsmNodeOperands);
+ break;
+ }
+ case InlineAsm::isInput: {
+ SDOperand InOperandVal = OpInfo.CallOperand;
+
+ if (isdigit(OpInfo.ConstraintCode[0])) { // Matching constraint?
+ // If this is required to match an output register we have already set,
+ // just use its register.
+ unsigned OperandNo = atoi(OpInfo.ConstraintCode.c_str());
+
+ // Scan until we find the definition we already emitted of this operand.
+ // When we find it, create a RegsForValue operand.
+ unsigned CurOp = 2; // The first operand.
+ for (; OperandNo; --OperandNo) {
+ // Advance to the next operand.
+ unsigned NumOps =
+ cast<ConstantSDNode>(AsmNodeOperands[CurOp])->getValue();
+ assert(((NumOps & 7) == 2 /*REGDEF*/ ||
+ (NumOps & 7) == 4 /*MEM*/) &&
+ "Skipped past definitions?");
+ CurOp += (NumOps>>3)+1;
+ }
+
+ unsigned NumOps =
+ cast<ConstantSDNode>(AsmNodeOperands[CurOp])->getValue();
+ if ((NumOps & 7) == 2 /*REGDEF*/) {
+ // Add NumOps>>3 registers to MatchedRegs.
+ RegsForValue MatchedRegs;
+ MatchedRegs.ValueVT = InOperandVal.getValueType();
+ MatchedRegs.RegVT = AsmNodeOperands[CurOp+1].getValueType();
+ for (unsigned i = 0, e = NumOps>>3; i != e; ++i) {
+ unsigned Reg =
+ cast<RegisterSDNode>(AsmNodeOperands[++CurOp])->getReg();
+ MatchedRegs.Regs.push_back(Reg);
+ }
+
+ // Use the produced MatchedRegs object to
+ MatchedRegs.getCopyToRegs(InOperandVal, DAG, Chain, &Flag);
+ MatchedRegs.AddInlineAsmOperands(1 /*REGUSE*/, DAG, AsmNodeOperands);
+ break;
+ } else {
+ assert((NumOps & 7) == 4/*MEM*/ && "Unknown matching constraint!");
+ assert(0 && "matching constraints for memory operands unimp");
+ }
+ }
+
+ if (OpInfo.ConstraintType == TargetLowering::C_Other) {
+ assert(!OpInfo.isIndirect &&
+ "Don't know how to handle indirect other inputs yet!");
+
+ InOperandVal = TLI.isOperandValidForConstraint(InOperandVal,
+ OpInfo.ConstraintCode[0],
+ DAG);
+ if (!InOperandVal.Val) {
+ cerr << "Invalid operand for inline asm constraint '"
+ << OpInfo.ConstraintCode << "'!\n";
+ exit(1);
+ }
+
+ // Add information to the INLINEASM node to know about this input.
+ unsigned ResOpType = 3 /*IMM*/ | (1 << 3);
+ AsmNodeOperands.push_back(DAG.getTargetConstant(ResOpType,
+ TLI.getPointerTy()));
+ AsmNodeOperands.push_back(InOperandVal);
+ break;
+ } else if (OpInfo.ConstraintType == TargetLowering::C_Memory) {
+ assert(OpInfo.isIndirect && "Operand must be indirect to be a mem!");
+ assert(InOperandVal.getValueType() == TLI.getPointerTy() &&
+ "Memory operands expect pointer values");
+
+ // Add information to the INLINEASM node to know about this input.
+ unsigned ResOpType = 4/*MEM*/ | (1 << 3);
+ AsmNodeOperands.push_back(DAG.getTargetConstant(ResOpType,
+ TLI.getPointerTy()));
+ AsmNodeOperands.push_back(InOperandVal);
+ break;
+ }
+
+ assert((OpInfo.ConstraintType == TargetLowering::C_RegisterClass ||
+ OpInfo.ConstraintType == TargetLowering::C_Register) &&
+ "Unknown constraint type!");
+ assert(!OpInfo.isIndirect &&
+ "Don't know how to handle indirect register inputs yet!");
+
+ // Copy the input into the appropriate registers.
+ assert(!OpInfo.AssignedRegs.Regs.empty() &&
+ "Couldn't allocate input reg!");
+
+ OpInfo.AssignedRegs.getCopyToRegs(InOperandVal, DAG, Chain, &Flag);
+
+ OpInfo.AssignedRegs.AddInlineAsmOperands(1/*REGUSE*/, DAG,
+ AsmNodeOperands);
+ break;
+ }
+ case InlineAsm::isClobber: {
+ // Add the clobbered value to the operand list, so that the register
+ // allocator is aware that the physreg got clobbered.
+ if (!OpInfo.AssignedRegs.Regs.empty())
+ OpInfo.AssignedRegs.AddInlineAsmOperands(2/*REGDEF*/, DAG,
+ AsmNodeOperands);
+ break;
+ }
+ }
+ }
+
+ // Finish up input operands.
+ AsmNodeOperands[0] = Chain;
+ if (Flag.Val) AsmNodeOperands.push_back(Flag);
+
+ Chain = DAG.getNode(ISD::INLINEASM,
+ DAG.getNodeValueTypes(MVT::Other, MVT::Flag), 2,
+ &AsmNodeOperands[0], AsmNodeOperands.size());
+ Flag = Chain.getValue(1);
+
+ // If this asm returns a register value, copy the result from that register
+ // and set it as the value of the call.
+ if (!RetValRegs.Regs.empty()) {
+ SDOperand Val = RetValRegs.getCopyFromRegs(DAG, Chain, &Flag);
+
+ // If the result of the inline asm is a vector, it may have the wrong
+ // width/num elts. Make sure to convert it to the right type with
+ // bit_convert.
+ if (MVT::isVector(Val.getValueType())) {
+ const VectorType *VTy = cast<VectorType>(I.getType());
+ MVT::ValueType DesiredVT = TLI.getValueType(VTy);
+
+ Val = DAG.getNode(ISD::BIT_CONVERT, DesiredVT, Val);
+ }
+
+ setValue(&I, Val);
+ }
+
+ std::vector<std::pair<SDOperand, Value*> > StoresToEmit;
+
+ // Process indirect outputs, first output all of the flagged copies out of
+ // physregs.
+ for (unsigned i = 0, e = IndirectStoresToEmit.size(); i != e; ++i) {
+ RegsForValue &OutRegs = IndirectStoresToEmit[i].first;
+ Value *Ptr = IndirectStoresToEmit[i].second;
+ SDOperand OutVal = OutRegs.getCopyFromRegs(DAG, Chain, &Flag);
+ StoresToEmit.push_back(std::make_pair(OutVal, Ptr));
+ }
+
+ // Emit the non-flagged stores from the physregs.
+ SmallVector<SDOperand, 8> OutChains;
+ for (unsigned i = 0, e = StoresToEmit.size(); i != e; ++i)
+ OutChains.push_back(DAG.getStore(Chain, StoresToEmit[i].first,
+ getValue(StoresToEmit[i].second),
+ StoresToEmit[i].second, 0));
+ if (!OutChains.empty())
+ Chain = DAG.getNode(ISD::TokenFactor, MVT::Other,
+ &OutChains[0], OutChains.size());
+ DAG.setRoot(Chain);
+}
+
+
+void SelectionDAGLowering::visitMalloc(MallocInst &I) {
+ SDOperand Src = getValue(I.getOperand(0));
+
+ MVT::ValueType IntPtr = TLI.getPointerTy();
+
+ if (IntPtr < Src.getValueType())
+ Src = DAG.getNode(ISD::TRUNCATE, IntPtr, Src);
+ else if (IntPtr > Src.getValueType())
+ Src = DAG.getNode(ISD::ZERO_EXTEND, IntPtr, Src);
+
+ // Scale the source by the type size.
+ uint64_t ElementSize = TD->getTypeSize(I.getType()->getElementType());
+ Src = DAG.getNode(ISD::MUL, Src.getValueType(),
+ Src, getIntPtrConstant(ElementSize));
+
+ TargetLowering::ArgListTy Args;
+ TargetLowering::ArgListEntry Entry;
+ Entry.Node = Src;
+ Entry.Ty = TLI.getTargetData()->getIntPtrType();
+ Args.push_back(Entry);
+
+ std::pair<SDOperand,SDOperand> Result =
+ TLI.LowerCallTo(getRoot(), I.getType(), false, false, CallingConv::C, true,
+ DAG.getExternalSymbol("malloc", IntPtr),
+ Args, DAG);
+ setValue(&I, Result.first); // Pointers always fit in registers
+ DAG.setRoot(Result.second);
+}
+
+void SelectionDAGLowering::visitFree(FreeInst &I) {
+ TargetLowering::ArgListTy Args;
+ TargetLowering::ArgListEntry Entry;
+ Entry.Node = getValue(I.getOperand(0));
+ Entry.Ty = TLI.getTargetData()->getIntPtrType();
+ Args.push_back(Entry);
+ MVT::ValueType IntPtr = TLI.getPointerTy();
+ std::pair<SDOperand,SDOperand> Result =
+ TLI.LowerCallTo(getRoot(), Type::VoidTy, false, false, CallingConv::C, true,
+ DAG.getExternalSymbol("free", IntPtr), Args, DAG);
+ DAG.setRoot(Result.second);
+}
+
+// InsertAtEndOfBasicBlock - This method should be implemented by targets that
+// mark instructions with the 'usesCustomDAGSchedInserter' flag. These
+// instructions are special in various ways, which require special support to
+// insert. The specified MachineInstr is created but not inserted into any
+// basic blocks, and the scheduler passes ownership of it to this method.
+MachineBasicBlock *TargetLowering::InsertAtEndOfBasicBlock(MachineInstr *MI,
+ MachineBasicBlock *MBB) {
+ cerr << "If a target marks an instruction with "
+ << "'usesCustomDAGSchedInserter', it must implement "
+ << "TargetLowering::InsertAtEndOfBasicBlock!\n";
+ abort();
+ return 0;
+}
+
+void SelectionDAGLowering::visitVAStart(CallInst &I) {
+ DAG.setRoot(DAG.getNode(ISD::VASTART, MVT::Other, getRoot(),
+ getValue(I.getOperand(1)),
+ DAG.getSrcValue(I.getOperand(1))));
+}
+
+void SelectionDAGLowering::visitVAArg(VAArgInst &I) {
+ SDOperand V = DAG.getVAArg(TLI.getValueType(I.getType()), getRoot(),
+ getValue(I.getOperand(0)),
+ DAG.getSrcValue(I.getOperand(0)));
+ setValue(&I, V);
+ DAG.setRoot(V.getValue(1));
+}
+
+void SelectionDAGLowering::visitVAEnd(CallInst &I) {
+ DAG.setRoot(DAG.getNode(ISD::VAEND, MVT::Other, getRoot(),
+ getValue(I.getOperand(1)),
+ DAG.getSrcValue(I.getOperand(1))));
+}
+
+void SelectionDAGLowering::visitVACopy(CallInst &I) {
+ DAG.setRoot(DAG.getNode(ISD::VACOPY, MVT::Other, getRoot(),
+ getValue(I.getOperand(1)),
+ getValue(I.getOperand(2)),
+ DAG.getSrcValue(I.getOperand(1)),
+ DAG.getSrcValue(I.getOperand(2))));
+}
+
+/// TargetLowering::LowerArguments - This is the default LowerArguments
+/// implementation, which just inserts a FORMAL_ARGUMENTS node. FIXME: When all
+/// targets are migrated to using FORMAL_ARGUMENTS, this hook should be
+/// integrated into SDISel.
+std::vector<SDOperand>
+TargetLowering::LowerArguments(Function &F, SelectionDAG &DAG) {
+ const FunctionType *FTy = F.getFunctionType();
+ const ParamAttrsList *Attrs = FTy->getParamAttrs();
+ // Add CC# and isVararg as operands to the FORMAL_ARGUMENTS node.
+ std::vector<SDOperand> Ops;
+ Ops.push_back(DAG.getRoot());
+ Ops.push_back(DAG.getConstant(F.getCallingConv(), getPointerTy()));
+ Ops.push_back(DAG.getConstant(F.isVarArg(), getPointerTy()));
+
+ // Add one result value for each formal argument.
+ std::vector<MVT::ValueType> RetVals;
+ unsigned j = 1;
+ for (Function::arg_iterator I = F.arg_begin(), E = F.arg_end();
+ I != E; ++I, ++j) {
+ MVT::ValueType VT = getValueType(I->getType());
+ unsigned Flags = ISD::ParamFlags::NoFlagSet;
+ unsigned OriginalAlignment =
+ getTargetData()->getABITypeAlignment(I->getType());
+
+ // FIXME: Distinguish between a formal with no [sz]ext attribute from one
+ // that is zero extended!
+ if (Attrs && Attrs->paramHasAttr(j, ParamAttr::ZExt))
+ Flags &= ~(ISD::ParamFlags::SExt);
+ if (Attrs && Attrs->paramHasAttr(j, ParamAttr::SExt))
+ Flags |= ISD::ParamFlags::SExt;
+ if (Attrs && Attrs->paramHasAttr(j, ParamAttr::InReg))
+ Flags |= ISD::ParamFlags::InReg;
+ if (Attrs && Attrs->paramHasAttr(j, ParamAttr::StructRet))
+ Flags |= ISD::ParamFlags::StructReturn;
+ if (Attrs && Attrs->paramHasAttr(j, ParamAttr::ByVal))
+ Flags |= ISD::ParamFlags::ByVal;
+ Flags |= (OriginalAlignment << ISD::ParamFlags::OrigAlignmentOffs);
+
+ switch (getTypeAction(VT)) {
+ default: assert(0 && "Unknown type action!");
+ case Legal:
+ RetVals.push_back(VT);
+ Ops.push_back(DAG.getConstant(Flags, MVT::i32));
+ break;
+ case Promote:
+ RetVals.push_back(getTypeToTransformTo(VT));
+ Ops.push_back(DAG.getConstant(Flags, MVT::i32));
+ break;
+ case Expand: {
+ // If this is an illegal type, it needs to be broken up to fit into
+ // registers.
+ MVT::ValueType RegisterVT = getRegisterType(VT);
+ unsigned NumRegs = getNumRegisters(VT);
+ for (unsigned i = 0; i != NumRegs; ++i) {
+ RetVals.push_back(RegisterVT);
+ // if it isn't first piece, alignment must be 1
+ if (i > 0)
+ Flags = (Flags & (~ISD::ParamFlags::OrigAlignment)) |
+ (1 << ISD::ParamFlags::OrigAlignmentOffs);
+ Ops.push_back(DAG.getConstant(Flags, MVT::i32));
+ }
+ break;
+ }
+ }
+ }
+
+ RetVals.push_back(MVT::Other);
+
+ // Create the node.
+ SDNode *Result = DAG.getNode(ISD::FORMAL_ARGUMENTS,
+ DAG.getNodeValueTypes(RetVals), RetVals.size(),
+ &Ops[0], Ops.size()).Val;
+ unsigned NumArgRegs = Result->getNumValues() - 1;
+ DAG.setRoot(SDOperand(Result, NumArgRegs));
+
+ // Set up the return result vector.
+ Ops.clear();
+ unsigned i = 0;
+ unsigned Idx = 1;
+ for (Function::arg_iterator I = F.arg_begin(), E = F.arg_end(); I != E;
+ ++I, ++Idx) {
+ MVT::ValueType VT = getValueType(I->getType());
+
+ switch (getTypeAction(VT)) {
+ default: assert(0 && "Unknown type action!");
+ case Legal:
+ Ops.push_back(SDOperand(Result, i++));
+ break;
+ case Promote: {
+ SDOperand Op(Result, i++);
+ if (MVT::isInteger(VT)) {
+ if (Attrs && Attrs->paramHasAttr(Idx, ParamAttr::SExt))
+ Op = DAG.getNode(ISD::AssertSext, Op.getValueType(), Op,
+ DAG.getValueType(VT));
+ else if (Attrs && Attrs->paramHasAttr(Idx, ParamAttr::ZExt))
+ Op = DAG.getNode(ISD::AssertZext, Op.getValueType(), Op,
+ DAG.getValueType(VT));
+ Op = DAG.getNode(ISD::TRUNCATE, VT, Op);
+ } else {
+ assert(MVT::isFloatingPoint(VT) && "Not int or FP?");
+ Op = DAG.getNode(ISD::FP_ROUND, VT, Op);
+ }
+ Ops.push_back(Op);
+ break;
+ }
+ case Expand: {
+ MVT::ValueType PartVT = getRegisterType(VT);
+ unsigned NumParts = getNumRegisters(VT);
+ SmallVector<SDOperand, 4> Parts(NumParts);
+ for (unsigned j = 0; j != NumParts; ++j)
+ Parts[j] = SDOperand(Result, i++);
+ Ops.push_back(getCopyFromParts(DAG, &Parts[0], NumParts, PartVT, VT));
+ break;
+ }
+ }
+ }
+ assert(i == NumArgRegs && "Argument register count mismatch!");
+ return Ops;
+}
+
+
+/// TargetLowering::LowerCallTo - This is the default LowerCallTo
+/// implementation, which just inserts an ISD::CALL node, which is later custom
+/// lowered by the target to something concrete. FIXME: When all targets are
+/// migrated to using ISD::CALL, this hook should be integrated into SDISel.
+std::pair<SDOperand, SDOperand>
+TargetLowering::LowerCallTo(SDOperand Chain, const Type *RetTy,
+ bool RetTyIsSigned, bool isVarArg,
+ unsigned CallingConv, bool isTailCall,
+ SDOperand Callee,
+ ArgListTy &Args, SelectionDAG &DAG) {
+ SmallVector<SDOperand, 32> Ops;
+ Ops.push_back(Chain); // Op#0 - Chain
+ Ops.push_back(DAG.getConstant(CallingConv, getPointerTy())); // Op#1 - CC
+ Ops.push_back(DAG.getConstant(isVarArg, getPointerTy())); // Op#2 - VarArg
+ Ops.push_back(DAG.getConstant(isTailCall, getPointerTy())); // Op#3 - Tail
+ Ops.push_back(Callee);
+
+ // Handle all of the outgoing arguments.
+ for (unsigned i = 0, e = Args.size(); i != e; ++i) {
+ MVT::ValueType VT = getValueType(Args[i].Ty);
+ SDOperand Op = Args[i].Node;
+ unsigned Flags = ISD::ParamFlags::NoFlagSet;
+ unsigned OriginalAlignment =
+ getTargetData()->getABITypeAlignment(Args[i].Ty);
+
+ if (Args[i].isSExt)
+ Flags |= ISD::ParamFlags::SExt;
+ if (Args[i].isZExt)
+ Flags |= ISD::ParamFlags::ZExt;
+ if (Args[i].isInReg)
+ Flags |= ISD::ParamFlags::InReg;
+ if (Args[i].isSRet)
+ Flags |= ISD::ParamFlags::StructReturn;
+ Flags |= OriginalAlignment << ISD::ParamFlags::OrigAlignmentOffs;
+
+ switch (getTypeAction(VT)) {
+ default: assert(0 && "Unknown type action!");
+ case Legal:
+ Ops.push_back(Op);
+ Ops.push_back(DAG.getConstant(Flags, MVT::i32));
+ break;
+ case Promote:
+ if (MVT::isInteger(VT)) {
+ unsigned ExtOp;
+ if (Args[i].isSExt)
+ ExtOp = ISD::SIGN_EXTEND;
+ else if (Args[i].isZExt)
+ ExtOp = ISD::ZERO_EXTEND;
+ else
+ ExtOp = ISD::ANY_EXTEND;
+ Op = DAG.getNode(ExtOp, getTypeToTransformTo(VT), Op);
+ } else {
+ assert(MVT::isFloatingPoint(VT) && "Not int or FP?");
+ Op = DAG.getNode(ISD::FP_EXTEND, getTypeToTransformTo(VT), Op);
+ }
+ Ops.push_back(Op);
+ Ops.push_back(DAG.getConstant(Flags, MVT::i32));
+ break;
+ case Expand: {
+ MVT::ValueType PartVT = getRegisterType(VT);
+ unsigned NumParts = getNumRegisters(VT);
+ SmallVector<SDOperand, 4> Parts(NumParts);
+ getCopyToParts(DAG, Op, &Parts[0], NumParts, PartVT);
+ for (unsigned i = 0; i != NumParts; ++i) {
+ // if it isn't first piece, alignment must be 1
+ unsigned MyFlags = Flags;
+ if (i != 0)
+ MyFlags = (MyFlags & (~ISD::ParamFlags::OrigAlignment)) |
+ (1 << ISD::ParamFlags::OrigAlignmentOffs);
+
+ Ops.push_back(Parts[i]);
+ Ops.push_back(DAG.getConstant(MyFlags, MVT::i32));
+ }
+ break;
+ }
+ }
+ }
+
+ // Figure out the result value types.
+ MVT::ValueType VT = getValueType(RetTy);
+ MVT::ValueType RegisterVT = getRegisterType(VT);
+ unsigned NumRegs = getNumRegisters(VT);
+ SmallVector<MVT::ValueType, 4> RetTys(NumRegs);
+ for (unsigned i = 0; i != NumRegs; ++i)
+ RetTys[i] = RegisterVT;
+
+ RetTys.push_back(MVT::Other); // Always has a chain.
+
+ // Create the CALL node.
+ SDOperand Res = DAG.getNode(ISD::CALL,
+ DAG.getVTList(&RetTys[0], NumRegs + 1),
+ &Ops[0], Ops.size());
+ SDOperand Chain = Res.getValue(NumRegs);
+
+ // Gather up the call result into a single value.
+ if (RetTy != Type::VoidTy) {
+ ISD::NodeType AssertOp = ISD::AssertSext;
+ if (!RetTyIsSigned)
+ AssertOp = ISD::AssertZext;
+ SmallVector<SDOperand, 4> Results(NumRegs);
+ for (unsigned i = 0; i != NumRegs; ++i)
+ Results[i] = Res.getValue(i);
+ Res = getCopyFromParts(DAG, &Results[0], NumRegs, RegisterVT, VT, AssertOp);
+ }
+
+ return std::make_pair(Res, Chain);
+}
+
+SDOperand TargetLowering::LowerOperation(SDOperand Op, SelectionDAG &DAG) {
+ assert(0 && "LowerOperation not implemented for this target!");
+ abort();
+ return SDOperand();
+}
+
+SDOperand TargetLowering::CustomPromoteOperation(SDOperand Op,
+ SelectionDAG &DAG) {
+ assert(0 && "CustomPromoteOperation not implemented for this target!");
+ abort();
+ return SDOperand();
+}
+
+/// getMemsetValue - Vectorized representation of the memset value
+/// operand.
+static SDOperand getMemsetValue(SDOperand Value, MVT::ValueType VT,
+ SelectionDAG &DAG) {
+ MVT::ValueType CurVT = VT;
+ if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Value)) {
+ uint64_t Val = C->getValue() & 255;
+ unsigned Shift = 8;
+ while (CurVT != MVT::i8) {
+ Val = (Val << Shift) | Val;
+ Shift <<= 1;
+ CurVT = (MVT::ValueType)((unsigned)CurVT - 1);
+ }
+ return DAG.getConstant(Val, VT);
+ } else {
+ Value = DAG.getNode(ISD::ZERO_EXTEND, VT, Value);
+ unsigned Shift = 8;
+ while (CurVT != MVT::i8) {
+ Value =
+ DAG.getNode(ISD::OR, VT,
+ DAG.getNode(ISD::SHL, VT, Value,
+ DAG.getConstant(Shift, MVT::i8)), Value);
+ Shift <<= 1;
+ CurVT = (MVT::ValueType)((unsigned)CurVT - 1);
+ }
+
+ return Value;
+ }
+}
+
+/// getMemsetStringVal - Similar to getMemsetValue. Except this is only
+/// used when a memcpy is turned into a memset when the source is a constant
+/// string ptr.
+static SDOperand getMemsetStringVal(MVT::ValueType VT,
+ SelectionDAG &DAG, TargetLowering &TLI,
+ std::string &Str, unsigned Offset) {
+ uint64_t Val = 0;
+ unsigned MSB = MVT::getSizeInBits(VT) / 8;
+ if (TLI.isLittleEndian())
+ Offset = Offset + MSB - 1;
+ for (unsigned i = 0; i != MSB; ++i) {
+ Val = (Val << 8) | (unsigned char)Str[Offset];
+ Offset += TLI.isLittleEndian() ? -1 : 1;
+ }
+ return DAG.getConstant(Val, VT);
+}
+
+/// getMemBasePlusOffset - Returns base and offset node for the
+static SDOperand getMemBasePlusOffset(SDOperand Base, unsigned Offset,
+ SelectionDAG &DAG, TargetLowering &TLI) {
+ MVT::ValueType VT = Base.getValueType();
+ return DAG.getNode(ISD::ADD, VT, Base, DAG.getConstant(Offset, VT));
+}
+
+/// MeetsMaxMemopRequirement - Determines if the number of memory ops required
+/// to replace the memset / memcpy is below the threshold. It also returns the
+/// types of the sequence of memory ops to perform memset / memcpy.
+static bool MeetsMaxMemopRequirement(std::vector<MVT::ValueType> &MemOps,
+ unsigned Limit, uint64_t Size,
+ unsigned Align, TargetLowering &TLI) {
+ MVT::ValueType VT;
+
+ if (TLI.allowsUnalignedMemoryAccesses()) {
+ VT = MVT::i64;
+ } else {
+ switch (Align & 7) {
+ case 0:
+ VT = MVT::i64;
+ break;
+ case 4:
+ VT = MVT::i32;
+ break;
+ case 2:
+ VT = MVT::i16;
+ break;
+ default:
+ VT = MVT::i8;
+ break;
+ }
+ }
+
+ MVT::ValueType LVT = MVT::i64;
+ while (!TLI.isTypeLegal(LVT))
+ LVT = (MVT::ValueType)((unsigned)LVT - 1);
+ assert(MVT::isInteger(LVT));
+
+ if (VT > LVT)
+ VT = LVT;
+
+ unsigned NumMemOps = 0;
+ while (Size != 0) {
+ unsigned VTSize = MVT::getSizeInBits(VT) / 8;
+ while (VTSize > Size) {
+ VT = (MVT::ValueType)((unsigned)VT - 1);
+ VTSize >>= 1;
+ }
+ assert(MVT::isInteger(VT));
+
+ if (++NumMemOps > Limit)
+ return false;
+ MemOps.push_back(VT);
+ Size -= VTSize;
+ }
+
+ return true;
+}
+
+void SelectionDAGLowering::visitMemIntrinsic(CallInst &I, unsigned Op) {
+ SDOperand Op1 = getValue(I.getOperand(1));
+ SDOperand Op2 = getValue(I.getOperand(2));
+ SDOperand Op3 = getValue(I.getOperand(3));
+ SDOperand Op4 = getValue(I.getOperand(4));
+ unsigned Align = (unsigned)cast<ConstantSDNode>(Op4)->getValue();
+ if (Align == 0) Align = 1;
+
+ if (ConstantSDNode *Size = dyn_cast<ConstantSDNode>(Op3)) {
+ std::vector<MVT::ValueType> MemOps;
+
+ // Expand memset / memcpy to a series of load / store ops
+ // if the size operand falls below a certain threshold.
+ SmallVector<SDOperand, 8> OutChains;
+ switch (Op) {
+ default: break; // Do nothing for now.
+ case ISD::MEMSET: {
+ if (MeetsMaxMemopRequirement(MemOps, TLI.getMaxStoresPerMemset(),
+ Size->getValue(), Align, TLI)) {
+ unsigned NumMemOps = MemOps.size();
+ unsigned Offset = 0;
+ for (unsigned i = 0; i < NumMemOps; i++) {
+ MVT::ValueType VT = MemOps[i];
+ unsigned VTSize = MVT::getSizeInBits(VT) / 8;
+ SDOperand Value = getMemsetValue(Op2, VT, DAG);
+ SDOperand Store = DAG.getStore(getRoot(), Value,
+ getMemBasePlusOffset(Op1, Offset, DAG, TLI),
+ I.getOperand(1), Offset);
+ OutChains.push_back(Store);
+ Offset += VTSize;
+ }
+ }
+ break;
+ }
+ case ISD::MEMCPY: {
+ if (MeetsMaxMemopRequirement(MemOps, TLI.getMaxStoresPerMemcpy(),
+ Size->getValue(), Align, TLI)) {
+ unsigned NumMemOps = MemOps.size();
+ unsigned SrcOff = 0, DstOff = 0, SrcDelta = 0;
+ GlobalAddressSDNode *G = NULL;
+ std::string Str;
+ bool CopyFromStr = false;
+
+ if (Op2.getOpcode() == ISD::GlobalAddress)
+ G = cast<GlobalAddressSDNode>(Op2);
+ else if (Op2.getOpcode() == ISD::ADD &&
+ Op2.getOperand(0).getOpcode() == ISD::GlobalAddress &&
+ Op2.getOperand(1).getOpcode() == ISD::Constant) {
+ G = cast<GlobalAddressSDNode>(Op2.getOperand(0));
+ SrcDelta = cast<ConstantSDNode>(Op2.getOperand(1))->getValue();
+ }
+ if (G) {
+ GlobalVariable *GV = dyn_cast<GlobalVariable>(G->getGlobal());
+ if (GV && GV->isConstant()) {
+ Str = GV->getStringValue(false);
+ if (!Str.empty()) {
+ CopyFromStr = true;
+ SrcOff += SrcDelta;
+ }
+ }
+ }
+
+ for (unsigned i = 0; i < NumMemOps; i++) {
+ MVT::ValueType VT = MemOps[i];
+ unsigned VTSize = MVT::getSizeInBits(VT) / 8;
+ SDOperand Value, Chain, Store;
+
+ if (CopyFromStr) {
+ Value = getMemsetStringVal(VT, DAG, TLI, Str, SrcOff);
+ Chain = getRoot();
+ Store =
+ DAG.getStore(Chain, Value,
+ getMemBasePlusOffset(Op1, DstOff, DAG, TLI),
+ I.getOperand(1), DstOff);
+ } else {
+ Value = DAG.getLoad(VT, getRoot(),
+ getMemBasePlusOffset(Op2, SrcOff, DAG, TLI),
+ I.getOperand(2), SrcOff);
+ Chain = Value.getValue(1);
+ Store =
+ DAG.getStore(Chain, Value,
+ getMemBasePlusOffset(Op1, DstOff, DAG, TLI),
+ I.getOperand(1), DstOff);
+ }
+ OutChains.push_back(Store);
+ SrcOff += VTSize;
+ DstOff += VTSize;
+ }
+ }
+ break;
+ }
+ }
+
+ if (!OutChains.empty()) {
+ DAG.setRoot(DAG.getNode(ISD::TokenFactor, MVT::Other,
+ &OutChains[0], OutChains.size()));
+ return;
+ }
+ }
+
+ DAG.setRoot(DAG.getNode(Op, MVT::Other, getRoot(), Op1, Op2, Op3, Op4));
+}
+
+//===----------------------------------------------------------------------===//
+// SelectionDAGISel code
+//===----------------------------------------------------------------------===//
+
+unsigned SelectionDAGISel::MakeReg(MVT::ValueType VT) {
+ return RegMap->createVirtualRegister(TLI.getRegClassFor(VT));
+}
+
+void SelectionDAGISel::getAnalysisUsage(AnalysisUsage &AU) const {
+ AU.addRequired<AliasAnalysis>();
+ AU.setPreservesAll();
+}
+
+
+
+bool SelectionDAGISel::runOnFunction(Function &Fn) {
+ MachineFunction &MF = MachineFunction::construct(&Fn, TLI.getTargetMachine());
+ RegMap = MF.getSSARegMap();
+ DOUT << "\n\n\n=== " << Fn.getName() << "\n";
+
+ FunctionLoweringInfo FuncInfo(TLI, Fn, MF);
+
+ if (ExceptionHandling)
+ for (Function::iterator I = Fn.begin(), E = Fn.end(); I != E; ++I)
+ if (InvokeInst *Invoke = dyn_cast<InvokeInst>(I->getTerminator()))
+ // Mark landing pad.
+ FuncInfo.MBBMap[Invoke->getSuccessor(1)]->setIsLandingPad();
+
+ for (Function::iterator I = Fn.begin(), E = Fn.end(); I != E; ++I)
+ SelectBasicBlock(I, MF, FuncInfo);
+
+ // Add function live-ins to entry block live-in set.
+ BasicBlock *EntryBB = &Fn.getEntryBlock();
+ BB = FuncInfo.MBBMap[EntryBB];
+ if (!MF.livein_empty())
+ for (MachineFunction::livein_iterator I = MF.livein_begin(),
+ E = MF.livein_end(); I != E; ++I)
+ BB->addLiveIn(I->first);
+
+#ifndef NDEBUG
+ assert(FuncInfo.CatchInfoFound.size() == FuncInfo.CatchInfoLost.size() &&
+ "Not all catch info was assigned to a landing pad!");
+#endif
+
+ return true;
+}
+
+SDOperand SelectionDAGLowering::CopyValueToVirtualRegister(Value *V,
+ unsigned Reg) {
+ SDOperand Op = getValue(V);
+ assert((Op.getOpcode() != ISD::CopyFromReg ||
+ cast<RegisterSDNode>(Op.getOperand(1))->getReg() != Reg) &&
+ "Copy from a reg to the same reg!");
+
+ MVT::ValueType SrcVT = Op.getValueType();
+ MVT::ValueType RegisterVT = TLI.getRegisterType(SrcVT);
+ unsigned NumRegs = TLI.getNumRegisters(SrcVT);
+ SmallVector<SDOperand, 8> Regs(NumRegs);
+ SmallVector<SDOperand, 8> Chains(NumRegs);
+
+ // Copy the value by legal parts into sequential virtual registers.
+ getCopyToParts(DAG, Op, &Regs[0], NumRegs, RegisterVT);
+ for (unsigned i = 0; i != NumRegs; ++i)
+ Chains[i] = DAG.getCopyToReg(getRoot(), Reg + i, Regs[i]);
+ return DAG.getNode(ISD::TokenFactor, MVT::Other, &Chains[0], NumRegs);
+}
+
+void SelectionDAGISel::
+LowerArguments(BasicBlock *LLVMBB, SelectionDAGLowering &SDL,
+ std::vector<SDOperand> &UnorderedChains) {
+ // If this is the entry block, emit arguments.
+ Function &F = *LLVMBB->getParent();
+ FunctionLoweringInfo &FuncInfo = SDL.FuncInfo;
+ SDOperand OldRoot = SDL.DAG.getRoot();
+ std::vector<SDOperand> Args = TLI.LowerArguments(F, SDL.DAG);
+
+ unsigned a = 0;
+ for (Function::arg_iterator AI = F.arg_begin(), E = F.arg_end();
+ AI != E; ++AI, ++a)
+ if (!AI->use_empty()) {
+ SDL.setValue(AI, Args[a]);
+
+ // If this argument is live outside of the entry block, insert a copy from
+ // whereever we got it to the vreg that other BB's will reference it as.
+ DenseMap<const Value*, unsigned>::iterator VMI=FuncInfo.ValueMap.find(AI);
+ if (VMI != FuncInfo.ValueMap.end()) {
+ SDOperand Copy = SDL.CopyValueToVirtualRegister(AI, VMI->second);
+ UnorderedChains.push_back(Copy);
+ }
+ }
+
+ // Finally, if the target has anything special to do, allow it to do so.
+ // FIXME: this should insert code into the DAG!
+ EmitFunctionEntryCode(F, SDL.DAG.getMachineFunction());
+}
+
+static void copyCatchInfo(BasicBlock *SrcBB, BasicBlock *DestBB,
+ MachineModuleInfo *MMI, FunctionLoweringInfo &FLI) {
+ assert(!FLI.MBBMap[SrcBB]->isLandingPad() &&
+ "Copying catch info out of a landing pad!");
+ for (BasicBlock::iterator I = SrcBB->begin(), E = --SrcBB->end(); I != E; ++I)
+ if (isSelector(I)) {
+ // Apply the catch info to DestBB.
+ addCatchInfo(cast<CallInst>(*I), MMI, FLI.MBBMap[DestBB]);
+#ifndef NDEBUG
+ FLI.CatchInfoFound.insert(I);
+#endif
+ }
+}
+
+void SelectionDAGISel::BuildSelectionDAG(SelectionDAG &DAG, BasicBlock *LLVMBB,
+ std::vector<std::pair<MachineInstr*, unsigned> > &PHINodesToUpdate,
+ FunctionLoweringInfo &FuncInfo) {
+ SelectionDAGLowering SDL(DAG, TLI, FuncInfo);
+
+ std::vector<SDOperand> UnorderedChains;
+
+ // Lower any arguments needed in this block if this is the entry block.
+ if (LLVMBB == &LLVMBB->getParent()->getEntryBlock())
+ LowerArguments(LLVMBB, SDL, UnorderedChains);
+
+ BB = FuncInfo.MBBMap[LLVMBB];
+ SDL.setCurrentBasicBlock(BB);
+
+ MachineModuleInfo *MMI = DAG.getMachineModuleInfo();
+
+ if (ExceptionHandling && MMI && BB->isLandingPad()) {
+ // Add a label to mark the beginning of the landing pad. Deletion of the
+ // landing pad can thus be detected via the MachineModuleInfo.
+ unsigned LabelID = MMI->addLandingPad(BB);
+ DAG.setRoot(DAG.getNode(ISD::LABEL, MVT::Other, DAG.getEntryNode(),
+ DAG.getConstant(LabelID, MVT::i32)));
+
+ // Mark exception register as live in.
+ unsigned Reg = TLI.getExceptionAddressRegister();
+ if (Reg) BB->addLiveIn(Reg);
+
+ // Mark exception selector register as live in.
+ Reg = TLI.getExceptionSelectorRegister();
+ if (Reg) BB->addLiveIn(Reg);
+
+ // FIXME: Hack around an exception handling flaw (PR1508): the personality
+ // function and list of typeids logically belong to the invoke (or, if you
+ // like, the basic block containing the invoke), and need to be associated
+ // with it in the dwarf exception handling tables. Currently however the
+ // information is provided by an intrinsic (eh.selector) that can be moved
+ // to unexpected places by the optimizers: if the unwind edge is critical,
+ // then breaking it can result in the intrinsics being in the successor of
+ // the landing pad, not the landing pad itself. This results in exceptions
+ // not being caught because no typeids are associated with the invoke.
+ // This may not be the only way things can go wrong, but it is the only way
+ // we try to work around for the moment.
+ BranchInst *Br = dyn_cast<BranchInst>(LLVMBB->getTerminator());
+
+ if (Br && Br->isUnconditional()) { // Critical edge?
+ BasicBlock::iterator I, E;
+ for (I = LLVMBB->begin(), E = --LLVMBB->end(); I != E; ++I)
+ if (isSelector(I))
+ break;
+
+ if (I == E)
+ // No catch info found - try to extract some from the successor.
+ copyCatchInfo(Br->getSuccessor(0), LLVMBB, MMI, FuncInfo);
+ }
+ }
+
+ // Lower all of the non-terminator instructions.
+ for (BasicBlock::iterator I = LLVMBB->begin(), E = --LLVMBB->end();
+ I != E; ++I)
+ SDL.visit(*I);
+
+ // Ensure that all instructions which are used outside of their defining
+ // blocks are available as virtual registers. Invoke is handled elsewhere.
+ for (BasicBlock::iterator I = LLVMBB->begin(), E = LLVMBB->end(); I != E;++I)
+ if (!I->use_empty() && !isa<PHINode>(I) && !isa<InvokeInst>(I)) {
+ DenseMap<const Value*, unsigned>::iterator VMI =FuncInfo.ValueMap.find(I);
+ if (VMI != FuncInfo.ValueMap.end())
+ UnorderedChains.push_back(
+ SDL.CopyValueToVirtualRegister(I, VMI->second));
+ }
+
+ // Handle PHI nodes in successor blocks. Emit code into the SelectionDAG to
+ // ensure constants are generated when needed. Remember the virtual registers
+ // that need to be added to the Machine PHI nodes as input. We cannot just
+ // directly add them, because expansion might result in multiple MBB's for one
+ // BB. As such, the start of the BB might correspond to a different MBB than
+ // the end.
+ //
+ TerminatorInst *TI = LLVMBB->getTerminator();
+
+ // Emit constants only once even if used by multiple PHI nodes.
+ std::map<Constant*, unsigned> ConstantsOut;
+
+ // Vector bool would be better, but vector<bool> is really slow.
+ std::vector<unsigned char> SuccsHandled;
+ if (TI->getNumSuccessors())
+ SuccsHandled.resize(BB->getParent()->getNumBlockIDs());
+
+ // Check successor nodes' PHI nodes that expect a constant to be available
+ // from this block.
+ for (unsigned succ = 0, e = TI->getNumSuccessors(); succ != e; ++succ) {
+ BasicBlock *SuccBB = TI->getSuccessor(succ);
+ if (!isa<PHINode>(SuccBB->begin())) continue;
+ MachineBasicBlock *SuccMBB = FuncInfo.MBBMap[SuccBB];
+
+ // If this terminator has multiple identical successors (common for
+ // switches), only handle each succ once.
+ unsigned SuccMBBNo = SuccMBB->getNumber();
+ if (SuccsHandled[SuccMBBNo]) continue;
+ SuccsHandled[SuccMBBNo] = true;
+
+ MachineBasicBlock::iterator MBBI = SuccMBB->begin();
+ PHINode *PN;
+
+ // At this point we know that there is a 1-1 correspondence between LLVM PHI
+ // nodes and Machine PHI nodes, but the incoming operands have not been
+ // emitted yet.
+ for (BasicBlock::iterator I = SuccBB->begin();
+ (PN = dyn_cast<PHINode>(I)); ++I) {
+ // Ignore dead phi's.
+ if (PN->use_empty()) continue;
+
+ unsigned Reg;
+ Value *PHIOp = PN->getIncomingValueForBlock(LLVMBB);
+
+ if (Constant *C = dyn_cast<Constant>(PHIOp)) {
+ unsigned &RegOut = ConstantsOut[C];
+ if (RegOut == 0) {
+ RegOut = FuncInfo.CreateRegForValue(C);
+ UnorderedChains.push_back(
+ SDL.CopyValueToVirtualRegister(C, RegOut));
+ }
+ Reg = RegOut;
+ } else {
+ Reg = FuncInfo.ValueMap[PHIOp];
+ if (Reg == 0) {
+ assert(isa<AllocaInst>(PHIOp) &&
+ FuncInfo.StaticAllocaMap.count(cast<AllocaInst>(PHIOp)) &&
+ "Didn't codegen value into a register!??");
+ Reg = FuncInfo.CreateRegForValue(PHIOp);
+ UnorderedChains.push_back(
+ SDL.CopyValueToVirtualRegister(PHIOp, Reg));
+ }
+ }
+
+ // Remember that this register needs to added to the machine PHI node as
+ // the input for this MBB.
+ MVT::ValueType VT = TLI.getValueType(PN->getType());
+ unsigned NumRegisters = TLI.getNumRegisters(VT);
+ for (unsigned i = 0, e = NumRegisters; i != e; ++i)
+ PHINodesToUpdate.push_back(std::make_pair(MBBI++, Reg+i));
+ }
+ }
+ ConstantsOut.clear();
+
+ // Turn all of the unordered chains into one factored node.
+ if (!UnorderedChains.empty()) {
+ SDOperand Root = SDL.getRoot();
+ if (Root.getOpcode() != ISD::EntryToken) {
+ unsigned i = 0, e = UnorderedChains.size();
+ for (; i != e; ++i) {
+ assert(UnorderedChains[i].Val->getNumOperands() > 1);
+ if (UnorderedChains[i].Val->getOperand(0) == Root)
+ break; // Don't add the root if we already indirectly depend on it.
+ }
+
+ if (i == e)
+ UnorderedChains.push_back(Root);
+ }
+ DAG.setRoot(DAG.getNode(ISD::TokenFactor, MVT::Other,
+ &UnorderedChains[0], UnorderedChains.size()));
+ }
+
+ // Lower the terminator after the copies are emitted.
+ SDL.visit(*LLVMBB->getTerminator());
+
+ // Copy over any CaseBlock records that may now exist due to SwitchInst
+ // lowering, as well as any jump table information.
+ SwitchCases.clear();
+ SwitchCases = SDL.SwitchCases;
+ JTCases.clear();
+ JTCases = SDL.JTCases;
+ BitTestCases.clear();
+ BitTestCases = SDL.BitTestCases;
+
+ // Make sure the root of the DAG is up-to-date.
+ DAG.setRoot(SDL.getRoot());
+}
+
+void SelectionDAGISel::CodeGenAndEmitDAG(SelectionDAG &DAG) {
+ // Get alias analysis for load/store combining.
+ AliasAnalysis &AA = getAnalysis<AliasAnalysis>();
+
+ // Run the DAG combiner in pre-legalize mode.
+ DAG.Combine(false, AA);
+
+ DOUT << "Lowered selection DAG:\n";
+ DEBUG(DAG.dump());
+
+ // Second step, hack on the DAG until it only uses operations and types that
+ // the target supports.
+ DAG.Legalize();
+
+ DOUT << "Legalized selection DAG:\n";
+ DEBUG(DAG.dump());
+
+ // Run the DAG combiner in post-legalize mode.
+ DAG.Combine(true, AA);
+
+ if (ViewISelDAGs) DAG.viewGraph();
+
+ // Third, instruction select all of the operations to machine code, adding the
+ // code to the MachineBasicBlock.
+ InstructionSelectBasicBlock(DAG);
+
+ DOUT << "Selected machine code:\n";
+ DEBUG(BB->dump());
+}
+
+void SelectionDAGISel::SelectBasicBlock(BasicBlock *LLVMBB, MachineFunction &MF,
+ FunctionLoweringInfo &FuncInfo) {
+ std::vector<std::pair<MachineInstr*, unsigned> > PHINodesToUpdate;
+ {
+ SelectionDAG DAG(TLI, MF, getAnalysisToUpdate<MachineModuleInfo>());
+ CurDAG = &DAG;
+
+ // First step, lower LLVM code to some DAG. This DAG may use operations and
+ // types that are not supported by the target.
+ BuildSelectionDAG(DAG, LLVMBB, PHINodesToUpdate, FuncInfo);
+
+ // Second step, emit the lowered DAG as machine code.
+ CodeGenAndEmitDAG(DAG);
+ }
+
+ DOUT << "Total amount of phi nodes to update: "
+ << PHINodesToUpdate.size() << "\n";
+ DEBUG(for (unsigned i = 0, e = PHINodesToUpdate.size(); i != e; ++i)
+ DOUT << "Node " << i << " : (" << PHINodesToUpdate[i].first
+ << ", " << PHINodesToUpdate[i].second << ")\n";);
+
+ // Next, now that we know what the last MBB the LLVM BB expanded is, update
+ // PHI nodes in successors.
+ if (SwitchCases.empty() && JTCases.empty() && BitTestCases.empty()) {
+ for (unsigned i = 0, e = PHINodesToUpdate.size(); i != e; ++i) {
+ MachineInstr *PHI = PHINodesToUpdate[i].first;
+ assert(PHI->getOpcode() == TargetInstrInfo::PHI &&
+ "This is not a machine PHI node that we are updating!");
+ PHI->addRegOperand(PHINodesToUpdate[i].second, false);
+ PHI->addMachineBasicBlockOperand(BB);
+ }
+ return;
+ }
+
+ for (unsigned i = 0, e = BitTestCases.size(); i != e; ++i) {
+ // Lower header first, if it wasn't already lowered
+ if (!BitTestCases[i].Emitted) {
+ SelectionDAG HSDAG(TLI, MF, getAnalysisToUpdate<MachineModuleInfo>());
+ CurDAG = &HSDAG;
+ SelectionDAGLowering HSDL(HSDAG, TLI, FuncInfo);
+ // Set the current basic block to the mbb we wish to insert the code into
+ BB = BitTestCases[i].Parent;
+ HSDL.setCurrentBasicBlock(BB);
+ // Emit the code
+ HSDL.visitBitTestHeader(BitTestCases[i]);
+ HSDAG.setRoot(HSDL.getRoot());
+ CodeGenAndEmitDAG(HSDAG);
+ }
+
+ for (unsigned j = 0, ej = BitTestCases[i].Cases.size(); j != ej; ++j) {
+ SelectionDAG BSDAG(TLI, MF, getAnalysisToUpdate<MachineModuleInfo>());
+ CurDAG = &BSDAG;
+ SelectionDAGLowering BSDL(BSDAG, TLI, FuncInfo);
+ // Set the current basic block to the mbb we wish to insert the code into
+ BB = BitTestCases[i].Cases[j].ThisBB;
+ BSDL.setCurrentBasicBlock(BB);
+ // Emit the code
+ if (j+1 != ej)
+ BSDL.visitBitTestCase(BitTestCases[i].Cases[j+1].ThisBB,
+ BitTestCases[i].Reg,
+ BitTestCases[i].Cases[j]);
+ else
+ BSDL.visitBitTestCase(BitTestCases[i].Default,
+ BitTestCases[i].Reg,
+ BitTestCases[i].Cases[j]);
+
+
+ BSDAG.setRoot(BSDL.getRoot());
+ CodeGenAndEmitDAG(BSDAG);
+ }
+
+ // Update PHI Nodes
+ for (unsigned pi = 0, pe = PHINodesToUpdate.size(); pi != pe; ++pi) {
+ MachineInstr *PHI = PHINodesToUpdate[pi].first;
+ MachineBasicBlock *PHIBB = PHI->getParent();
+ assert(PHI->getOpcode() == TargetInstrInfo::PHI &&
+ "This is not a machine PHI node that we are updating!");
+ // This is "default" BB. We have two jumps to it. From "header" BB and
+ // from last "case" BB.
+ if (PHIBB == BitTestCases[i].Default) {
+ PHI->addRegOperand(PHINodesToUpdate[pi].second, false);
+ PHI->addMachineBasicBlockOperand(BitTestCases[i].Parent);
+ PHI->addRegOperand(PHINodesToUpdate[pi].second, false);
+ PHI->addMachineBasicBlockOperand(BitTestCases[i].Cases.back().ThisBB);
+ }
+ // One of "cases" BB.
+ for (unsigned j = 0, ej = BitTestCases[i].Cases.size(); j != ej; ++j) {
+ MachineBasicBlock* cBB = BitTestCases[i].Cases[j].ThisBB;
+ if (cBB->succ_end() !=
+ std::find(cBB->succ_begin(),cBB->succ_end(), PHIBB)) {
+ PHI->addRegOperand(PHINodesToUpdate[pi].second, false);
+ PHI->addMachineBasicBlockOperand(cBB);
+ }
+ }
+ }
+ }
+
+ // If the JumpTable record is filled in, then we need to emit a jump table.
+ // Updating the PHI nodes is tricky in this case, since we need to determine
+ // whether the PHI is a successor of the range check MBB or the jump table MBB
+ for (unsigned i = 0, e = JTCases.size(); i != e; ++i) {
+ // Lower header first, if it wasn't already lowered
+ if (!JTCases[i].first.Emitted) {
+ SelectionDAG HSDAG(TLI, MF, getAnalysisToUpdate<MachineModuleInfo>());
+ CurDAG = &HSDAG;
+ SelectionDAGLowering HSDL(HSDAG, TLI, FuncInfo);
+ // Set the current basic block to the mbb we wish to insert the code into
+ BB = JTCases[i].first.HeaderBB;
+ HSDL.setCurrentBasicBlock(BB);
+ // Emit the code
+ HSDL.visitJumpTableHeader(JTCases[i].second, JTCases[i].first);
+ HSDAG.setRoot(HSDL.getRoot());
+ CodeGenAndEmitDAG(HSDAG);
+ }
+
+ SelectionDAG JSDAG(TLI, MF, getAnalysisToUpdate<MachineModuleInfo>());
+ CurDAG = &JSDAG;
+ SelectionDAGLowering JSDL(JSDAG, TLI, FuncInfo);
+ // Set the current basic block to the mbb we wish to insert the code into
+ BB = JTCases[i].second.MBB;
+ JSDL.setCurrentBasicBlock(BB);
+ // Emit the code
+ JSDL.visitJumpTable(JTCases[i].second);
+ JSDAG.setRoot(JSDL.getRoot());
+ CodeGenAndEmitDAG(JSDAG);
+
+ // Update PHI Nodes
+ for (unsigned pi = 0, pe = PHINodesToUpdate.size(); pi != pe; ++pi) {
+ MachineInstr *PHI = PHINodesToUpdate[pi].first;
+ MachineBasicBlock *PHIBB = PHI->getParent();
+ assert(PHI->getOpcode() == TargetInstrInfo::PHI &&
+ "This is not a machine PHI node that we are updating!");
+ // "default" BB. We can go there only from header BB.
+ if (PHIBB == JTCases[i].second.Default) {
+ PHI->addRegOperand(PHINodesToUpdate[pi].second, false);
+ PHI->addMachineBasicBlockOperand(JTCases[i].first.HeaderBB);
+ }
+ // JT BB. Just iterate over successors here
+ if (BB->succ_end() != std::find(BB->succ_begin(),BB->succ_end(), PHIBB)) {
+ PHI->addRegOperand(PHINodesToUpdate[pi].second, false);
+ PHI->addMachineBasicBlockOperand(BB);
+ }
+ }
+ }
+
+ // If the switch block involved a branch to one of the actual successors, we
+ // need to update PHI nodes in that block.
+ for (unsigned i = 0, e = PHINodesToUpdate.size(); i != e; ++i) {
+ MachineInstr *PHI = PHINodesToUpdate[i].first;
+ assert(PHI->getOpcode() == TargetInstrInfo::PHI &&
+ "This is not a machine PHI node that we are updating!");
+ if (BB->isSuccessor(PHI->getParent())) {
+ PHI->addRegOperand(PHINodesToUpdate[i].second, false);
+ PHI->addMachineBasicBlockOperand(BB);
+ }
+ }
+
+ // If we generated any switch lowering information, build and codegen any
+ // additional DAGs necessary.
+ for (unsigned i = 0, e = SwitchCases.size(); i != e; ++i) {
+ SelectionDAG SDAG(TLI, MF, getAnalysisToUpdate<MachineModuleInfo>());
+ CurDAG = &SDAG;
+ SelectionDAGLowering SDL(SDAG, TLI, FuncInfo);
+
+ // Set the current basic block to the mbb we wish to insert the code into
+ BB = SwitchCases[i].ThisBB;
+ SDL.setCurrentBasicBlock(BB);
+
+ // Emit the code
+ SDL.visitSwitchCase(SwitchCases[i]);
+ SDAG.setRoot(SDL.getRoot());
+ CodeGenAndEmitDAG(SDAG);
+
+ // Handle any PHI nodes in successors of this chunk, as if we were coming
+ // from the original BB before switch expansion. Note that PHI nodes can
+ // occur multiple times in PHINodesToUpdate. We have to be very careful to
+ // handle them the right number of times.
+ while ((BB = SwitchCases[i].TrueBB)) { // Handle LHS and RHS.
+ for (MachineBasicBlock::iterator Phi = BB->begin();
+ Phi != BB->end() && Phi->getOpcode() == TargetInstrInfo::PHI; ++Phi){
+ // This value for this PHI node is recorded in PHINodesToUpdate, get it.
+ for (unsigned pn = 0; ; ++pn) {
+ assert(pn != PHINodesToUpdate.size() && "Didn't find PHI entry!");
+ if (PHINodesToUpdate[pn].first == Phi) {
+ Phi->addRegOperand(PHINodesToUpdate[pn].second, false);
+ Phi->addMachineBasicBlockOperand(SwitchCases[i].ThisBB);
+ break;
+ }
+ }
+ }
+
+ // Don't process RHS if same block as LHS.
+ if (BB == SwitchCases[i].FalseBB)
+ SwitchCases[i].FalseBB = 0;
+
+ // If we haven't handled the RHS, do so now. Otherwise, we're done.
+ SwitchCases[i].TrueBB = SwitchCases[i].FalseBB;
+ SwitchCases[i].FalseBB = 0;
+ }
+ assert(SwitchCases[i].TrueBB == 0 && SwitchCases[i].FalseBB == 0);
+ }
+}
+
+
+//===----------------------------------------------------------------------===//
+/// ScheduleAndEmitDAG - Pick a safe ordering and emit instructions for each
+/// target node in the graph.
+void SelectionDAGISel::ScheduleAndEmitDAG(SelectionDAG &DAG) {
+ if (ViewSchedDAGs) DAG.viewGraph();
+
+ RegisterScheduler::FunctionPassCtor Ctor = RegisterScheduler::getDefault();
+
+ if (!Ctor) {
+ Ctor = ISHeuristic;
+ RegisterScheduler::setDefault(Ctor);
+ }
+
+ ScheduleDAG *SL = Ctor(this, &DAG, BB);
+ BB = SL->Run();
+ delete SL;
+}
+
+
+HazardRecognizer *SelectionDAGISel::CreateTargetHazardRecognizer() {
+ return new HazardRecognizer();
+}
+
+//===----------------------------------------------------------------------===//
+// Helper functions used by the generated instruction selector.
+//===----------------------------------------------------------------------===//
+// Calls to these methods are generated by tblgen.
+
+/// CheckAndMask - The isel is trying to match something like (and X, 255). If
+/// the dag combiner simplified the 255, we still want to match. RHS is the
+/// actual value in the DAG on the RHS of an AND, and DesiredMaskS is the value
+/// specified in the .td file (e.g. 255).
+bool SelectionDAGISel::CheckAndMask(SDOperand LHS, ConstantSDNode *RHS,
+ int64_t DesiredMaskS) {
+ uint64_t ActualMask = RHS->getValue();
+ uint64_t DesiredMask =DesiredMaskS & MVT::getIntVTBitMask(LHS.getValueType());
+
+ // If the actual mask exactly matches, success!
+ if (ActualMask == DesiredMask)
+ return true;
+
+ // If the actual AND mask is allowing unallowed bits, this doesn't match.
+ if (ActualMask & ~DesiredMask)
+ return false;
+
+ // Otherwise, the DAG Combiner may have proven that the value coming in is
+ // either already zero or is not demanded. Check for known zero input bits.
+ uint64_t NeededMask = DesiredMask & ~ActualMask;
+ if (CurDAG->MaskedValueIsZero(LHS, NeededMask))
+ return true;
+
+ // TODO: check to see if missing bits are just not demanded.
+
+ // Otherwise, this pattern doesn't match.
+ return false;
+}
+
+/// CheckOrMask - The isel is trying to match something like (or X, 255). If
+/// the dag combiner simplified the 255, we still want to match. RHS is the
+/// actual value in the DAG on the RHS of an OR, and DesiredMaskS is the value
+/// specified in the .td file (e.g. 255).
+bool SelectionDAGISel::CheckOrMask(SDOperand LHS, ConstantSDNode *RHS,
+ int64_t DesiredMaskS) {
+ uint64_t ActualMask = RHS->getValue();
+ uint64_t DesiredMask =DesiredMaskS & MVT::getIntVTBitMask(LHS.getValueType());
+
+ // If the actual mask exactly matches, success!
+ if (ActualMask == DesiredMask)
+ return true;
+
+ // If the actual AND mask is allowing unallowed bits, this doesn't match.
+ if (ActualMask & ~DesiredMask)
+ return false;
+
+ // Otherwise, the DAG Combiner may have proven that the value coming in is
+ // either already zero or is not demanded. Check for known zero input bits.
+ uint64_t NeededMask = DesiredMask & ~ActualMask;
+
+ uint64_t KnownZero, KnownOne;
+ CurDAG->ComputeMaskedBits(LHS, NeededMask, KnownZero, KnownOne);
+
+ // If all the missing bits in the or are already known to be set, match!
+ if ((NeededMask & KnownOne) == NeededMask)
+ return true;
+
+ // TODO: check to see if missing bits are just not demanded.
+
+ // Otherwise, this pattern doesn't match.
+ return false;
+}
+
+
+/// SelectInlineAsmMemoryOperands - Calls to this are automatically generated
+/// by tblgen. Others should not call it.
+void SelectionDAGISel::
+SelectInlineAsmMemoryOperands(std::vector<SDOperand> &Ops, SelectionDAG &DAG) {
+ std::vector<SDOperand> InOps;
+ std::swap(InOps, Ops);
+
+ Ops.push_back(InOps[0]); // input chain.
+ Ops.push_back(InOps[1]); // input asm string.
+
+ unsigned i = 2, e = InOps.size();
+ if (InOps[e-1].getValueType() == MVT::Flag)
+ --e; // Don't process a flag operand if it is here.
+
+ while (i != e) {
+ unsigned Flags = cast<ConstantSDNode>(InOps[i])->getValue();
+ if ((Flags & 7) != 4 /*MEM*/) {
+ // Just skip over this operand, copying the operands verbatim.
+ Ops.insert(Ops.end(), InOps.begin()+i, InOps.begin()+i+(Flags >> 3) + 1);
+ i += (Flags >> 3) + 1;
+ } else {
+ assert((Flags >> 3) == 1 && "Memory operand with multiple values?");
+ // Otherwise, this is a memory operand. Ask the target to select it.
+ std::vector<SDOperand> SelOps;
+ if (SelectInlineAsmMemoryOperand(InOps[i+1], 'm', SelOps, DAG)) {
+ cerr << "Could not match memory address. Inline asm failure!\n";
+ exit(1);
+ }
+
+ // Add this to the output node.
+ MVT::ValueType IntPtrTy = DAG.getTargetLoweringInfo().getPointerTy();
+ Ops.push_back(DAG.getTargetConstant(4/*MEM*/ | (SelOps.size() << 3),
+ IntPtrTy));
+ Ops.insert(Ops.end(), SelOps.begin(), SelOps.end());
+ i += 2;
+ }
+ }
+
+ // Add the flag input back if present.
+ if (e != InOps.size())
+ Ops.push_back(InOps.back());
+}
+
+char SelectionDAGISel::ID = 0;
diff --git a/lib/CodeGen/SelectionDAG/SelectionDAGPrinter.cpp b/lib/CodeGen/SelectionDAG/SelectionDAGPrinter.cpp
new file mode 100644
index 0000000..12b5682
--- /dev/null
+++ b/lib/CodeGen/SelectionDAG/SelectionDAGPrinter.cpp
@@ -0,0 +1,245 @@
+//===-- SelectionDAGPrinter.cpp - Implement SelectionDAG::viewGraph() -----===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file was developed by the LLVM research group and is distributed under
+// the University of Illinois Open Source License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This implements the SelectionDAG::viewGraph method.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Constants.h"
+#include "llvm/Function.h"
+#include "llvm/Assembly/Writer.h"
+#include "llvm/CodeGen/SelectionDAG.h"
+#include "llvm/CodeGen/MachineConstantPool.h"
+#include "llvm/CodeGen/MachineFunction.h"
+#include "llvm/Target/MRegisterInfo.h"
+#include "llvm/Target/TargetMachine.h"
+#include "llvm/Support/GraphWriter.h"
+#include "llvm/ADT/StringExtras.h"
+#include "llvm/Config/config.h"
+#include <fstream>
+#include <sstream>
+using namespace llvm;
+
+namespace llvm {
+ template<>
+ struct DOTGraphTraits<SelectionDAG*> : public DefaultDOTGraphTraits {
+ static std::string getGraphName(const SelectionDAG *G) {
+ return G->getMachineFunction().getFunction()->getName();
+ }
+
+ static bool renderGraphFromBottomUp() {
+ return true;
+ }
+
+ static bool hasNodeAddressLabel(const SDNode *Node,
+ const SelectionDAG *Graph) {
+ return true;
+ }
+
+ /// If you want to override the dot attributes printed for a particular
+ /// edge, override this method.
+ template<typename EdgeIter>
+ static std::string getEdgeAttributes(const void *Node, EdgeIter EI) {
+ SDOperand Op = EI.getNode()->getOperand(EI.getOperand());
+ MVT::ValueType VT = Op.getValueType();
+ if (VT == MVT::Flag)
+ return "color=red,style=bold";
+ else if (VT == MVT::Other)
+ return "color=blue,style=dashed";
+ return "";
+ }
+
+
+ static std::string getNodeLabel(const SDNode *Node,
+ const SelectionDAG *Graph);
+ static std::string getNodeAttributes(const SDNode *N,
+ const SelectionDAG *Graph) {
+#ifndef NDEBUG
+ const std::string &Attrs = Graph->getGraphAttrs(N);
+ if (!Attrs.empty()) {
+ if (Attrs.find("shape=") == std::string::npos)
+ return std::string("shape=Mrecord,") + Attrs;
+ else
+ return Attrs;
+ }
+#endif
+ return "shape=Mrecord";
+ }
+
+ static void addCustomGraphFeatures(SelectionDAG *G,
+ GraphWriter<SelectionDAG*> &GW) {
+ GW.emitSimpleNode(0, "plaintext=circle", "GraphRoot");
+ if (G->getRoot().Val)
+ GW.emitEdge(0, -1, G->getRoot().Val, -1, "");
+ }
+ };
+}
+
+std::string DOTGraphTraits<SelectionDAG*>::getNodeLabel(const SDNode *Node,
+ const SelectionDAG *G) {
+ std::string Op = Node->getOperationName(G);
+
+ for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i)
+ if (Node->getValueType(i) == MVT::Other)
+ Op += ":ch";
+ else
+ Op = Op + ":" + MVT::getValueTypeString(Node->getValueType(i));
+
+ if (const ConstantSDNode *CSDN = dyn_cast<ConstantSDNode>(Node)) {
+ Op += ": " + utostr(CSDN->getValue());
+ } else if (const ConstantFPSDNode *CSDN = dyn_cast<ConstantFPSDNode>(Node)) {
+ Op += ": " + ftostr(CSDN->getValue());
+ } else if (const GlobalAddressSDNode *GADN =
+ dyn_cast<GlobalAddressSDNode>(Node)) {
+ int offset = GADN->getOffset();
+ Op += ": " + GADN->getGlobal()->getName();
+ if (offset > 0)
+ Op += "+" + itostr(offset);
+ else
+ Op += itostr(offset);
+ } else if (const FrameIndexSDNode *FIDN = dyn_cast<FrameIndexSDNode>(Node)) {
+ Op += " " + itostr(FIDN->getIndex());
+ } else if (const JumpTableSDNode *JTDN = dyn_cast<JumpTableSDNode>(Node)) {
+ Op += " " + itostr(JTDN->getIndex());
+ } else if (const ConstantPoolSDNode *CP = dyn_cast<ConstantPoolSDNode>(Node)){
+ if (CP->isMachineConstantPoolEntry()) {
+ std::ostringstream SS;
+ CP->getMachineCPVal()->print(SS);
+ Op += "<" + SS.str() + ">";
+ } else {
+ if (ConstantFP *CFP = dyn_cast<ConstantFP>(CP->getConstVal()))
+ Op += "<" + ftostr(CFP->getValue()) + ">";
+ else if (ConstantInt *CI = dyn_cast<ConstantInt>(CP->getConstVal()))
+ Op += "<" + utostr(CI->getZExtValue()) + ">";
+ else {
+ std::ostringstream SS;
+ WriteAsOperand(SS, CP->getConstVal(), false);
+ Op += "<" + SS.str() + ">";
+ }
+ }
+ } else if (const BasicBlockSDNode *BBDN = dyn_cast<BasicBlockSDNode>(Node)) {
+ Op = "BB: ";
+ const Value *LBB = (const Value*)BBDN->getBasicBlock()->getBasicBlock();
+ if (LBB)
+ Op += LBB->getName();
+ //Op += " " + (const void*)BBDN->getBasicBlock();
+ } else if (const RegisterSDNode *R = dyn_cast<RegisterSDNode>(Node)) {
+ if (G && R->getReg() != 0 &&
+ MRegisterInfo::isPhysicalRegister(R->getReg())) {
+ Op = Op + " " + G->getTarget().getRegisterInfo()->getName(R->getReg());
+ } else {
+ Op += " #" + utostr(R->getReg());
+ }
+ } else if (const ExternalSymbolSDNode *ES =
+ dyn_cast<ExternalSymbolSDNode>(Node)) {
+ Op += "'" + std::string(ES->getSymbol()) + "'";
+ } else if (const SrcValueSDNode *M = dyn_cast<SrcValueSDNode>(Node)) {
+ if (M->getValue())
+ Op += "<" + M->getValue()->getName() + ":" + itostr(M->getOffset()) + ">";
+ else
+ Op += "<null:" + itostr(M->getOffset()) + ">";
+ } else if (const VTSDNode *N = dyn_cast<VTSDNode>(Node)) {
+ Op = Op + " VT=" + MVT::getValueTypeString(N->getVT());
+ } else if (const StringSDNode *N = dyn_cast<StringSDNode>(Node)) {
+ Op = Op + "\"" + N->getValue() + "\"";
+ } else if (const LoadSDNode *LD = dyn_cast<LoadSDNode>(Node)) {
+ bool doExt = true;
+ switch (LD->getExtensionType()) {
+ default: doExt = false; break;
+ case ISD::EXTLOAD:
+ Op = Op + "<anyext ";
+ break;
+ case ISD::SEXTLOAD:
+ Op = Op + " <sext ";
+ break;
+ case ISD::ZEXTLOAD:
+ Op = Op + " <zext ";
+ break;
+ }
+ if (doExt)
+ Op = Op + MVT::getValueTypeString(LD->getLoadedVT()) + ">";
+
+ Op += LD->getIndexedModeName(LD->getAddressingMode());
+ } else if (const StoreSDNode *ST = dyn_cast<StoreSDNode>(Node)) {
+ if (ST->isTruncatingStore())
+ Op = Op + "<trunc " + MVT::getValueTypeString(ST->getStoredVT()) + ">";
+ Op += ST->getIndexedModeName(ST->getAddressingMode());
+ }
+
+ return Op;
+}
+
+
+/// viewGraph - Pop up a ghostview window with the reachable parts of the DAG
+/// rendered using 'dot'.
+///
+void SelectionDAG::viewGraph() {
+// This code is only for debugging!
+#ifndef NDEBUG
+ ViewGraph(this, "dag." + getMachineFunction().getFunction()->getName());
+#else
+ cerr << "SelectionDAG::viewGraph is only available in debug builds on "
+ << "systems with Graphviz or gv!\n";
+#endif // NDEBUG
+}
+
+
+/// clearGraphAttrs - Clear all previously defined node graph attributes.
+/// Intended to be used from a debugging tool (eg. gdb).
+void SelectionDAG::clearGraphAttrs() {
+#ifndef NDEBUG
+ NodeGraphAttrs.clear();
+#else
+ cerr << "SelectionDAG::clearGraphAttrs is only available in debug builds"
+ << " on systems with Graphviz or gv!\n";
+#endif
+}
+
+
+/// setGraphAttrs - Set graph attributes for a node. (eg. "color=red".)
+///
+void SelectionDAG::setGraphAttrs(const SDNode *N, const char *Attrs) {
+#ifndef NDEBUG
+ NodeGraphAttrs[N] = Attrs;
+#else
+ cerr << "SelectionDAG::setGraphAttrs is only available in debug builds"
+ << " on systems with Graphviz or gv!\n";
+#endif
+}
+
+
+/// getGraphAttrs - Get graph attributes for a node. (eg. "color=red".)
+/// Used from getNodeAttributes.
+const std::string SelectionDAG::getGraphAttrs(const SDNode *N) const {
+#ifndef NDEBUG
+ std::map<const SDNode *, std::string>::const_iterator I =
+ NodeGraphAttrs.find(N);
+
+ if (I != NodeGraphAttrs.end())
+ return I->second;
+ else
+ return "";
+#else
+ cerr << "SelectionDAG::getGraphAttrs is only available in debug builds"
+ << " on systems with Graphviz or gv!\n";
+ return std::string("");
+#endif
+}
+
+/// setGraphColor - Convenience for setting node color attribute.
+///
+void SelectionDAG::setGraphColor(const SDNode *N, const char *Color) {
+#ifndef NDEBUG
+ NodeGraphAttrs[N] = std::string("color=") + Color;
+#else
+ cerr << "SelectionDAG::setGraphColor is only available in debug builds"
+ << " on systems with Graphviz or gv!\n";
+#endif
+}
+
diff --git a/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/lib/CodeGen/SelectionDAG/TargetLowering.cpp
new file mode 100644
index 0000000..1b7b436
--- /dev/null
+++ b/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -0,0 +1,1753 @@
+//===-- TargetLowering.cpp - Implement the TargetLowering class -----------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file was developed by the LLVM research group and is distributed under
+// the University of Illinois Open Source License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This implements the TargetLowering class.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Target/TargetLowering.h"
+#include "llvm/Target/TargetData.h"
+#include "llvm/Target/TargetMachine.h"
+#include "llvm/Target/MRegisterInfo.h"
+#include "llvm/DerivedTypes.h"
+#include "llvm/CodeGen/SelectionDAG.h"
+#include "llvm/ADT/StringExtras.h"
+#include "llvm/Support/MathExtras.h"
+using namespace llvm;
+
+/// InitLibcallNames - Set default libcall names.
+///
+static void InitLibcallNames(const char **Names) {
+ Names[RTLIB::SHL_I32] = "__ashlsi3";
+ Names[RTLIB::SHL_I64] = "__ashldi3";
+ Names[RTLIB::SRL_I32] = "__lshrsi3";
+ Names[RTLIB::SRL_I64] = "__lshrdi3";
+ Names[RTLIB::SRA_I32] = "__ashrsi3";
+ Names[RTLIB::SRA_I64] = "__ashrdi3";
+ Names[RTLIB::MUL_I32] = "__mulsi3";
+ Names[RTLIB::MUL_I64] = "__muldi3";
+ Names[RTLIB::SDIV_I32] = "__divsi3";
+ Names[RTLIB::SDIV_I64] = "__divdi3";
+ Names[RTLIB::UDIV_I32] = "__udivsi3";
+ Names[RTLIB::UDIV_I64] = "__udivdi3";
+ Names[RTLIB::SREM_I32] = "__modsi3";
+ Names[RTLIB::SREM_I64] = "__moddi3";
+ Names[RTLIB::UREM_I32] = "__umodsi3";
+ Names[RTLIB::UREM_I64] = "__umoddi3";
+ Names[RTLIB::NEG_I32] = "__negsi2";
+ Names[RTLIB::NEG_I64] = "__negdi2";
+ Names[RTLIB::ADD_F32] = "__addsf3";
+ Names[RTLIB::ADD_F64] = "__adddf3";
+ Names[RTLIB::SUB_F32] = "__subsf3";
+ Names[RTLIB::SUB_F64] = "__subdf3";
+ Names[RTLIB::MUL_F32] = "__mulsf3";
+ Names[RTLIB::MUL_F64] = "__muldf3";
+ Names[RTLIB::DIV_F32] = "__divsf3";
+ Names[RTLIB::DIV_F64] = "__divdf3";
+ Names[RTLIB::REM_F32] = "fmodf";
+ Names[RTLIB::REM_F64] = "fmod";
+ Names[RTLIB::NEG_F32] = "__negsf2";
+ Names[RTLIB::NEG_F64] = "__negdf2";
+ Names[RTLIB::POWI_F32] = "__powisf2";
+ Names[RTLIB::POWI_F64] = "__powidf2";
+ Names[RTLIB::SQRT_F32] = "sqrtf";
+ Names[RTLIB::SQRT_F64] = "sqrt";
+ Names[RTLIB::SIN_F32] = "sinf";
+ Names[RTLIB::SIN_F64] = "sin";
+ Names[RTLIB::COS_F32] = "cosf";
+ Names[RTLIB::COS_F64] = "cos";
+ Names[RTLIB::FPEXT_F32_F64] = "__extendsfdf2";
+ Names[RTLIB::FPROUND_F64_F32] = "__truncdfsf2";
+ Names[RTLIB::FPTOSINT_F32_I32] = "__fixsfsi";
+ Names[RTLIB::FPTOSINT_F32_I64] = "__fixsfdi";
+ Names[RTLIB::FPTOSINT_F64_I32] = "__fixdfsi";
+ Names[RTLIB::FPTOSINT_F64_I64] = "__fixdfdi";
+ Names[RTLIB::FPTOUINT_F32_I32] = "__fixunssfsi";
+ Names[RTLIB::FPTOUINT_F32_I64] = "__fixunssfdi";
+ Names[RTLIB::FPTOUINT_F64_I32] = "__fixunsdfsi";
+ Names[RTLIB::FPTOUINT_F64_I64] = "__fixunsdfdi";
+ Names[RTLIB::SINTTOFP_I32_F32] = "__floatsisf";
+ Names[RTLIB::SINTTOFP_I32_F64] = "__floatsidf";
+ Names[RTLIB::SINTTOFP_I64_F32] = "__floatdisf";
+ Names[RTLIB::SINTTOFP_I64_F64] = "__floatdidf";
+ Names[RTLIB::UINTTOFP_I32_F32] = "__floatunsisf";
+ Names[RTLIB::UINTTOFP_I32_F64] = "__floatunsidf";
+ Names[RTLIB::UINTTOFP_I64_F32] = "__floatundisf";
+ Names[RTLIB::UINTTOFP_I64_F64] = "__floatundidf";
+ Names[RTLIB::OEQ_F32] = "__eqsf2";
+ Names[RTLIB::OEQ_F64] = "__eqdf2";
+ Names[RTLIB::UNE_F32] = "__nesf2";
+ Names[RTLIB::UNE_F64] = "__nedf2";
+ Names[RTLIB::OGE_F32] = "__gesf2";
+ Names[RTLIB::OGE_F64] = "__gedf2";
+ Names[RTLIB::OLT_F32] = "__ltsf2";
+ Names[RTLIB::OLT_F64] = "__ltdf2";
+ Names[RTLIB::OLE_F32] = "__lesf2";
+ Names[RTLIB::OLE_F64] = "__ledf2";
+ Names[RTLIB::OGT_F32] = "__gtsf2";
+ Names[RTLIB::OGT_F64] = "__gtdf2";
+ Names[RTLIB::UO_F32] = "__unordsf2";
+ Names[RTLIB::UO_F64] = "__unorddf2";
+ Names[RTLIB::O_F32] = "__unordsf2";
+ Names[RTLIB::O_F64] = "__unorddf2";
+}
+
+/// InitCmpLibcallCCs - Set default comparison libcall CC.
+///
+static void InitCmpLibcallCCs(ISD::CondCode *CCs) {
+ memset(CCs, ISD::SETCC_INVALID, sizeof(ISD::CondCode)*RTLIB::UNKNOWN_LIBCALL);
+ CCs[RTLIB::OEQ_F32] = ISD::SETEQ;
+ CCs[RTLIB::OEQ_F64] = ISD::SETEQ;
+ CCs[RTLIB::UNE_F32] = ISD::SETNE;
+ CCs[RTLIB::UNE_F64] = ISD::SETNE;
+ CCs[RTLIB::OGE_F32] = ISD::SETGE;
+ CCs[RTLIB::OGE_F64] = ISD::SETGE;
+ CCs[RTLIB::OLT_F32] = ISD::SETLT;
+ CCs[RTLIB::OLT_F64] = ISD::SETLT;
+ CCs[RTLIB::OLE_F32] = ISD::SETLE;
+ CCs[RTLIB::OLE_F64] = ISD::SETLE;
+ CCs[RTLIB::OGT_F32] = ISD::SETGT;
+ CCs[RTLIB::OGT_F64] = ISD::SETGT;
+ CCs[RTLIB::UO_F32] = ISD::SETNE;
+ CCs[RTLIB::UO_F64] = ISD::SETNE;
+ CCs[RTLIB::O_F32] = ISD::SETEQ;
+ CCs[RTLIB::O_F64] = ISD::SETEQ;
+}
+
+TargetLowering::TargetLowering(TargetMachine &tm)
+ : TM(tm), TD(TM.getTargetData()) {
+ assert(ISD::BUILTIN_OP_END <= 156 &&
+ "Fixed size array in TargetLowering is not large enough!");
+ // All operations default to being supported.
+ memset(OpActions, 0, sizeof(OpActions));
+ memset(LoadXActions, 0, sizeof(LoadXActions));
+ memset(&StoreXActions, 0, sizeof(StoreXActions));
+ memset(&IndexedModeActions, 0, sizeof(IndexedModeActions));
+
+ // Set all indexed load / store to expand.
+ for (unsigned VT = 0; VT != (unsigned)MVT::LAST_VALUETYPE; ++VT) {
+ for (unsigned IM = (unsigned)ISD::PRE_INC;
+ IM != (unsigned)ISD::LAST_INDEXED_MODE; ++IM) {
+ setIndexedLoadAction(IM, (MVT::ValueType)VT, Expand);
+ setIndexedStoreAction(IM, (MVT::ValueType)VT, Expand);
+ }
+ }
+
+ IsLittleEndian = TD->isLittleEndian();
+ UsesGlobalOffsetTable = false;
+ ShiftAmountTy = SetCCResultTy = PointerTy = getValueType(TD->getIntPtrType());
+ ShiftAmtHandling = Undefined;
+ memset(RegClassForVT, 0,MVT::LAST_VALUETYPE*sizeof(TargetRegisterClass*));
+ memset(TargetDAGCombineArray, 0,
+ sizeof(TargetDAGCombineArray)/sizeof(TargetDAGCombineArray[0]));
+ maxStoresPerMemset = maxStoresPerMemcpy = maxStoresPerMemmove = 8;
+ allowUnalignedMemoryAccesses = false;
+ UseUnderscoreSetJmp = false;
+ UseUnderscoreLongJmp = false;
+ SelectIsExpensive = false;
+ IntDivIsCheap = false;
+ Pow2DivIsCheap = false;
+ StackPointerRegisterToSaveRestore = 0;
+ ExceptionPointerRegister = 0;
+ ExceptionSelectorRegister = 0;
+ SchedPreferenceInfo = SchedulingForLatency;
+ JumpBufSize = 0;
+ JumpBufAlignment = 0;
+ IfCvtBlockSizeLimit = 2;
+
+ InitLibcallNames(LibcallRoutineNames);
+ InitCmpLibcallCCs(CmpLibcallCCs);
+}
+
+TargetLowering::~TargetLowering() {}
+
+/// computeRegisterProperties - Once all of the register classes are added,
+/// this allows us to compute derived properties we expose.
+void TargetLowering::computeRegisterProperties() {
+ assert(MVT::LAST_VALUETYPE <= 32 &&
+ "Too many value types for ValueTypeActions to hold!");
+
+ // Everything defaults to needing one register.
+ for (unsigned i = 0; i != MVT::LAST_VALUETYPE; ++i) {
+ NumRegistersForVT[i] = 1;
+ RegisterTypeForVT[i] = TransformToType[i] = i;
+ }
+ // ...except isVoid, which doesn't need any registers.
+ NumRegistersForVT[MVT::isVoid] = 0;
+
+ // Find the largest integer register class.
+ unsigned LargestIntReg = MVT::i128;
+ for (; RegClassForVT[LargestIntReg] == 0; --LargestIntReg)
+ assert(LargestIntReg != MVT::i1 && "No integer registers defined!");
+
+ // Every integer value type larger than this largest register takes twice as
+ // many registers to represent as the previous ValueType.
+ for (MVT::ValueType ExpandedReg = LargestIntReg + 1;
+ MVT::isInteger(ExpandedReg); ++ExpandedReg) {
+ NumRegistersForVT[ExpandedReg] = 2*NumRegistersForVT[ExpandedReg-1];
+ RegisterTypeForVT[ExpandedReg] = LargestIntReg;
+ TransformToType[ExpandedReg] = ExpandedReg - 1;
+ ValueTypeActions.setTypeAction(ExpandedReg, Expand);
+ }
+
+ // Inspect all of the ValueType's smaller than the largest integer
+ // register to see which ones need promotion.
+ MVT::ValueType LegalIntReg = LargestIntReg;
+ for (MVT::ValueType IntReg = LargestIntReg - 1;
+ IntReg >= MVT::i1; --IntReg) {
+ if (isTypeLegal(IntReg)) {
+ LegalIntReg = IntReg;
+ } else {
+ RegisterTypeForVT[IntReg] = TransformToType[IntReg] = LegalIntReg;
+ ValueTypeActions.setTypeAction(IntReg, Promote);
+ }
+ }
+
+ // Decide how to handle f64. If the target does not have native f64 support,
+ // expand it to i64 and we will be generating soft float library calls.
+ if (!isTypeLegal(MVT::f64)) {
+ NumRegistersForVT[MVT::f64] = NumRegistersForVT[MVT::i64];
+ RegisterTypeForVT[MVT::f64] = RegisterTypeForVT[MVT::i64];
+ TransformToType[MVT::f64] = MVT::i64;
+ ValueTypeActions.setTypeAction(MVT::f64, Expand);
+ }
+
+ // Decide how to handle f32. If the target does not have native support for
+ // f32, promote it to f64 if it is legal. Otherwise, expand it to i32.
+ if (!isTypeLegal(MVT::f32)) {
+ if (isTypeLegal(MVT::f64)) {
+ NumRegistersForVT[MVT::f32] = NumRegistersForVT[MVT::f64];
+ RegisterTypeForVT[MVT::f32] = RegisterTypeForVT[MVT::f64];
+ TransformToType[MVT::f32] = MVT::f64;
+ ValueTypeActions.setTypeAction(MVT::f32, Promote);
+ } else {
+ NumRegistersForVT[MVT::f32] = NumRegistersForVT[MVT::i32];
+ RegisterTypeForVT[MVT::f32] = RegisterTypeForVT[MVT::i32];
+ TransformToType[MVT::f32] = MVT::i32;
+ ValueTypeActions.setTypeAction(MVT::f32, Expand);
+ }
+ }
+
+ // Loop over all of the vector value types to see which need transformations.
+ for (MVT::ValueType i = MVT::FIRST_VECTOR_VALUETYPE;
+ i <= MVT::LAST_VECTOR_VALUETYPE; ++i) {
+ if (!isTypeLegal(i)) {
+ MVT::ValueType IntermediateVT, RegisterVT;
+ unsigned NumIntermediates;
+ NumRegistersForVT[i] =
+ getVectorTypeBreakdown(i,
+ IntermediateVT, NumIntermediates,
+ RegisterVT);
+ RegisterTypeForVT[i] = RegisterVT;
+ TransformToType[i] = MVT::Other; // this isn't actually used
+ ValueTypeActions.setTypeAction(i, Expand);
+ }
+ }
+}
+
+const char *TargetLowering::getTargetNodeName(unsigned Opcode) const {
+ return NULL;
+}
+
+/// getVectorTypeBreakdown - Vector types are broken down into some number of
+/// legal first class types. For example, MVT::v8f32 maps to 2 MVT::v4f32
+/// with Altivec or SSE1, or 8 promoted MVT::f64 values with the X86 FP stack.
+/// Similarly, MVT::v2i64 turns into 4 MVT::i32 values with both PPC and X86.
+///
+/// This method returns the number of registers needed, and the VT for each
+/// register. It also returns the VT and quantity of the intermediate values
+/// before they are promoted/expanded.
+///
+unsigned TargetLowering::getVectorTypeBreakdown(MVT::ValueType VT,
+ MVT::ValueType &IntermediateVT,
+ unsigned &NumIntermediates,
+ MVT::ValueType &RegisterVT) const {
+ // Figure out the right, legal destination reg to copy into.
+ unsigned NumElts = MVT::getVectorNumElements(VT);
+ MVT::ValueType EltTy = MVT::getVectorElementType(VT);
+
+ unsigned NumVectorRegs = 1;
+
+ // Divide the input until we get to a supported size. This will always
+ // end with a scalar if the target doesn't support vectors.
+ while (NumElts > 1 &&
+ !isTypeLegal(MVT::getVectorType(EltTy, NumElts))) {
+ NumElts >>= 1;
+ NumVectorRegs <<= 1;
+ }
+
+ NumIntermediates = NumVectorRegs;
+
+ MVT::ValueType NewVT = MVT::getVectorType(EltTy, NumElts);
+ if (!isTypeLegal(NewVT))
+ NewVT = EltTy;
+ IntermediateVT = NewVT;
+
+ MVT::ValueType DestVT = getTypeToTransformTo(NewVT);
+ RegisterVT = DestVT;
+ if (DestVT < NewVT) {
+ // Value is expanded, e.g. i64 -> i16.
+ return NumVectorRegs*(MVT::getSizeInBits(NewVT)/MVT::getSizeInBits(DestVT));
+ } else {
+ // Otherwise, promotion or legal types use the same number of registers as
+ // the vector decimated to the appropriate level.
+ return NumVectorRegs;
+ }
+
+ return 1;
+}
+
+//===----------------------------------------------------------------------===//
+// Optimization Methods
+//===----------------------------------------------------------------------===//
+
+/// ShrinkDemandedConstant - Check to see if the specified operand of the
+/// specified instruction is a constant integer. If so, check to see if there
+/// are any bits set in the constant that are not demanded. If so, shrink the
+/// constant and return true.
+bool TargetLowering::TargetLoweringOpt::ShrinkDemandedConstant(SDOperand Op,
+ uint64_t Demanded) {
+ // FIXME: ISD::SELECT, ISD::SELECT_CC
+ switch(Op.getOpcode()) {
+ default: break;
+ case ISD::AND:
+ case ISD::OR:
+ case ISD::XOR:
+ if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1)))
+ if ((~Demanded & C->getValue()) != 0) {
+ MVT::ValueType VT = Op.getValueType();
+ SDOperand New = DAG.getNode(Op.getOpcode(), VT, Op.getOperand(0),
+ DAG.getConstant(Demanded & C->getValue(),
+ VT));
+ return CombineTo(Op, New);
+ }
+ break;
+ }
+ return false;
+}
+
+/// SimplifyDemandedBits - Look at Op. At this point, we know that only the
+/// DemandedMask bits of the result of Op are ever used downstream. If we can
+/// use this information to simplify Op, create a new simplified DAG node and
+/// return true, returning the original and new nodes in Old and New. Otherwise,
+/// analyze the expression and return a mask of KnownOne and KnownZero bits for
+/// the expression (used to simplify the caller). The KnownZero/One bits may
+/// only be accurate for those bits in the DemandedMask.
+bool TargetLowering::SimplifyDemandedBits(SDOperand Op, uint64_t DemandedMask,
+ uint64_t &KnownZero,
+ uint64_t &KnownOne,
+ TargetLoweringOpt &TLO,
+ unsigned Depth) const {
+ KnownZero = KnownOne = 0; // Don't know anything.
+
+ // The masks are not wide enough to represent this type! Should use APInt.
+ if (Op.getValueType() == MVT::i128)
+ return false;
+
+ // Other users may use these bits.
+ if (!Op.Val->hasOneUse()) {
+ if (Depth != 0) {
+ // If not at the root, Just compute the KnownZero/KnownOne bits to
+ // simplify things downstream.
+ TLO.DAG.ComputeMaskedBits(Op, DemandedMask, KnownZero, KnownOne, Depth);
+ return false;
+ }
+ // If this is the root being simplified, allow it to have multiple uses,
+ // just set the DemandedMask to all bits.
+ DemandedMask = MVT::getIntVTBitMask(Op.getValueType());
+ } else if (DemandedMask == 0) {
+ // Not demanding any bits from Op.
+ if (Op.getOpcode() != ISD::UNDEF)
+ return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::UNDEF, Op.getValueType()));
+ return false;
+ } else if (Depth == 6) { // Limit search depth.
+ return false;
+ }
+
+ uint64_t KnownZero2, KnownOne2, KnownZeroOut, KnownOneOut;
+ switch (Op.getOpcode()) {
+ case ISD::Constant:
+ // We know all of the bits for a constant!
+ KnownOne = cast<ConstantSDNode>(Op)->getValue() & DemandedMask;
+ KnownZero = ~KnownOne & DemandedMask;
+ return false; // Don't fall through, will infinitely loop.
+ case ISD::AND:
+ // If the RHS is a constant, check to see if the LHS would be zero without
+ // using the bits from the RHS. Below, we use knowledge about the RHS to
+ // simplify the LHS, here we're using information from the LHS to simplify
+ // the RHS.
+ if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
+ uint64_t LHSZero, LHSOne;
+ TLO.DAG.ComputeMaskedBits(Op.getOperand(0), DemandedMask,
+ LHSZero, LHSOne, Depth+1);
+ // If the LHS already has zeros where RHSC does, this and is dead.
+ if ((LHSZero & DemandedMask) == (~RHSC->getValue() & DemandedMask))
+ return TLO.CombineTo(Op, Op.getOperand(0));
+ // If any of the set bits in the RHS are known zero on the LHS, shrink
+ // the constant.
+ if (TLO.ShrinkDemandedConstant(Op, ~LHSZero & DemandedMask))
+ return true;
+ }
+
+ if (SimplifyDemandedBits(Op.getOperand(1), DemandedMask, KnownZero,
+ KnownOne, TLO, Depth+1))
+ return true;
+ assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?");
+ if (SimplifyDemandedBits(Op.getOperand(0), DemandedMask & ~KnownZero,
+ KnownZero2, KnownOne2, TLO, Depth+1))
+ return true;
+ assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?");
+
+ // If all of the demanded bits are known one on one side, return the other.
+ // These bits cannot contribute to the result of the 'and'.
+ if ((DemandedMask & ~KnownZero2 & KnownOne)==(DemandedMask & ~KnownZero2))
+ return TLO.CombineTo(Op, Op.getOperand(0));
+ if ((DemandedMask & ~KnownZero & KnownOne2)==(DemandedMask & ~KnownZero))
+ return TLO.CombineTo(Op, Op.getOperand(1));
+ // If all of the demanded bits in the inputs are known zeros, return zero.
+ if ((DemandedMask & (KnownZero|KnownZero2)) == DemandedMask)
+ return TLO.CombineTo(Op, TLO.DAG.getConstant(0, Op.getValueType()));
+ // If the RHS is a constant, see if we can simplify it.
+ if (TLO.ShrinkDemandedConstant(Op, DemandedMask & ~KnownZero2))
+ return true;
+
+ // Output known-1 bits are only known if set in both the LHS & RHS.
+ KnownOne &= KnownOne2;
+ // Output known-0 are known to be clear if zero in either the LHS | RHS.
+ KnownZero |= KnownZero2;
+ break;
+ case ISD::OR:
+ if (SimplifyDemandedBits(Op.getOperand(1), DemandedMask, KnownZero,
+ KnownOne, TLO, Depth+1))
+ return true;
+ assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?");
+ if (SimplifyDemandedBits(Op.getOperand(0), DemandedMask & ~KnownOne,
+ KnownZero2, KnownOne2, TLO, Depth+1))
+ return true;
+ assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?");
+
+ // If all of the demanded bits are known zero on one side, return the other.
+ // These bits cannot contribute to the result of the 'or'.
+ if ((DemandedMask & ~KnownOne2 & KnownZero) == (DemandedMask & ~KnownOne2))
+ return TLO.CombineTo(Op, Op.getOperand(0));
+ if ((DemandedMask & ~KnownOne & KnownZero2) == (DemandedMask & ~KnownOne))
+ return TLO.CombineTo(Op, Op.getOperand(1));
+ // If all of the potentially set bits on one side are known to be set on
+ // the other side, just use the 'other' side.
+ if ((DemandedMask & (~KnownZero) & KnownOne2) ==
+ (DemandedMask & (~KnownZero)))
+ return TLO.CombineTo(Op, Op.getOperand(0));
+ if ((DemandedMask & (~KnownZero2) & KnownOne) ==
+ (DemandedMask & (~KnownZero2)))
+ return TLO.CombineTo(Op, Op.getOperand(1));
+ // If the RHS is a constant, see if we can simplify it.
+ if (TLO.ShrinkDemandedConstant(Op, DemandedMask))
+ return true;
+
+ // Output known-0 bits are only known if clear in both the LHS & RHS.
+ KnownZero &= KnownZero2;
+ // Output known-1 are known to be set if set in either the LHS | RHS.
+ KnownOne |= KnownOne2;
+ break;
+ case ISD::XOR:
+ if (SimplifyDemandedBits(Op.getOperand(1), DemandedMask, KnownZero,
+ KnownOne, TLO, Depth+1))
+ return true;
+ assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?");
+ if (SimplifyDemandedBits(Op.getOperand(0), DemandedMask, KnownZero2,
+ KnownOne2, TLO, Depth+1))
+ return true;
+ assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?");
+
+ // If all of the demanded bits are known zero on one side, return the other.
+ // These bits cannot contribute to the result of the 'xor'.
+ if ((DemandedMask & KnownZero) == DemandedMask)
+ return TLO.CombineTo(Op, Op.getOperand(0));
+ if ((DemandedMask & KnownZero2) == DemandedMask)
+ return TLO.CombineTo(Op, Op.getOperand(1));
+
+ // If all of the unknown bits are known to be zero on one side or the other
+ // (but not both) turn this into an *inclusive* or.
+ // e.g. (A & C1)^(B & C2) -> (A & C1)|(B & C2) iff C1&C2 == 0
+ if ((DemandedMask & ~KnownZero & ~KnownZero2) == 0)
+ return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::OR, Op.getValueType(),
+ Op.getOperand(0),
+ Op.getOperand(1)));
+
+ // Output known-0 bits are known if clear or set in both the LHS & RHS.
+ KnownZeroOut = (KnownZero & KnownZero2) | (KnownOne & KnownOne2);
+ // Output known-1 are known to be set if set in only one of the LHS, RHS.
+ KnownOneOut = (KnownZero & KnownOne2) | (KnownOne & KnownZero2);
+
+ // If all of the demanded bits on one side are known, and all of the set
+ // bits on that side are also known to be set on the other side, turn this
+ // into an AND, as we know the bits will be cleared.
+ // e.g. (X | C1) ^ C2 --> (X | C1) & ~C2 iff (C1&C2) == C2
+ if ((DemandedMask & (KnownZero|KnownOne)) == DemandedMask) { // all known
+ if ((KnownOne & KnownOne2) == KnownOne) {
+ MVT::ValueType VT = Op.getValueType();
+ SDOperand ANDC = TLO.DAG.getConstant(~KnownOne & DemandedMask, VT);
+ return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::AND, VT, Op.getOperand(0),
+ ANDC));
+ }
+ }
+
+ // If the RHS is a constant, see if we can simplify it.
+ // FIXME: for XOR, we prefer to force bits to 1 if they will make a -1.
+ if (TLO.ShrinkDemandedConstant(Op, DemandedMask))
+ return true;
+
+ KnownZero = KnownZeroOut;
+ KnownOne = KnownOneOut;
+ break;
+ case ISD::SETCC:
+ // If we know the result of a setcc has the top bits zero, use this info.
+ if (getSetCCResultContents() == TargetLowering::ZeroOrOneSetCCResult)
+ KnownZero |= (MVT::getIntVTBitMask(Op.getValueType()) ^ 1ULL);
+ break;
+ case ISD::SELECT:
+ if (SimplifyDemandedBits(Op.getOperand(2), DemandedMask, KnownZero,
+ KnownOne, TLO, Depth+1))
+ return true;
+ if (SimplifyDemandedBits(Op.getOperand(1), DemandedMask, KnownZero2,
+ KnownOne2, TLO, Depth+1))
+ return true;
+ assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?");
+ assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?");
+
+ // If the operands are constants, see if we can simplify them.
+ if (TLO.ShrinkDemandedConstant(Op, DemandedMask))
+ return true;
+
+ // Only known if known in both the LHS and RHS.
+ KnownOne &= KnownOne2;
+ KnownZero &= KnownZero2;
+ break;
+ case ISD::SELECT_CC:
+ if (SimplifyDemandedBits(Op.getOperand(3), DemandedMask, KnownZero,
+ KnownOne, TLO, Depth+1))
+ return true;
+ if (SimplifyDemandedBits(Op.getOperand(2), DemandedMask, KnownZero2,
+ KnownOne2, TLO, Depth+1))
+ return true;
+ assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?");
+ assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?");
+
+ // If the operands are constants, see if we can simplify them.
+ if (TLO.ShrinkDemandedConstant(Op, DemandedMask))
+ return true;
+
+ // Only known if known in both the LHS and RHS.
+ KnownOne &= KnownOne2;
+ KnownZero &= KnownZero2;
+ break;
+ case ISD::SHL:
+ if (ConstantSDNode *SA = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
+ unsigned ShAmt = SA->getValue();
+ SDOperand InOp = Op.getOperand(0);
+
+ // If this is ((X >>u C1) << ShAmt), see if we can simplify this into a
+ // single shift. We can do this if the bottom bits (which are shifted
+ // out) are never demanded.
+ if (InOp.getOpcode() == ISD::SRL &&
+ isa<ConstantSDNode>(InOp.getOperand(1))) {
+ if (ShAmt && (DemandedMask & ((1ULL << ShAmt)-1)) == 0) {
+ unsigned C1 = cast<ConstantSDNode>(InOp.getOperand(1))->getValue();
+ unsigned Opc = ISD::SHL;
+ int Diff = ShAmt-C1;
+ if (Diff < 0) {
+ Diff = -Diff;
+ Opc = ISD::SRL;
+ }
+
+ SDOperand NewSA =
+ TLO.DAG.getConstant(Diff, Op.getOperand(1).getValueType());
+ MVT::ValueType VT = Op.getValueType();
+ return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, VT,
+ InOp.getOperand(0), NewSA));
+ }
+ }
+
+ if (SimplifyDemandedBits(Op.getOperand(0), DemandedMask >> ShAmt,
+ KnownZero, KnownOne, TLO, Depth+1))
+ return true;
+ KnownZero <<= SA->getValue();
+ KnownOne <<= SA->getValue();
+ KnownZero |= (1ULL << SA->getValue())-1; // low bits known zero.
+ }
+ break;
+ case ISD::SRL:
+ if (ConstantSDNode *SA = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
+ MVT::ValueType VT = Op.getValueType();
+ unsigned ShAmt = SA->getValue();
+ uint64_t TypeMask = MVT::getIntVTBitMask(VT);
+ unsigned VTSize = MVT::getSizeInBits(VT);
+ SDOperand InOp = Op.getOperand(0);
+
+ // If this is ((X << C1) >>u ShAmt), see if we can simplify this into a
+ // single shift. We can do this if the top bits (which are shifted out)
+ // are never demanded.
+ if (InOp.getOpcode() == ISD::SHL &&
+ isa<ConstantSDNode>(InOp.getOperand(1))) {
+ if (ShAmt && (DemandedMask & (~0ULL << (VTSize-ShAmt))) == 0) {
+ unsigned C1 = cast<ConstantSDNode>(InOp.getOperand(1))->getValue();
+ unsigned Opc = ISD::SRL;
+ int Diff = ShAmt-C1;
+ if (Diff < 0) {
+ Diff = -Diff;
+ Opc = ISD::SHL;
+ }
+
+ SDOperand NewSA =
+ TLO.DAG.getConstant(Diff, Op.getOperand(1).getValueType());
+ return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, VT,
+ InOp.getOperand(0), NewSA));
+ }
+ }
+
+ // Compute the new bits that are at the top now.
+ if (SimplifyDemandedBits(InOp, (DemandedMask << ShAmt) & TypeMask,
+ KnownZero, KnownOne, TLO, Depth+1))
+ return true;
+ assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?");
+ KnownZero &= TypeMask;
+ KnownOne &= TypeMask;
+ KnownZero >>= ShAmt;
+ KnownOne >>= ShAmt;
+
+ uint64_t HighBits = (1ULL << ShAmt)-1;
+ HighBits <<= VTSize - ShAmt;
+ KnownZero |= HighBits; // High bits known zero.
+ }
+ break;
+ case ISD::SRA:
+ if (ConstantSDNode *SA = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
+ MVT::ValueType VT = Op.getValueType();
+ unsigned ShAmt = SA->getValue();
+
+ // Compute the new bits that are at the top now.
+ uint64_t TypeMask = MVT::getIntVTBitMask(VT);
+
+ uint64_t InDemandedMask = (DemandedMask << ShAmt) & TypeMask;
+
+ // If any of the demanded bits are produced by the sign extension, we also
+ // demand the input sign bit.
+ uint64_t HighBits = (1ULL << ShAmt)-1;
+ HighBits <<= MVT::getSizeInBits(VT) - ShAmt;
+ if (HighBits & DemandedMask)
+ InDemandedMask |= MVT::getIntVTSignBit(VT);
+
+ if (SimplifyDemandedBits(Op.getOperand(0), InDemandedMask,
+ KnownZero, KnownOne, TLO, Depth+1))
+ return true;
+ assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?");
+ KnownZero &= TypeMask;
+ KnownOne &= TypeMask;
+ KnownZero >>= ShAmt;
+ KnownOne >>= ShAmt;
+
+ // Handle the sign bits.
+ uint64_t SignBit = MVT::getIntVTSignBit(VT);
+ SignBit >>= ShAmt; // Adjust to where it is now in the mask.
+
+ // If the input sign bit is known to be zero, or if none of the top bits
+ // are demanded, turn this into an unsigned shift right.
+ if ((KnownZero & SignBit) || (HighBits & ~DemandedMask) == HighBits) {
+ return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SRL, VT, Op.getOperand(0),
+ Op.getOperand(1)));
+ } else if (KnownOne & SignBit) { // New bits are known one.
+ KnownOne |= HighBits;
+ }
+ }
+ break;
+ case ISD::SIGN_EXTEND_INREG: {
+ MVT::ValueType EVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
+
+ // Sign extension. Compute the demanded bits in the result that are not
+ // present in the input.
+ uint64_t NewBits = ~MVT::getIntVTBitMask(EVT) & DemandedMask;
+
+ // If none of the extended bits are demanded, eliminate the sextinreg.
+ if (NewBits == 0)
+ return TLO.CombineTo(Op, Op.getOperand(0));
+
+ uint64_t InSignBit = MVT::getIntVTSignBit(EVT);
+ int64_t InputDemandedBits = DemandedMask & MVT::getIntVTBitMask(EVT);
+
+ // Since the sign extended bits are demanded, we know that the sign
+ // bit is demanded.
+ InputDemandedBits |= InSignBit;
+
+ if (SimplifyDemandedBits(Op.getOperand(0), InputDemandedBits,
+ KnownZero, KnownOne, TLO, Depth+1))
+ return true;
+ assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?");
+
+ // If the sign bit of the input is known set or clear, then we know the
+ // top bits of the result.
+
+ // If the input sign bit is known zero, convert this into a zero extension.
+ if (KnownZero & InSignBit)
+ return TLO.CombineTo(Op,
+ TLO.DAG.getZeroExtendInReg(Op.getOperand(0), EVT));
+
+ if (KnownOne & InSignBit) { // Input sign bit known set
+ KnownOne |= NewBits;
+ KnownZero &= ~NewBits;
+ } else { // Input sign bit unknown
+ KnownZero &= ~NewBits;
+ KnownOne &= ~NewBits;
+ }
+ break;
+ }
+ case ISD::CTTZ:
+ case ISD::CTLZ:
+ case ISD::CTPOP: {
+ MVT::ValueType VT = Op.getValueType();
+ unsigned LowBits = Log2_32(MVT::getSizeInBits(VT))+1;
+ KnownZero = ~((1ULL << LowBits)-1) & MVT::getIntVTBitMask(VT);
+ KnownOne = 0;
+ break;
+ }
+ case ISD::LOAD: {
+ if (ISD::isZEXTLoad(Op.Val)) {
+ LoadSDNode *LD = cast<LoadSDNode>(Op);
+ MVT::ValueType VT = LD->getLoadedVT();
+ KnownZero |= ~MVT::getIntVTBitMask(VT) & DemandedMask;
+ }
+ break;
+ }
+ case ISD::ZERO_EXTEND: {
+ uint64_t InMask = MVT::getIntVTBitMask(Op.getOperand(0).getValueType());
+
+ // If none of the top bits are demanded, convert this into an any_extend.
+ uint64_t NewBits = (~InMask) & DemandedMask;
+ if (NewBits == 0)
+ return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::ANY_EXTEND,
+ Op.getValueType(),
+ Op.getOperand(0)));
+
+ if (SimplifyDemandedBits(Op.getOperand(0), DemandedMask & InMask,
+ KnownZero, KnownOne, TLO, Depth+1))
+ return true;
+ assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?");
+ KnownZero |= NewBits;
+ break;
+ }
+ case ISD::SIGN_EXTEND: {
+ MVT::ValueType InVT = Op.getOperand(0).getValueType();
+ uint64_t InMask = MVT::getIntVTBitMask(InVT);
+ uint64_t InSignBit = MVT::getIntVTSignBit(InVT);
+ uint64_t NewBits = (~InMask) & DemandedMask;
+
+ // If none of the top bits are demanded, convert this into an any_extend.
+ if (NewBits == 0)
+ return TLO.CombineTo(Op,TLO.DAG.getNode(ISD::ANY_EXTEND,Op.getValueType(),
+ Op.getOperand(0)));
+
+ // Since some of the sign extended bits are demanded, we know that the sign
+ // bit is demanded.
+ uint64_t InDemandedBits = DemandedMask & InMask;
+ InDemandedBits |= InSignBit;
+
+ if (SimplifyDemandedBits(Op.getOperand(0), InDemandedBits, KnownZero,
+ KnownOne, TLO, Depth+1))
+ return true;
+
+ // If the sign bit is known zero, convert this to a zero extend.
+ if (KnownZero & InSignBit)
+ return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::ZERO_EXTEND,
+ Op.getValueType(),
+ Op.getOperand(0)));
+
+ // If the sign bit is known one, the top bits match.
+ if (KnownOne & InSignBit) {
+ KnownOne |= NewBits;
+ KnownZero &= ~NewBits;
+ } else { // Otherwise, top bits aren't known.
+ KnownOne &= ~NewBits;
+ KnownZero &= ~NewBits;
+ }
+ break;
+ }
+ case ISD::ANY_EXTEND: {
+ uint64_t InMask = MVT::getIntVTBitMask(Op.getOperand(0).getValueType());
+ if (SimplifyDemandedBits(Op.getOperand(0), DemandedMask & InMask,
+ KnownZero, KnownOne, TLO, Depth+1))
+ return true;
+ assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?");
+ break;
+ }
+ case ISD::TRUNCATE: {
+ // Simplify the input, using demanded bit information, and compute the known
+ // zero/one bits live out.
+ if (SimplifyDemandedBits(Op.getOperand(0), DemandedMask,
+ KnownZero, KnownOne, TLO, Depth+1))
+ return true;
+
+ // If the input is only used by this truncate, see if we can shrink it based
+ // on the known demanded bits.
+ if (Op.getOperand(0).Val->hasOneUse()) {
+ SDOperand In = Op.getOperand(0);
+ switch (In.getOpcode()) {
+ default: break;
+ case ISD::SRL:
+ // Shrink SRL by a constant if none of the high bits shifted in are
+ // demanded.
+ if (ConstantSDNode *ShAmt = dyn_cast<ConstantSDNode>(In.getOperand(1))){
+ uint64_t HighBits = MVT::getIntVTBitMask(In.getValueType());
+ HighBits &= ~MVT::getIntVTBitMask(Op.getValueType());
+ HighBits >>= ShAmt->getValue();
+
+ if (ShAmt->getValue() < MVT::getSizeInBits(Op.getValueType()) &&
+ (DemandedMask & HighBits) == 0) {
+ // None of the shifted in bits are needed. Add a truncate of the
+ // shift input, then shift it.
+ SDOperand NewTrunc = TLO.DAG.getNode(ISD::TRUNCATE,
+ Op.getValueType(),
+ In.getOperand(0));
+ return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SRL,Op.getValueType(),
+ NewTrunc, In.getOperand(1)));
+ }
+ }
+ break;
+ }
+ }
+
+ assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?");
+ uint64_t OutMask = MVT::getIntVTBitMask(Op.getValueType());
+ KnownZero &= OutMask;
+ KnownOne &= OutMask;
+ break;
+ }
+ case ISD::AssertZext: {
+ MVT::ValueType VT = cast<VTSDNode>(Op.getOperand(1))->getVT();
+ uint64_t InMask = MVT::getIntVTBitMask(VT);
+ if (SimplifyDemandedBits(Op.getOperand(0), DemandedMask & InMask,
+ KnownZero, KnownOne, TLO, Depth+1))
+ return true;
+ assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?");
+ KnownZero |= ~InMask & DemandedMask;
+ break;
+ }
+ case ISD::ADD:
+ case ISD::SUB:
+ case ISD::INTRINSIC_WO_CHAIN:
+ case ISD::INTRINSIC_W_CHAIN:
+ case ISD::INTRINSIC_VOID:
+ // Just use ComputeMaskedBits to compute output bits.
+ TLO.DAG.ComputeMaskedBits(Op, DemandedMask, KnownZero, KnownOne, Depth);
+ break;
+ }
+
+ // If we know the value of all of the demanded bits, return this as a
+ // constant.
+ if ((DemandedMask & (KnownZero|KnownOne)) == DemandedMask)
+ return TLO.CombineTo(Op, TLO.DAG.getConstant(KnownOne, Op.getValueType()));
+
+ return false;
+}
+
+/// computeMaskedBitsForTargetNode - Determine which of the bits specified
+/// in Mask are known to be either zero or one and return them in the
+/// KnownZero/KnownOne bitsets.
+void TargetLowering::computeMaskedBitsForTargetNode(const SDOperand Op,
+ uint64_t Mask,
+ uint64_t &KnownZero,
+ uint64_t &KnownOne,
+ const SelectionDAG &DAG,
+ unsigned Depth) const {
+ assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||
+ Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
+ Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
+ Op.getOpcode() == ISD::INTRINSIC_VOID) &&
+ "Should use MaskedValueIsZero if you don't know whether Op"
+ " is a target node!");
+ KnownZero = 0;
+ KnownOne = 0;
+}
+
+/// ComputeNumSignBitsForTargetNode - This method can be implemented by
+/// targets that want to expose additional information about sign bits to the
+/// DAG Combiner.
+unsigned TargetLowering::ComputeNumSignBitsForTargetNode(SDOperand Op,
+ unsigned Depth) const {
+ assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||
+ Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
+ Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
+ Op.getOpcode() == ISD::INTRINSIC_VOID) &&
+ "Should use ComputeNumSignBits if you don't know whether Op"
+ " is a target node!");
+ return 1;
+}
+
+
+/// SimplifySetCC - Try to simplify a setcc built with the specified operands
+/// and cc. If it is unable to simplify it, return a null SDOperand.
+SDOperand
+TargetLowering::SimplifySetCC(MVT::ValueType VT, SDOperand N0, SDOperand N1,
+ ISD::CondCode Cond, bool foldBooleans,
+ DAGCombinerInfo &DCI) const {
+ SelectionDAG &DAG = DCI.DAG;
+
+ // These setcc operations always fold.
+ switch (Cond) {
+ default: break;
+ case ISD::SETFALSE:
+ case ISD::SETFALSE2: return DAG.getConstant(0, VT);
+ case ISD::SETTRUE:
+ case ISD::SETTRUE2: return DAG.getConstant(1, VT);
+ }
+
+ if (ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.Val)) {
+ uint64_t C1 = N1C->getValue();
+ if (isa<ConstantSDNode>(N0.Val)) {
+ return DAG.FoldSetCC(VT, N0, N1, Cond);
+ } else {
+ // If the LHS is '(srl (ctlz x), 5)', the RHS is 0/1, and this is an
+ // equality comparison, then we're just comparing whether X itself is
+ // zero.
+ if (N0.getOpcode() == ISD::SRL && (C1 == 0 || C1 == 1) &&
+ N0.getOperand(0).getOpcode() == ISD::CTLZ &&
+ N0.getOperand(1).getOpcode() == ISD::Constant) {
+ unsigned ShAmt = cast<ConstantSDNode>(N0.getOperand(1))->getValue();
+ if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
+ ShAmt == Log2_32(MVT::getSizeInBits(N0.getValueType()))) {
+ if ((C1 == 0) == (Cond == ISD::SETEQ)) {
+ // (srl (ctlz x), 5) == 0 -> X != 0
+ // (srl (ctlz x), 5) != 1 -> X != 0
+ Cond = ISD::SETNE;
+ } else {
+ // (srl (ctlz x), 5) != 0 -> X == 0
+ // (srl (ctlz x), 5) == 1 -> X == 0
+ Cond = ISD::SETEQ;
+ }
+ SDOperand Zero = DAG.getConstant(0, N0.getValueType());
+ return DAG.getSetCC(VT, N0.getOperand(0).getOperand(0),
+ Zero, Cond);
+ }
+ }
+
+ // If the LHS is a ZERO_EXTEND, perform the comparison on the input.
+ if (N0.getOpcode() == ISD::ZERO_EXTEND) {
+ unsigned InSize = MVT::getSizeInBits(N0.getOperand(0).getValueType());
+
+ // If the comparison constant has bits in the upper part, the
+ // zero-extended value could never match.
+ if (C1 & (~0ULL << InSize)) {
+ unsigned VSize = MVT::getSizeInBits(N0.getValueType());
+ switch (Cond) {
+ case ISD::SETUGT:
+ case ISD::SETUGE:
+ case ISD::SETEQ: return DAG.getConstant(0, VT);
+ case ISD::SETULT:
+ case ISD::SETULE:
+ case ISD::SETNE: return DAG.getConstant(1, VT);
+ case ISD::SETGT:
+ case ISD::SETGE:
+ // True if the sign bit of C1 is set.
+ return DAG.getConstant((C1 & (1ULL << (VSize-1))) != 0, VT);
+ case ISD::SETLT:
+ case ISD::SETLE:
+ // True if the sign bit of C1 isn't set.
+ return DAG.getConstant((C1 & (1ULL << (VSize-1))) == 0, VT);
+ default:
+ break;
+ }
+ }
+
+ // Otherwise, we can perform the comparison with the low bits.
+ switch (Cond) {
+ case ISD::SETEQ:
+ case ISD::SETNE:
+ case ISD::SETUGT:
+ case ISD::SETUGE:
+ case ISD::SETULT:
+ case ISD::SETULE:
+ return DAG.getSetCC(VT, N0.getOperand(0),
+ DAG.getConstant(C1, N0.getOperand(0).getValueType()),
+ Cond);
+ default:
+ break; // todo, be more careful with signed comparisons
+ }
+ } else if (N0.getOpcode() == ISD::SIGN_EXTEND_INREG &&
+ (Cond == ISD::SETEQ || Cond == ISD::SETNE)) {
+ MVT::ValueType ExtSrcTy = cast<VTSDNode>(N0.getOperand(1))->getVT();
+ unsigned ExtSrcTyBits = MVT::getSizeInBits(ExtSrcTy);
+ MVT::ValueType ExtDstTy = N0.getValueType();
+ unsigned ExtDstTyBits = MVT::getSizeInBits(ExtDstTy);
+
+ // If the extended part has any inconsistent bits, it cannot ever
+ // compare equal. In other words, they have to be all ones or all
+ // zeros.
+ uint64_t ExtBits =
+ (~0ULL >> (64-ExtSrcTyBits)) & (~0ULL << (ExtDstTyBits-1));
+ if ((C1 & ExtBits) != 0 && (C1 & ExtBits) != ExtBits)
+ return DAG.getConstant(Cond == ISD::SETNE, VT);
+
+ SDOperand ZextOp;
+ MVT::ValueType Op0Ty = N0.getOperand(0).getValueType();
+ if (Op0Ty == ExtSrcTy) {
+ ZextOp = N0.getOperand(0);
+ } else {
+ int64_t Imm = ~0ULL >> (64-ExtSrcTyBits);
+ ZextOp = DAG.getNode(ISD::AND, Op0Ty, N0.getOperand(0),
+ DAG.getConstant(Imm, Op0Ty));
+ }
+ if (!DCI.isCalledByLegalizer())
+ DCI.AddToWorklist(ZextOp.Val);
+ // Otherwise, make this a use of a zext.
+ return DAG.getSetCC(VT, ZextOp,
+ DAG.getConstant(C1 & (~0ULL>>(64-ExtSrcTyBits)),
+ ExtDstTy),
+ Cond);
+ } else if ((N1C->getValue() == 0 || N1C->getValue() == 1) &&
+ (Cond == ISD::SETEQ || Cond == ISD::SETNE)) {
+
+ // SETCC (SETCC), [0|1], [EQ|NE] -> SETCC
+ if (N0.getOpcode() == ISD::SETCC) {
+ bool TrueWhenTrue = (Cond == ISD::SETEQ) ^ (N1C->getValue() != 1);
+ if (TrueWhenTrue)
+ return N0;
+
+ // Invert the condition.
+ ISD::CondCode CC = cast<CondCodeSDNode>(N0.getOperand(2))->get();
+ CC = ISD::getSetCCInverse(CC,
+ MVT::isInteger(N0.getOperand(0).getValueType()));
+ return DAG.getSetCC(VT, N0.getOperand(0), N0.getOperand(1), CC);
+ }
+
+ if ((N0.getOpcode() == ISD::XOR ||
+ (N0.getOpcode() == ISD::AND &&
+ N0.getOperand(0).getOpcode() == ISD::XOR &&
+ N0.getOperand(1) == N0.getOperand(0).getOperand(1))) &&
+ isa<ConstantSDNode>(N0.getOperand(1)) &&
+ cast<ConstantSDNode>(N0.getOperand(1))->getValue() == 1) {
+ // If this is (X^1) == 0/1, swap the RHS and eliminate the xor. We
+ // can only do this if the top bits are known zero.
+ if (DAG.MaskedValueIsZero(N0,
+ MVT::getIntVTBitMask(N0.getValueType())-1)){
+ // Okay, get the un-inverted input value.
+ SDOperand Val;
+ if (N0.getOpcode() == ISD::XOR)
+ Val = N0.getOperand(0);
+ else {
+ assert(N0.getOpcode() == ISD::AND &&
+ N0.getOperand(0).getOpcode() == ISD::XOR);
+ // ((X^1)&1)^1 -> X & 1
+ Val = DAG.getNode(ISD::AND, N0.getValueType(),
+ N0.getOperand(0).getOperand(0),
+ N0.getOperand(1));
+ }
+ return DAG.getSetCC(VT, Val, N1,
+ Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);
+ }
+ }
+ }
+
+ uint64_t MinVal, MaxVal;
+ unsigned OperandBitSize = MVT::getSizeInBits(N1C->getValueType(0));
+ if (ISD::isSignedIntSetCC(Cond)) {
+ MinVal = 1ULL << (OperandBitSize-1);
+ if (OperandBitSize != 1) // Avoid X >> 64, which is undefined.
+ MaxVal = ~0ULL >> (65-OperandBitSize);
+ else
+ MaxVal = 0;
+ } else {
+ MinVal = 0;
+ MaxVal = ~0ULL >> (64-OperandBitSize);
+ }
+
+ // Canonicalize GE/LE comparisons to use GT/LT comparisons.
+ if (Cond == ISD::SETGE || Cond == ISD::SETUGE) {
+ if (C1 == MinVal) return DAG.getConstant(1, VT); // X >= MIN --> true
+ --C1; // X >= C0 --> X > (C0-1)
+ return DAG.getSetCC(VT, N0, DAG.getConstant(C1, N1.getValueType()),
+ (Cond == ISD::SETGE) ? ISD::SETGT : ISD::SETUGT);
+ }
+
+ if (Cond == ISD::SETLE || Cond == ISD::SETULE) {
+ if (C1 == MaxVal) return DAG.getConstant(1, VT); // X <= MAX --> true
+ ++C1; // X <= C0 --> X < (C0+1)
+ return DAG.getSetCC(VT, N0, DAG.getConstant(C1, N1.getValueType()),
+ (Cond == ISD::SETLE) ? ISD::SETLT : ISD::SETULT);
+ }
+
+ if ((Cond == ISD::SETLT || Cond == ISD::SETULT) && C1 == MinVal)
+ return DAG.getConstant(0, VT); // X < MIN --> false
+ if ((Cond == ISD::SETGE || Cond == ISD::SETUGE) && C1 == MinVal)
+ return DAG.getConstant(1, VT); // X >= MIN --> true
+ if ((Cond == ISD::SETGT || Cond == ISD::SETUGT) && C1 == MaxVal)
+ return DAG.getConstant(0, VT); // X > MAX --> false
+ if ((Cond == ISD::SETLE || Cond == ISD::SETULE) && C1 == MaxVal)
+ return DAG.getConstant(1, VT); // X <= MAX --> true
+
+ // Canonicalize setgt X, Min --> setne X, Min
+ if ((Cond == ISD::SETGT || Cond == ISD::SETUGT) && C1 == MinVal)
+ return DAG.getSetCC(VT, N0, N1, ISD::SETNE);
+ // Canonicalize setlt X, Max --> setne X, Max
+ if ((Cond == ISD::SETLT || Cond == ISD::SETULT) && C1 == MaxVal)
+ return DAG.getSetCC(VT, N0, N1, ISD::SETNE);
+
+ // If we have setult X, 1, turn it into seteq X, 0
+ if ((Cond == ISD::SETLT || Cond == ISD::SETULT) && C1 == MinVal+1)
+ return DAG.getSetCC(VT, N0, DAG.getConstant(MinVal, N0.getValueType()),
+ ISD::SETEQ);
+ // If we have setugt X, Max-1, turn it into seteq X, Max
+ else if ((Cond == ISD::SETGT || Cond == ISD::SETUGT) && C1 == MaxVal-1)
+ return DAG.getSetCC(VT, N0, DAG.getConstant(MaxVal, N0.getValueType()),
+ ISD::SETEQ);
+
+ // If we have "setcc X, C0", check to see if we can shrink the immediate
+ // by changing cc.
+
+ // SETUGT X, SINTMAX -> SETLT X, 0
+ if (Cond == ISD::SETUGT && OperandBitSize != 1 &&
+ C1 == (~0ULL >> (65-OperandBitSize)))
+ return DAG.getSetCC(VT, N0, DAG.getConstant(0, N1.getValueType()),
+ ISD::SETLT);
+
+ // FIXME: Implement the rest of these.
+
+ // Fold bit comparisons when we can.
+ if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
+ VT == N0.getValueType() && N0.getOpcode() == ISD::AND)
+ if (ConstantSDNode *AndRHS =
+ dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
+ if (Cond == ISD::SETNE && C1 == 0) {// (X & 8) != 0 --> (X & 8) >> 3
+ // Perform the xform if the AND RHS is a single bit.
+ if (isPowerOf2_64(AndRHS->getValue())) {
+ return DAG.getNode(ISD::SRL, VT, N0,
+ DAG.getConstant(Log2_64(AndRHS->getValue()),
+ getShiftAmountTy()));
+ }
+ } else if (Cond == ISD::SETEQ && C1 == AndRHS->getValue()) {
+ // (X & 8) == 8 --> (X & 8) >> 3
+ // Perform the xform if C1 is a single bit.
+ if (isPowerOf2_64(C1)) {
+ return DAG.getNode(ISD::SRL, VT, N0,
+ DAG.getConstant(Log2_64(C1), getShiftAmountTy()));
+ }
+ }
+ }
+ }
+ } else if (isa<ConstantSDNode>(N0.Val)) {
+ // Ensure that the constant occurs on the RHS.
+ return DAG.getSetCC(VT, N1, N0, ISD::getSetCCSwappedOperands(Cond));
+ }
+
+ if (isa<ConstantFPSDNode>(N0.Val)) {
+ // Constant fold or commute setcc.
+ SDOperand O = DAG.FoldSetCC(VT, N0, N1, Cond);
+ if (O.Val) return O;
+ }
+
+ if (N0 == N1) {
+ // We can always fold X == X for integer setcc's.
+ if (MVT::isInteger(N0.getValueType()))
+ return DAG.getConstant(ISD::isTrueWhenEqual(Cond), VT);
+ unsigned UOF = ISD::getUnorderedFlavor(Cond);
+ if (UOF == 2) // FP operators that are undefined on NaNs.
+ return DAG.getConstant(ISD::isTrueWhenEqual(Cond), VT);
+ if (UOF == unsigned(ISD::isTrueWhenEqual(Cond)))
+ return DAG.getConstant(UOF, VT);
+ // Otherwise, we can't fold it. However, we can simplify it to SETUO/SETO
+ // if it is not already.
+ ISD::CondCode NewCond = UOF == 0 ? ISD::SETO : ISD::SETUO;
+ if (NewCond != Cond)
+ return DAG.getSetCC(VT, N0, N1, NewCond);
+ }
+
+ if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
+ MVT::isInteger(N0.getValueType())) {
+ if (N0.getOpcode() == ISD::ADD || N0.getOpcode() == ISD::SUB ||
+ N0.getOpcode() == ISD::XOR) {
+ // Simplify (X+Y) == (X+Z) --> Y == Z
+ if (N0.getOpcode() == N1.getOpcode()) {
+ if (N0.getOperand(0) == N1.getOperand(0))
+ return DAG.getSetCC(VT, N0.getOperand(1), N1.getOperand(1), Cond);
+ if (N0.getOperand(1) == N1.getOperand(1))
+ return DAG.getSetCC(VT, N0.getOperand(0), N1.getOperand(0), Cond);
+ if (DAG.isCommutativeBinOp(N0.getOpcode())) {
+ // If X op Y == Y op X, try other combinations.
+ if (N0.getOperand(0) == N1.getOperand(1))
+ return DAG.getSetCC(VT, N0.getOperand(1), N1.getOperand(0), Cond);
+ if (N0.getOperand(1) == N1.getOperand(0))
+ return DAG.getSetCC(VT, N0.getOperand(0), N1.getOperand(1), Cond);
+ }
+ }
+
+ if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(N1)) {
+ if (ConstantSDNode *LHSR = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
+ // Turn (X+C1) == C2 --> X == C2-C1
+ if (N0.getOpcode() == ISD::ADD && N0.Val->hasOneUse()) {
+ return DAG.getSetCC(VT, N0.getOperand(0),
+ DAG.getConstant(RHSC->getValue()-LHSR->getValue(),
+ N0.getValueType()), Cond);
+ }
+
+ // Turn (X^C1) == C2 into X == C1^C2 iff X&~C1 = 0.
+ if (N0.getOpcode() == ISD::XOR)
+ // If we know that all of the inverted bits are zero, don't bother
+ // performing the inversion.
+ if (DAG.MaskedValueIsZero(N0.getOperand(0), ~LHSR->getValue()))
+ return DAG.getSetCC(VT, N0.getOperand(0),
+ DAG.getConstant(LHSR->getValue()^RHSC->getValue(),
+ N0.getValueType()), Cond);
+ }
+
+ // Turn (C1-X) == C2 --> X == C1-C2
+ if (ConstantSDNode *SUBC = dyn_cast<ConstantSDNode>(N0.getOperand(0))) {
+ if (N0.getOpcode() == ISD::SUB && N0.Val->hasOneUse()) {
+ return DAG.getSetCC(VT, N0.getOperand(1),
+ DAG.getConstant(SUBC->getValue()-RHSC->getValue(),
+ N0.getValueType()), Cond);
+ }
+ }
+ }
+
+ // Simplify (X+Z) == X --> Z == 0
+ if (N0.getOperand(0) == N1)
+ return DAG.getSetCC(VT, N0.getOperand(1),
+ DAG.getConstant(0, N0.getValueType()), Cond);
+ if (N0.getOperand(1) == N1) {
+ if (DAG.isCommutativeBinOp(N0.getOpcode()))
+ return DAG.getSetCC(VT, N0.getOperand(0),
+ DAG.getConstant(0, N0.getValueType()), Cond);
+ else if (N0.Val->hasOneUse()) {
+ assert(N0.getOpcode() == ISD::SUB && "Unexpected operation!");
+ // (Z-X) == X --> Z == X<<1
+ SDOperand SH = DAG.getNode(ISD::SHL, N1.getValueType(),
+ N1,
+ DAG.getConstant(1, getShiftAmountTy()));
+ if (!DCI.isCalledByLegalizer())
+ DCI.AddToWorklist(SH.Val);
+ return DAG.getSetCC(VT, N0.getOperand(0), SH, Cond);
+ }
+ }
+ }
+
+ if (N1.getOpcode() == ISD::ADD || N1.getOpcode() == ISD::SUB ||
+ N1.getOpcode() == ISD::XOR) {
+ // Simplify X == (X+Z) --> Z == 0
+ if (N1.getOperand(0) == N0) {
+ return DAG.getSetCC(VT, N1.getOperand(1),
+ DAG.getConstant(0, N1.getValueType()), Cond);
+ } else if (N1.getOperand(1) == N0) {
+ if (DAG.isCommutativeBinOp(N1.getOpcode())) {
+ return DAG.getSetCC(VT, N1.getOperand(0),
+ DAG.getConstant(0, N1.getValueType()), Cond);
+ } else if (N1.Val->hasOneUse()) {
+ assert(N1.getOpcode() == ISD::SUB && "Unexpected operation!");
+ // X == (Z-X) --> X<<1 == Z
+ SDOperand SH = DAG.getNode(ISD::SHL, N1.getValueType(), N0,
+ DAG.getConstant(1, getShiftAmountTy()));
+ if (!DCI.isCalledByLegalizer())
+ DCI.AddToWorklist(SH.Val);
+ return DAG.getSetCC(VT, SH, N1.getOperand(0), Cond);
+ }
+ }
+ }
+ }
+
+ // Fold away ALL boolean setcc's.
+ SDOperand Temp;
+ if (N0.getValueType() == MVT::i1 && foldBooleans) {
+ switch (Cond) {
+ default: assert(0 && "Unknown integer setcc!");
+ case ISD::SETEQ: // X == Y -> (X^Y)^1
+ Temp = DAG.getNode(ISD::XOR, MVT::i1, N0, N1);
+ N0 = DAG.getNode(ISD::XOR, MVT::i1, Temp, DAG.getConstant(1, MVT::i1));
+ if (!DCI.isCalledByLegalizer())
+ DCI.AddToWorklist(Temp.Val);
+ break;
+ case ISD::SETNE: // X != Y --> (X^Y)
+ N0 = DAG.getNode(ISD::XOR, MVT::i1, N0, N1);
+ break;
+ case ISD::SETGT: // X >s Y --> X == 0 & Y == 1 --> X^1 & Y
+ case ISD::SETULT: // X <u Y --> X == 0 & Y == 1 --> X^1 & Y
+ Temp = DAG.getNode(ISD::XOR, MVT::i1, N0, DAG.getConstant(1, MVT::i1));
+ N0 = DAG.getNode(ISD::AND, MVT::i1, N1, Temp);
+ if (!DCI.isCalledByLegalizer())
+ DCI.AddToWorklist(Temp.Val);
+ break;
+ case ISD::SETLT: // X <s Y --> X == 1 & Y == 0 --> Y^1 & X
+ case ISD::SETUGT: // X >u Y --> X == 1 & Y == 0 --> Y^1 & X
+ Temp = DAG.getNode(ISD::XOR, MVT::i1, N1, DAG.getConstant(1, MVT::i1));
+ N0 = DAG.getNode(ISD::AND, MVT::i1, N0, Temp);
+ if (!DCI.isCalledByLegalizer())
+ DCI.AddToWorklist(Temp.Val);
+ break;
+ case ISD::SETULE: // X <=u Y --> X == 0 | Y == 1 --> X^1 | Y
+ case ISD::SETGE: // X >=s Y --> X == 0 | Y == 1 --> X^1 | Y
+ Temp = DAG.getNode(ISD::XOR, MVT::i1, N0, DAG.getConstant(1, MVT::i1));
+ N0 = DAG.getNode(ISD::OR, MVT::i1, N1, Temp);
+ if (!DCI.isCalledByLegalizer())
+ DCI.AddToWorklist(Temp.Val);
+ break;
+ case ISD::SETUGE: // X >=u Y --> X == 1 | Y == 0 --> Y^1 | X
+ case ISD::SETLE: // X <=s Y --> X == 1 | Y == 0 --> Y^1 | X
+ Temp = DAG.getNode(ISD::XOR, MVT::i1, N1, DAG.getConstant(1, MVT::i1));
+ N0 = DAG.getNode(ISD::OR, MVT::i1, N0, Temp);
+ break;
+ }
+ if (VT != MVT::i1) {
+ if (!DCI.isCalledByLegalizer())
+ DCI.AddToWorklist(N0.Val);
+ // FIXME: If running after legalize, we probably can't do this.
+ N0 = DAG.getNode(ISD::ZERO_EXTEND, VT, N0);
+ }
+ return N0;
+ }
+
+ // Could not fold it.
+ return SDOperand();
+}
+
+SDOperand TargetLowering::
+PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const {
+ // Default implementation: no optimization.
+ return SDOperand();
+}
+
+//===----------------------------------------------------------------------===//
+// Inline Assembler Implementation Methods
+//===----------------------------------------------------------------------===//
+
+TargetLowering::ConstraintType
+TargetLowering::getConstraintType(const std::string &Constraint) const {
+ // FIXME: lots more standard ones to handle.
+ if (Constraint.size() == 1) {
+ switch (Constraint[0]) {
+ default: break;
+ case 'r': return C_RegisterClass;
+ case 'm': // memory
+ case 'o': // offsetable
+ case 'V': // not offsetable
+ return C_Memory;
+ case 'i': // Simple Integer or Relocatable Constant
+ case 'n': // Simple Integer
+ case 's': // Relocatable Constant
+ case 'X': // Allow ANY value.
+ case 'I': // Target registers.
+ case 'J':
+ case 'K':
+ case 'L':
+ case 'M':
+ case 'N':
+ case 'O':
+ case 'P':
+ return C_Other;
+ }
+ }
+
+ if (Constraint.size() > 1 && Constraint[0] == '{' &&
+ Constraint[Constraint.size()-1] == '}')
+ return C_Register;
+ return C_Unknown;
+}
+
+/// isOperandValidForConstraint - Return the specified operand (possibly
+/// modified) if the specified SDOperand is valid for the specified target
+/// constraint letter, otherwise return null.
+SDOperand TargetLowering::isOperandValidForConstraint(SDOperand Op,
+ char ConstraintLetter,
+ SelectionDAG &DAG) {
+ switch (ConstraintLetter) {
+ default: break;
+ case 'i': // Simple Integer or Relocatable Constant
+ case 'n': // Simple Integer
+ case 's': // Relocatable Constant
+ case 'X': { // Allows any operand.
+ // These operands are interested in values of the form (GV+C), where C may
+ // be folded in as an offset of GV, or it may be explicitly added. Also, it
+ // is possible and fine if either GV or C are missing.
+ ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op);
+ GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Op);
+
+ // If we have "(add GV, C)", pull out GV/C
+ if (Op.getOpcode() == ISD::ADD) {
+ C = dyn_cast<ConstantSDNode>(Op.getOperand(1));
+ GA = dyn_cast<GlobalAddressSDNode>(Op.getOperand(0));
+ if (C == 0 || GA == 0) {
+ C = dyn_cast<ConstantSDNode>(Op.getOperand(0));
+ GA = dyn_cast<GlobalAddressSDNode>(Op.getOperand(1));
+ }
+ if (C == 0 || GA == 0)
+ C = 0, GA = 0;
+ }
+
+ // If we find a valid operand, map to the TargetXXX version so that the
+ // value itself doesn't get selected.
+ if (GA) { // Either &GV or &GV+C
+ if (ConstraintLetter != 'n') {
+ int64_t Offs = GA->getOffset();
+ if (C) Offs += C->getValue();
+ return DAG.getTargetGlobalAddress(GA->getGlobal(), Op.getValueType(),
+ Offs);
+ }
+ }
+ if (C) { // just C, no GV.
+ // Simple constants are not allowed for 's'.
+ if (ConstraintLetter != 's')
+ return DAG.getTargetConstant(C->getValue(), Op.getValueType());
+ }
+ break;
+ }
+ }
+ return SDOperand(0,0);
+}
+
+std::vector<unsigned> TargetLowering::
+getRegClassForInlineAsmConstraint(const std::string &Constraint,
+ MVT::ValueType VT) const {
+ return std::vector<unsigned>();
+}
+
+
+std::pair<unsigned, const TargetRegisterClass*> TargetLowering::
+getRegForInlineAsmConstraint(const std::string &Constraint,
+ MVT::ValueType VT) const {
+ if (Constraint[0] != '{')
+ return std::pair<unsigned, const TargetRegisterClass*>(0, 0);
+ assert(*(Constraint.end()-1) == '}' && "Not a brace enclosed constraint?");
+
+ // Remove the braces from around the name.
+ std::string RegName(Constraint.begin()+1, Constraint.end()-1);
+
+ // Figure out which register class contains this reg.
+ const MRegisterInfo *RI = TM.getRegisterInfo();
+ for (MRegisterInfo::regclass_iterator RCI = RI->regclass_begin(),
+ E = RI->regclass_end(); RCI != E; ++RCI) {
+ const TargetRegisterClass *RC = *RCI;
+
+ // If none of the the value types for this register class are valid, we
+ // can't use it. For example, 64-bit reg classes on 32-bit targets.
+ bool isLegal = false;
+ for (TargetRegisterClass::vt_iterator I = RC->vt_begin(), E = RC->vt_end();
+ I != E; ++I) {
+ if (isTypeLegal(*I)) {
+ isLegal = true;
+ break;
+ }
+ }
+
+ if (!isLegal) continue;
+
+ for (TargetRegisterClass::iterator I = RC->begin(), E = RC->end();
+ I != E; ++I) {
+ if (StringsEqualNoCase(RegName, RI->get(*I).Name))
+ return std::make_pair(*I, RC);
+ }
+ }
+
+ return std::pair<unsigned, const TargetRegisterClass*>(0, 0);
+}
+
+//===----------------------------------------------------------------------===//
+// Loop Strength Reduction hooks
+//===----------------------------------------------------------------------===//
+
+/// isLegalAddressingMode - Return true if the addressing mode represented
+/// by AM is legal for this target, for a load/store of the specified type.
+bool TargetLowering::isLegalAddressingMode(const AddrMode &AM,
+ const Type *Ty) const {
+ // The default implementation of this implements a conservative RISCy, r+r and
+ // r+i addr mode.
+
+ // Allows a sign-extended 16-bit immediate field.
+ if (AM.BaseOffs <= -(1LL << 16) || AM.BaseOffs >= (1LL << 16)-1)
+ return false;
+
+ // No global is ever allowed as a base.
+ if (AM.BaseGV)
+ return false;
+
+ // Only support r+r,
+ switch (AM.Scale) {
+ case 0: // "r+i" or just "i", depending on HasBaseReg.
+ break;
+ case 1:
+ if (AM.HasBaseReg && AM.BaseOffs) // "r+r+i" is not allowed.
+ return false;
+ // Otherwise we have r+r or r+i.
+ break;
+ case 2:
+ if (AM.HasBaseReg || AM.BaseOffs) // 2*r+r or 2*r+i is not allowed.
+ return false;
+ // Allow 2*r as r+r.
+ break;
+ }
+
+ return true;
+}
+
+// Magic for divide replacement
+
+struct ms {
+ int64_t m; // magic number
+ int64_t s; // shift amount
+};
+
+struct mu {
+ uint64_t m; // magic number
+ int64_t a; // add indicator
+ int64_t s; // shift amount
+};
+
+/// magic - calculate the magic numbers required to codegen an integer sdiv as
+/// a sequence of multiply and shifts. Requires that the divisor not be 0, 1,
+/// or -1.
+static ms magic32(int32_t d) {
+ int32_t p;
+ uint32_t ad, anc, delta, q1, r1, q2, r2, t;
+ const uint32_t two31 = 0x80000000U;
+ struct ms mag;
+
+ ad = abs(d);
+ t = two31 + ((uint32_t)d >> 31);
+ anc = t - 1 - t%ad; // absolute value of nc
+ p = 31; // initialize p
+ q1 = two31/anc; // initialize q1 = 2p/abs(nc)
+ r1 = two31 - q1*anc; // initialize r1 = rem(2p,abs(nc))
+ q2 = two31/ad; // initialize q2 = 2p/abs(d)
+ r2 = two31 - q2*ad; // initialize r2 = rem(2p,abs(d))
+ do {
+ p = p + 1;
+ q1 = 2*q1; // update q1 = 2p/abs(nc)
+ r1 = 2*r1; // update r1 = rem(2p/abs(nc))
+ if (r1 >= anc) { // must be unsigned comparison
+ q1 = q1 + 1;
+ r1 = r1 - anc;
+ }
+ q2 = 2*q2; // update q2 = 2p/abs(d)
+ r2 = 2*r2; // update r2 = rem(2p/abs(d))
+ if (r2 >= ad) { // must be unsigned comparison
+ q2 = q2 + 1;
+ r2 = r2 - ad;
+ }
+ delta = ad - r2;
+ } while (q1 < delta || (q1 == delta && r1 == 0));
+
+ mag.m = (int32_t)(q2 + 1); // make sure to sign extend
+ if (d < 0) mag.m = -mag.m; // resulting magic number
+ mag.s = p - 32; // resulting shift
+ return mag;
+}
+
+/// magicu - calculate the magic numbers required to codegen an integer udiv as
+/// a sequence of multiply, add and shifts. Requires that the divisor not be 0.
+static mu magicu32(uint32_t d) {
+ int32_t p;
+ uint32_t nc, delta, q1, r1, q2, r2;
+ struct mu magu;
+ magu.a = 0; // initialize "add" indicator
+ nc = - 1 - (-d)%d;
+ p = 31; // initialize p
+ q1 = 0x80000000/nc; // initialize q1 = 2p/nc
+ r1 = 0x80000000 - q1*nc; // initialize r1 = rem(2p,nc)
+ q2 = 0x7FFFFFFF/d; // initialize q2 = (2p-1)/d
+ r2 = 0x7FFFFFFF - q2*d; // initialize r2 = rem((2p-1),d)
+ do {
+ p = p + 1;
+ if (r1 >= nc - r1 ) {
+ q1 = 2*q1 + 1; // update q1
+ r1 = 2*r1 - nc; // update r1
+ }
+ else {
+ q1 = 2*q1; // update q1
+ r1 = 2*r1; // update r1
+ }
+ if (r2 + 1 >= d - r2) {
+ if (q2 >= 0x7FFFFFFF) magu.a = 1;
+ q2 = 2*q2 + 1; // update q2
+ r2 = 2*r2 + 1 - d; // update r2
+ }
+ else {
+ if (q2 >= 0x80000000) magu.a = 1;
+ q2 = 2*q2; // update q2
+ r2 = 2*r2 + 1; // update r2
+ }
+ delta = d - 1 - r2;
+ } while (p < 64 && (q1 < delta || (q1 == delta && r1 == 0)));
+ magu.m = q2 + 1; // resulting magic number
+ magu.s = p - 32; // resulting shift
+ return magu;
+}
+
+/// magic - calculate the magic numbers required to codegen an integer sdiv as
+/// a sequence of multiply and shifts. Requires that the divisor not be 0, 1,
+/// or -1.
+static ms magic64(int64_t d) {
+ int64_t p;
+ uint64_t ad, anc, delta, q1, r1, q2, r2, t;
+ const uint64_t two63 = 9223372036854775808ULL; // 2^63
+ struct ms mag;
+
+ ad = d >= 0 ? d : -d;
+ t = two63 + ((uint64_t)d >> 63);
+ anc = t - 1 - t%ad; // absolute value of nc
+ p = 63; // initialize p
+ q1 = two63/anc; // initialize q1 = 2p/abs(nc)
+ r1 = two63 - q1*anc; // initialize r1 = rem(2p,abs(nc))
+ q2 = two63/ad; // initialize q2 = 2p/abs(d)
+ r2 = two63 - q2*ad; // initialize r2 = rem(2p,abs(d))
+ do {
+ p = p + 1;
+ q1 = 2*q1; // update q1 = 2p/abs(nc)
+ r1 = 2*r1; // update r1 = rem(2p/abs(nc))
+ if (r1 >= anc) { // must be unsigned comparison
+ q1 = q1 + 1;
+ r1 = r1 - anc;
+ }
+ q2 = 2*q2; // update q2 = 2p/abs(d)
+ r2 = 2*r2; // update r2 = rem(2p/abs(d))
+ if (r2 >= ad) { // must be unsigned comparison
+ q2 = q2 + 1;
+ r2 = r2 - ad;
+ }
+ delta = ad - r2;
+ } while (q1 < delta || (q1 == delta && r1 == 0));
+
+ mag.m = q2 + 1;
+ if (d < 0) mag.m = -mag.m; // resulting magic number
+ mag.s = p - 64; // resulting shift
+ return mag;
+}
+
+/// magicu - calculate the magic numbers required to codegen an integer udiv as
+/// a sequence of multiply, add and shifts. Requires that the divisor not be 0.
+static mu magicu64(uint64_t d)
+{
+ int64_t p;
+ uint64_t nc, delta, q1, r1, q2, r2;
+ struct mu magu;
+ magu.a = 0; // initialize "add" indicator
+ nc = - 1 - (-d)%d;
+ p = 63; // initialize p
+ q1 = 0x8000000000000000ull/nc; // initialize q1 = 2p/nc
+ r1 = 0x8000000000000000ull - q1*nc; // initialize r1 = rem(2p,nc)
+ q2 = 0x7FFFFFFFFFFFFFFFull/d; // initialize q2 = (2p-1)/d
+ r2 = 0x7FFFFFFFFFFFFFFFull - q2*d; // initialize r2 = rem((2p-1),d)
+ do {
+ p = p + 1;
+ if (r1 >= nc - r1 ) {
+ q1 = 2*q1 + 1; // update q1
+ r1 = 2*r1 - nc; // update r1
+ }
+ else {
+ q1 = 2*q1; // update q1
+ r1 = 2*r1; // update r1
+ }
+ if (r2 + 1 >= d - r2) {
+ if (q2 >= 0x7FFFFFFFFFFFFFFFull) magu.a = 1;
+ q2 = 2*q2 + 1; // update q2
+ r2 = 2*r2 + 1 - d; // update r2
+ }
+ else {
+ if (q2 >= 0x8000000000000000ull) magu.a = 1;
+ q2 = 2*q2; // update q2
+ r2 = 2*r2 + 1; // update r2
+ }
+ delta = d - 1 - r2;
+ } while (p < 128 && (q1 < delta || (q1 == delta && r1 == 0)));
+ magu.m = q2 + 1; // resulting magic number
+ magu.s = p - 64; // resulting shift
+ return magu;
+}
+
+/// BuildSDIVSequence - Given an ISD::SDIV node expressing a divide by constant,
+/// return a DAG expression to select that will generate the same value by
+/// multiplying by a magic number. See:
+/// <http://the.wall.riscom.net/books/proc/ppc/cwg/code2.html>
+SDOperand TargetLowering::BuildSDIV(SDNode *N, SelectionDAG &DAG,
+ std::vector<SDNode*>* Created) const {
+ MVT::ValueType VT = N->getValueType(0);
+
+ // Check to see if we can do this.
+ if (!isTypeLegal(VT) || (VT != MVT::i32 && VT != MVT::i64))
+ return SDOperand(); // BuildSDIV only operates on i32 or i64
+ if (!isOperationLegal(ISD::MULHS, VT))
+ return SDOperand(); // Make sure the target supports MULHS.
+
+ int64_t d = cast<ConstantSDNode>(N->getOperand(1))->getSignExtended();
+ ms magics = (VT == MVT::i32) ? magic32(d) : magic64(d);
+
+ // Multiply the numerator (operand 0) by the magic value
+ SDOperand Q = DAG.getNode(ISD::MULHS, VT, N->getOperand(0),
+ DAG.getConstant(magics.m, VT));
+ // If d > 0 and m < 0, add the numerator
+ if (d > 0 && magics.m < 0) {
+ Q = DAG.getNode(ISD::ADD, VT, Q, N->getOperand(0));
+ if (Created)
+ Created->push_back(Q.Val);
+ }
+ // If d < 0 and m > 0, subtract the numerator.
+ if (d < 0 && magics.m > 0) {
+ Q = DAG.getNode(ISD::SUB, VT, Q, N->getOperand(0));
+ if (Created)
+ Created->push_back(Q.Val);
+ }
+ // Shift right algebraic if shift value is nonzero
+ if (magics.s > 0) {
+ Q = DAG.getNode(ISD::SRA, VT, Q,
+ DAG.getConstant(magics.s, getShiftAmountTy()));
+ if (Created)
+ Created->push_back(Q.Val);
+ }
+ // Extract the sign bit and add it to the quotient
+ SDOperand T =
+ DAG.getNode(ISD::SRL, VT, Q, DAG.getConstant(MVT::getSizeInBits(VT)-1,
+ getShiftAmountTy()));
+ if (Created)
+ Created->push_back(T.Val);
+ return DAG.getNode(ISD::ADD, VT, Q, T);
+}
+
+/// BuildUDIVSequence - Given an ISD::UDIV node expressing a divide by constant,
+/// return a DAG expression to select that will generate the same value by
+/// multiplying by a magic number. See:
+/// <http://the.wall.riscom.net/books/proc/ppc/cwg/code2.html>
+SDOperand TargetLowering::BuildUDIV(SDNode *N, SelectionDAG &DAG,
+ std::vector<SDNode*>* Created) const {
+ MVT::ValueType VT = N->getValueType(0);
+
+ // Check to see if we can do this.
+ if (!isTypeLegal(VT) || (VT != MVT::i32 && VT != MVT::i64))
+ return SDOperand(); // BuildUDIV only operates on i32 or i64
+ if (!isOperationLegal(ISD::MULHU, VT))
+ return SDOperand(); // Make sure the target supports MULHU.
+
+ uint64_t d = cast<ConstantSDNode>(N->getOperand(1))->getValue();
+ mu magics = (VT == MVT::i32) ? magicu32(d) : magicu64(d);
+
+ // Multiply the numerator (operand 0) by the magic value
+ SDOperand Q = DAG.getNode(ISD::MULHU, VT, N->getOperand(0),
+ DAG.getConstant(magics.m, VT));
+ if (Created)
+ Created->push_back(Q.Val);
+
+ if (magics.a == 0) {
+ return DAG.getNode(ISD::SRL, VT, Q,
+ DAG.getConstant(magics.s, getShiftAmountTy()));
+ } else {
+ SDOperand NPQ = DAG.getNode(ISD::SUB, VT, N->getOperand(0), Q);
+ if (Created)
+ Created->push_back(NPQ.Val);
+ NPQ = DAG.getNode(ISD::SRL, VT, NPQ,
+ DAG.getConstant(1, getShiftAmountTy()));
+ if (Created)
+ Created->push_back(NPQ.Val);
+ NPQ = DAG.getNode(ISD::ADD, VT, NPQ, Q);
+ if (Created)
+ Created->push_back(NPQ.Val);
+ return DAG.getNode(ISD::SRL, VT, NPQ,
+ DAG.getConstant(magics.s-1, getShiftAmountTy()));
+ }
+}
diff --git a/lib/CodeGen/SimpleRegisterCoalescing.cpp b/lib/CodeGen/SimpleRegisterCoalescing.cpp
new file mode 100644
index 0000000..3d8618f
--- /dev/null
+++ b/lib/CodeGen/SimpleRegisterCoalescing.cpp
@@ -0,0 +1,1138 @@
+//===-- SimpleRegisterCoalescing.cpp - Register Coalescing ----------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file was developed by the LLVM research group and is distributed under
+// the University of Illinois Open Source License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements a simple register coalescing pass that attempts to
+// aggressively coalesce every register copy that it can.
+//
+//===----------------------------------------------------------------------===//
+
+#define DEBUG_TYPE "simpleregistercoalescing"
+#include "llvm/CodeGen/SimpleRegisterCoalescing.h"
+#include "llvm/CodeGen/LiveIntervalAnalysis.h"
+#include "VirtRegMap.h"
+#include "llvm/Value.h"
+#include "llvm/Analysis/LoopInfo.h"
+#include "llvm/CodeGen/LiveVariables.h"
+#include "llvm/CodeGen/MachineFrameInfo.h"
+#include "llvm/CodeGen/MachineInstr.h"
+#include "llvm/CodeGen/Passes.h"
+#include "llvm/CodeGen/SSARegMap.h"
+#include "llvm/Target/MRegisterInfo.h"
+#include "llvm/Target/TargetInstrInfo.h"
+#include "llvm/Target/TargetMachine.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/ADT/SmallSet.h"
+#include "llvm/ADT/Statistic.h"
+#include "llvm/ADT/STLExtras.h"
+#include <algorithm>
+#include <cmath>
+using namespace llvm;
+
+STATISTIC(numJoins , "Number of interval joins performed");
+STATISTIC(numPeep , "Number of identity moves eliminated after coalescing");
+STATISTIC(numAborts , "Number of times interval joining aborted");
+
+char SimpleRegisterCoalescing::ID = 0;
+namespace {
+ static cl::opt<bool>
+ EnableJoining("join-liveintervals",
+ cl::desc("Coalesce copies (default=true)"),
+ cl::init(true));
+
+ RegisterPass<SimpleRegisterCoalescing>
+ X("simple-register-coalescing",
+ "Simple register coalescing to eliminate all possible register copies");
+}
+
+const PassInfo *llvm::SimpleRegisterCoalescingID = X.getPassInfo();
+
+void SimpleRegisterCoalescing::getAnalysisUsage(AnalysisUsage &AU) const {
+ //AU.addPreserved<LiveVariables>();
+ AU.addPreserved<LiveIntervals>();
+ AU.addPreservedID(PHIEliminationID);
+ AU.addPreservedID(TwoAddressInstructionPassID);
+ AU.addRequired<LiveVariables>();
+ AU.addRequired<LiveIntervals>();
+ AU.addRequired<LoopInfo>();
+ MachineFunctionPass::getAnalysisUsage(AU);
+}
+
+/// AdjustCopiesBackFrom - We found a non-trivially-coalescable copy with IntA
+/// being the source and IntB being the dest, thus this defines a value number
+/// in IntB. If the source value number (in IntA) is defined by a copy from B,
+/// see if we can merge these two pieces of B into a single value number,
+/// eliminating a copy. For example:
+///
+/// A3 = B0
+/// ...
+/// B1 = A3 <- this copy
+///
+/// In this case, B0 can be extended to where the B1 copy lives, allowing the B1
+/// value number to be replaced with B0 (which simplifies the B liveinterval).
+///
+/// This returns true if an interval was modified.
+///
+bool SimpleRegisterCoalescing::AdjustCopiesBackFrom(LiveInterval &IntA, LiveInterval &IntB,
+ MachineInstr *CopyMI) {
+ unsigned CopyIdx = li_->getDefIndex(li_->getInstructionIndex(CopyMI));
+
+ // BValNo is a value number in B that is defined by a copy from A. 'B3' in
+ // the example above.
+ LiveInterval::iterator BLR = IntB.FindLiveRangeContaining(CopyIdx);
+ unsigned BValNo = BLR->ValId;
+
+ // Get the location that B is defined at. Two options: either this value has
+ // an unknown definition point or it is defined at CopyIdx. If unknown, we
+ // can't process it.
+ unsigned BValNoDefIdx = IntB.getInstForValNum(BValNo);
+ if (BValNoDefIdx == ~0U) return false;
+ assert(BValNoDefIdx == CopyIdx &&
+ "Copy doesn't define the value?");
+
+ // AValNo is the value number in A that defines the copy, A0 in the example.
+ LiveInterval::iterator AValLR = IntA.FindLiveRangeContaining(CopyIdx-1);
+ unsigned AValNo = AValLR->ValId;
+
+ // If AValNo is defined as a copy from IntB, we can potentially process this.
+
+ // Get the instruction that defines this value number.
+ unsigned SrcReg = IntA.getSrcRegForValNum(AValNo);
+ if (!SrcReg) return false; // Not defined by a copy.
+
+ // If the value number is not defined by a copy instruction, ignore it.
+
+ // If the source register comes from an interval other than IntB, we can't
+ // handle this.
+ if (rep(SrcReg) != IntB.reg) return false;
+
+ // Get the LiveRange in IntB that this value number starts with.
+ unsigned AValNoInstIdx = IntA.getInstForValNum(AValNo);
+ LiveInterval::iterator ValLR = IntB.FindLiveRangeContaining(AValNoInstIdx-1);
+
+ // Make sure that the end of the live range is inside the same block as
+ // CopyMI.
+ MachineInstr *ValLREndInst = li_->getInstructionFromIndex(ValLR->end-1);
+ if (!ValLREndInst ||
+ ValLREndInst->getParent() != CopyMI->getParent()) return false;
+
+ // Okay, we now know that ValLR ends in the same block that the CopyMI
+ // live-range starts. If there are no intervening live ranges between them in
+ // IntB, we can merge them.
+ if (ValLR+1 != BLR) return false;
+
+ DOUT << "\nExtending: "; IntB.print(DOUT, mri_);
+
+ // We are about to delete CopyMI, so need to remove it as the 'instruction
+ // that defines this value #'.
+ IntB.setValueNumberInfo(BValNo, std::make_pair(~0U, 0));
+
+ // Okay, we can merge them. We need to insert a new liverange:
+ // [ValLR.end, BLR.begin) of either value number, then we merge the
+ // two value numbers.
+ unsigned FillerStart = ValLR->end, FillerEnd = BLR->start;
+ IntB.addRange(LiveRange(FillerStart, FillerEnd, BValNo));
+
+ // If the IntB live range is assigned to a physical register, and if that
+ // physreg has aliases,
+ if (MRegisterInfo::isPhysicalRegister(IntB.reg)) {
+ // Update the liveintervals of sub-registers.
+ for (const unsigned *AS = mri_->getSubRegisters(IntB.reg); *AS; ++AS) {
+ LiveInterval &AliasLI = li_->getInterval(*AS);
+ AliasLI.addRange(LiveRange(FillerStart, FillerEnd,
+ AliasLI.getNextValue(~0U, 0)));
+ }
+ }
+
+ // Okay, merge "B1" into the same value number as "B0".
+ if (BValNo != ValLR->ValId)
+ IntB.MergeValueNumberInto(BValNo, ValLR->ValId);
+ DOUT << " result = "; IntB.print(DOUT, mri_);
+ DOUT << "\n";
+
+ // If the source instruction was killing the source register before the
+ // merge, unset the isKill marker given the live range has been extended.
+ int UIdx = ValLREndInst->findRegisterUseOperandIdx(IntB.reg, true);
+ if (UIdx != -1)
+ ValLREndInst->getOperand(UIdx).unsetIsKill();
+
+ // Finally, delete the copy instruction.
+ li_->RemoveMachineInstrFromMaps(CopyMI);
+ CopyMI->eraseFromParent();
+ ++numPeep;
+ return true;
+}
+
+/// JoinCopy - Attempt to join intervals corresponding to SrcReg/DstReg,
+/// which are the src/dst of the copy instruction CopyMI. This returns true
+/// if the copy was successfully coalesced away, or if it is never possible
+/// to coalesce this copy, due to register constraints. It returns
+/// false if it is not currently possible to coalesce this interval, but
+/// it may be possible if other things get coalesced.
+bool SimpleRegisterCoalescing::JoinCopy(MachineInstr *CopyMI,
+ unsigned SrcReg, unsigned DstReg, bool PhysOnly) {
+ DOUT << li_->getInstructionIndex(CopyMI) << '\t' << *CopyMI;
+
+ // Get representative registers.
+ unsigned repSrcReg = rep(SrcReg);
+ unsigned repDstReg = rep(DstReg);
+
+ // If they are already joined we continue.
+ if (repSrcReg == repDstReg) {
+ DOUT << "\tCopy already coalesced.\n";
+ return true; // Not coalescable.
+ }
+
+ bool SrcIsPhys = MRegisterInfo::isPhysicalRegister(repSrcReg);
+ bool DstIsPhys = MRegisterInfo::isPhysicalRegister(repDstReg);
+ if (PhysOnly && !SrcIsPhys && !DstIsPhys)
+ // Only joining physical registers with virtual registers in this round.
+ return true;
+
+ // If they are both physical registers, we cannot join them.
+ if (SrcIsPhys && DstIsPhys) {
+ DOUT << "\tCan not coalesce physregs.\n";
+ return true; // Not coalescable.
+ }
+
+ // We only join virtual registers with allocatable physical registers.
+ if (SrcIsPhys && !allocatableRegs_[repSrcReg]) {
+ DOUT << "\tSrc reg is unallocatable physreg.\n";
+ return true; // Not coalescable.
+ }
+ if (DstIsPhys && !allocatableRegs_[repDstReg]) {
+ DOUT << "\tDst reg is unallocatable physreg.\n";
+ return true; // Not coalescable.
+ }
+
+ // If they are not of the same register class, we cannot join them.
+ if (differingRegisterClasses(repSrcReg, repDstReg)) {
+ DOUT << "\tSrc/Dest are different register classes.\n";
+ return true; // Not coalescable.
+ }
+
+ LiveInterval &SrcInt = li_->getInterval(repSrcReg);
+ LiveInterval &DstInt = li_->getInterval(repDstReg);
+ assert(SrcInt.reg == repSrcReg && DstInt.reg == repDstReg &&
+ "Register mapping is horribly broken!");
+
+ DOUT << "\t\tInspecting "; SrcInt.print(DOUT, mri_);
+ DOUT << " and "; DstInt.print(DOUT, mri_);
+ DOUT << ": ";
+
+ // Check if it is necessary to propagate "isDead" property before intervals
+ // are joined.
+ MachineOperand *mopd = CopyMI->findRegisterDefOperand(DstReg);
+ bool isDead = mopd->isDead();
+ bool isShorten = false;
+ unsigned SrcStart = 0, RemoveStart = 0;
+ unsigned SrcEnd = 0, RemoveEnd = 0;
+ if (isDead) {
+ unsigned CopyIdx = li_->getInstructionIndex(CopyMI);
+ LiveInterval::iterator SrcLR =
+ SrcInt.FindLiveRangeContaining(li_->getUseIndex(CopyIdx));
+ RemoveStart = SrcStart = SrcLR->start;
+ RemoveEnd = SrcEnd = SrcLR->end;
+ // The instruction which defines the src is only truly dead if there are
+ // no intermediate uses and there isn't a use beyond the copy.
+ // FIXME: find the last use, mark is kill and shorten the live range.
+ if (SrcEnd > li_->getDefIndex(CopyIdx)) {
+ isDead = false;
+ } else {
+ MachineOperand *MOU;
+ MachineInstr *LastUse= lastRegisterUse(SrcStart, CopyIdx, repSrcReg, MOU);
+ if (LastUse) {
+ // Shorten the liveinterval to the end of last use.
+ MOU->setIsKill();
+ isDead = false;
+ isShorten = true;
+ RemoveStart = li_->getDefIndex(li_->getInstructionIndex(LastUse));
+ RemoveEnd = SrcEnd;
+ } else {
+ MachineInstr *SrcMI = li_->getInstructionFromIndex(SrcStart);
+ if (SrcMI) {
+ MachineOperand *mops = findDefOperand(SrcMI, repSrcReg);
+ if (mops)
+ // A dead def should have a single cycle interval.
+ ++RemoveStart;
+ }
+ }
+ }
+ }
+
+ // We need to be careful about coalescing a source physical register with a
+ // virtual register. Once the coalescing is done, it cannot be broken and
+ // these are not spillable! If the destination interval uses are far away,
+ // think twice about coalescing them!
+ if (!mopd->isDead() && (SrcIsPhys || DstIsPhys)) {
+ LiveInterval &JoinVInt = SrcIsPhys ? DstInt : SrcInt;
+ unsigned JoinVReg = SrcIsPhys ? repDstReg : repSrcReg;
+ unsigned JoinPReg = SrcIsPhys ? repSrcReg : repDstReg;
+ const TargetRegisterClass *RC = mf_->getSSARegMap()->getRegClass(JoinVReg);
+ unsigned Threshold = allocatableRCRegs_[RC].count();
+
+ // If the virtual register live interval is long has it has low use desity,
+ // do not join them, instead mark the physical register as its allocation
+ // preference.
+ unsigned Length = JoinVInt.getSize() / InstrSlots::NUM;
+ LiveVariables::VarInfo &vi = lv_->getVarInfo(JoinVReg);
+ if (Length > Threshold &&
+ (((float)vi.NumUses / Length) < (1.0 / Threshold))) {
+ JoinVInt.preference = JoinPReg;
+ ++numAborts;
+ DOUT << "\tMay tie down a physical register, abort!\n";
+ return false;
+ }
+ }
+
+ // Okay, attempt to join these two intervals. On failure, this returns false.
+ // Otherwise, if one of the intervals being joined is a physreg, this method
+ // always canonicalizes DstInt to be it. The output "SrcInt" will not have
+ // been modified, so we can use this information below to update aliases.
+ if (JoinIntervals(DstInt, SrcInt)) {
+ if (isDead) {
+ // Result of the copy is dead. Propagate this property.
+ if (SrcStart == 0) {
+ assert(MRegisterInfo::isPhysicalRegister(repSrcReg) &&
+ "Live-in must be a physical register!");
+ // Live-in to the function but dead. Remove it from entry live-in set.
+ // JoinIntervals may end up swapping the two intervals.
+ mf_->begin()->removeLiveIn(repSrcReg);
+ } else {
+ MachineInstr *SrcMI = li_->getInstructionFromIndex(SrcStart);
+ if (SrcMI) {
+ MachineOperand *mops = findDefOperand(SrcMI, repSrcReg);
+ if (mops)
+ mops->setIsDead();
+ }
+ }
+ }
+
+ if (isShorten || isDead) {
+ // Shorten the live interval.
+ LiveInterval &LiveInInt = (repSrcReg == DstInt.reg) ? DstInt : SrcInt;
+ LiveInInt.removeRange(RemoveStart, RemoveEnd);
+ }
+ } else {
+ // Coalescing failed.
+
+ // If we can eliminate the copy without merging the live ranges, do so now.
+ if (AdjustCopiesBackFrom(SrcInt, DstInt, CopyMI))
+ return true;
+
+ // Otherwise, we are unable to join the intervals.
+ DOUT << "Interference!\n";
+ return false;
+ }
+
+ bool Swapped = repSrcReg == DstInt.reg;
+ if (Swapped)
+ std::swap(repSrcReg, repDstReg);
+ assert(MRegisterInfo::isVirtualRegister(repSrcReg) &&
+ "LiveInterval::join didn't work right!");
+
+ // If we're about to merge live ranges into a physical register live range,
+ // we have to update any aliased register's live ranges to indicate that they
+ // have clobbered values for this range.
+ if (MRegisterInfo::isPhysicalRegister(repDstReg)) {
+ // Unset unnecessary kills.
+ if (!DstInt.containsOneValue()) {
+ for (LiveInterval::Ranges::const_iterator I = SrcInt.begin(),
+ E = SrcInt.end(); I != E; ++I)
+ unsetRegisterKills(I->start, I->end, repDstReg);
+ }
+
+ // Update the liveintervals of sub-registers.
+ for (const unsigned *AS = mri_->getSubRegisters(repDstReg); *AS; ++AS)
+ li_->getInterval(*AS).MergeInClobberRanges(SrcInt);
+ } else {
+ // Merge use info if the destination is a virtual register.
+ LiveVariables::VarInfo& dVI = lv_->getVarInfo(repDstReg);
+ LiveVariables::VarInfo& sVI = lv_->getVarInfo(repSrcReg);
+ dVI.NumUses += sVI.NumUses;
+ }
+
+ DOUT << "\n\t\tJoined. Result = "; DstInt.print(DOUT, mri_);
+ DOUT << "\n";
+
+ // Remember these liveintervals have been joined.
+ JoinedLIs.set(repSrcReg - MRegisterInfo::FirstVirtualRegister);
+ if (MRegisterInfo::isVirtualRegister(repDstReg))
+ JoinedLIs.set(repDstReg - MRegisterInfo::FirstVirtualRegister);
+
+ // If the intervals were swapped by Join, swap them back so that the register
+ // mapping (in the r2i map) is correct.
+ if (Swapped) SrcInt.swap(DstInt);
+ li_->removeInterval(repSrcReg);
+ r2rMap_[repSrcReg] = repDstReg;
+
+ // Finally, delete the copy instruction.
+ li_->RemoveMachineInstrFromMaps(CopyMI);
+ CopyMI->eraseFromParent();
+ ++numPeep;
+ ++numJoins;
+ return true;
+}
+
+/// ComputeUltimateVN - Assuming we are going to join two live intervals,
+/// compute what the resultant value numbers for each value in the input two
+/// ranges will be. This is complicated by copies between the two which can
+/// and will commonly cause multiple value numbers to be merged into one.
+///
+/// VN is the value number that we're trying to resolve. InstDefiningValue
+/// keeps track of the new InstDefiningValue assignment for the result
+/// LiveInterval. ThisFromOther/OtherFromThis are sets that keep track of
+/// whether a value in this or other is a copy from the opposite set.
+/// ThisValNoAssignments/OtherValNoAssignments keep track of value #'s that have
+/// already been assigned.
+///
+/// ThisFromOther[x] - If x is defined as a copy from the other interval, this
+/// contains the value number the copy is from.
+///
+static unsigned ComputeUltimateVN(unsigned VN,
+ SmallVector<std::pair<unsigned,
+ unsigned>, 16> &ValueNumberInfo,
+ SmallVector<int, 16> &ThisFromOther,
+ SmallVector<int, 16> &OtherFromThis,
+ SmallVector<int, 16> &ThisValNoAssignments,
+ SmallVector<int, 16> &OtherValNoAssignments,
+ LiveInterval &ThisLI, LiveInterval &OtherLI) {
+ // If the VN has already been computed, just return it.
+ if (ThisValNoAssignments[VN] >= 0)
+ return ThisValNoAssignments[VN];
+// assert(ThisValNoAssignments[VN] != -2 && "Cyclic case?");
+
+ // If this val is not a copy from the other val, then it must be a new value
+ // number in the destination.
+ int OtherValNo = ThisFromOther[VN];
+ if (OtherValNo == -1) {
+ ValueNumberInfo.push_back(ThisLI.getValNumInfo(VN));
+ return ThisValNoAssignments[VN] = ValueNumberInfo.size()-1;
+ }
+
+ // Otherwise, this *is* a copy from the RHS. If the other side has already
+ // been computed, return it.
+ if (OtherValNoAssignments[OtherValNo] >= 0)
+ return ThisValNoAssignments[VN] = OtherValNoAssignments[OtherValNo];
+
+ // Mark this value number as currently being computed, then ask what the
+ // ultimate value # of the other value is.
+ ThisValNoAssignments[VN] = -2;
+ unsigned UltimateVN =
+ ComputeUltimateVN(OtherValNo, ValueNumberInfo,
+ OtherFromThis, ThisFromOther,
+ OtherValNoAssignments, ThisValNoAssignments,
+ OtherLI, ThisLI);
+ return ThisValNoAssignments[VN] = UltimateVN;
+}
+
+static bool InVector(unsigned Val, const SmallVector<unsigned, 8> &V) {
+ return std::find(V.begin(), V.end(), Val) != V.end();
+}
+
+/// SimpleJoin - Attempt to joint the specified interval into this one. The
+/// caller of this method must guarantee that the RHS only contains a single
+/// value number and that the RHS is not defined by a copy from this
+/// interval. This returns false if the intervals are not joinable, or it
+/// joins them and returns true.
+bool SimpleRegisterCoalescing::SimpleJoin(LiveInterval &LHS, LiveInterval &RHS) {
+ assert(RHS.containsOneValue());
+
+ // Some number (potentially more than one) value numbers in the current
+ // interval may be defined as copies from the RHS. Scan the overlapping
+ // portions of the LHS and RHS, keeping track of this and looking for
+ // overlapping live ranges that are NOT defined as copies. If these exist, we
+ // cannot coalesce.
+
+ LiveInterval::iterator LHSIt = LHS.begin(), LHSEnd = LHS.end();
+ LiveInterval::iterator RHSIt = RHS.begin(), RHSEnd = RHS.end();
+
+ if (LHSIt->start < RHSIt->start) {
+ LHSIt = std::upper_bound(LHSIt, LHSEnd, RHSIt->start);
+ if (LHSIt != LHS.begin()) --LHSIt;
+ } else if (RHSIt->start < LHSIt->start) {
+ RHSIt = std::upper_bound(RHSIt, RHSEnd, LHSIt->start);
+ if (RHSIt != RHS.begin()) --RHSIt;
+ }
+
+ SmallVector<unsigned, 8> EliminatedLHSVals;
+
+ while (1) {
+ // Determine if these live intervals overlap.
+ bool Overlaps = false;
+ if (LHSIt->start <= RHSIt->start)
+ Overlaps = LHSIt->end > RHSIt->start;
+ else
+ Overlaps = RHSIt->end > LHSIt->start;
+
+ // If the live intervals overlap, there are two interesting cases: if the
+ // LHS interval is defined by a copy from the RHS, it's ok and we record
+ // that the LHS value # is the same as the RHS. If it's not, then we cannot
+ // coalesce these live ranges and we bail out.
+ if (Overlaps) {
+ // If we haven't already recorded that this value # is safe, check it.
+ if (!InVector(LHSIt->ValId, EliminatedLHSVals)) {
+ // Copy from the RHS?
+ unsigned SrcReg = LHS.getSrcRegForValNum(LHSIt->ValId);
+ if (rep(SrcReg) != RHS.reg)
+ return false; // Nope, bail out.
+
+ EliminatedLHSVals.push_back(LHSIt->ValId);
+ }
+
+ // We know this entire LHS live range is okay, so skip it now.
+ if (++LHSIt == LHSEnd) break;
+ continue;
+ }
+
+ if (LHSIt->end < RHSIt->end) {
+ if (++LHSIt == LHSEnd) break;
+ } else {
+ // One interesting case to check here. It's possible that we have
+ // something like "X3 = Y" which defines a new value number in the LHS,
+ // and is the last use of this liverange of the RHS. In this case, we
+ // want to notice this copy (so that it gets coalesced away) even though
+ // the live ranges don't actually overlap.
+ if (LHSIt->start == RHSIt->end) {
+ if (InVector(LHSIt->ValId, EliminatedLHSVals)) {
+ // We already know that this value number is going to be merged in
+ // if coalescing succeeds. Just skip the liverange.
+ if (++LHSIt == LHSEnd) break;
+ } else {
+ // Otherwise, if this is a copy from the RHS, mark it as being merged
+ // in.
+ if (rep(LHS.getSrcRegForValNum(LHSIt->ValId)) == RHS.reg) {
+ EliminatedLHSVals.push_back(LHSIt->ValId);
+
+ // We know this entire LHS live range is okay, so skip it now.
+ if (++LHSIt == LHSEnd) break;
+ }
+ }
+ }
+
+ if (++RHSIt == RHSEnd) break;
+ }
+ }
+
+ // If we got here, we know that the coalescing will be successful and that
+ // the value numbers in EliminatedLHSVals will all be merged together. Since
+ // the most common case is that EliminatedLHSVals has a single number, we
+ // optimize for it: if there is more than one value, we merge them all into
+ // the lowest numbered one, then handle the interval as if we were merging
+ // with one value number.
+ unsigned LHSValNo;
+ if (EliminatedLHSVals.size() > 1) {
+ // Loop through all the equal value numbers merging them into the smallest
+ // one.
+ unsigned Smallest = EliminatedLHSVals[0];
+ for (unsigned i = 1, e = EliminatedLHSVals.size(); i != e; ++i) {
+ if (EliminatedLHSVals[i] < Smallest) {
+ // Merge the current notion of the smallest into the smaller one.
+ LHS.MergeValueNumberInto(Smallest, EliminatedLHSVals[i]);
+ Smallest = EliminatedLHSVals[i];
+ } else {
+ // Merge into the smallest.
+ LHS.MergeValueNumberInto(EliminatedLHSVals[i], Smallest);
+ }
+ }
+ LHSValNo = Smallest;
+ } else {
+ assert(!EliminatedLHSVals.empty() && "No copies from the RHS?");
+ LHSValNo = EliminatedLHSVals[0];
+ }
+
+ // Okay, now that there is a single LHS value number that we're merging the
+ // RHS into, update the value number info for the LHS to indicate that the
+ // value number is defined where the RHS value number was.
+ LHS.setValueNumberInfo(LHSValNo, RHS.getValNumInfo(0));
+
+ // Okay, the final step is to loop over the RHS live intervals, adding them to
+ // the LHS.
+ LHS.MergeRangesInAsValue(RHS, LHSValNo);
+ LHS.weight += RHS.weight;
+ if (RHS.preference && !LHS.preference)
+ LHS.preference = RHS.preference;
+
+ return true;
+}
+
+/// JoinIntervals - Attempt to join these two intervals. On failure, this
+/// returns false. Otherwise, if one of the intervals being joined is a
+/// physreg, this method always canonicalizes LHS to be it. The output
+/// "RHS" will not have been modified, so we can use this information
+/// below to update aliases.
+bool SimpleRegisterCoalescing::JoinIntervals(LiveInterval &LHS, LiveInterval &RHS) {
+ // Compute the final value assignment, assuming that the live ranges can be
+ // coalesced.
+ SmallVector<int, 16> LHSValNoAssignments;
+ SmallVector<int, 16> RHSValNoAssignments;
+ SmallVector<std::pair<unsigned,unsigned>, 16> ValueNumberInfo;
+
+ // If a live interval is a physical register, conservatively check if any
+ // of its sub-registers is overlapping the live interval of the virtual
+ // register. If so, do not coalesce.
+ if (MRegisterInfo::isPhysicalRegister(LHS.reg) &&
+ *mri_->getSubRegisters(LHS.reg)) {
+ for (const unsigned* SR = mri_->getSubRegisters(LHS.reg); *SR; ++SR)
+ if (li_->hasInterval(*SR) && RHS.overlaps(li_->getInterval(*SR))) {
+ DOUT << "Interfere with sub-register ";
+ DEBUG(li_->getInterval(*SR).print(DOUT, mri_));
+ return false;
+ }
+ } else if (MRegisterInfo::isPhysicalRegister(RHS.reg) &&
+ *mri_->getSubRegisters(RHS.reg)) {
+ for (const unsigned* SR = mri_->getSubRegisters(RHS.reg); *SR; ++SR)
+ if (li_->hasInterval(*SR) && LHS.overlaps(li_->getInterval(*SR))) {
+ DOUT << "Interfere with sub-register ";
+ DEBUG(li_->getInterval(*SR).print(DOUT, mri_));
+ return false;
+ }
+ }
+
+ // Compute ultimate value numbers for the LHS and RHS values.
+ if (RHS.containsOneValue()) {
+ // Copies from a liveinterval with a single value are simple to handle and
+ // very common, handle the special case here. This is important, because
+ // often RHS is small and LHS is large (e.g. a physreg).
+
+ // Find out if the RHS is defined as a copy from some value in the LHS.
+ int RHSValID = -1;
+ std::pair<unsigned,unsigned> RHSValNoInfo;
+ unsigned RHSSrcReg = RHS.getSrcRegForValNum(0);
+ if ((RHSSrcReg == 0 || rep(RHSSrcReg) != LHS.reg)) {
+ // If RHS is not defined as a copy from the LHS, we can use simpler and
+ // faster checks to see if the live ranges are coalescable. This joiner
+ // can't swap the LHS/RHS intervals though.
+ if (!MRegisterInfo::isPhysicalRegister(RHS.reg)) {
+ return SimpleJoin(LHS, RHS);
+ } else {
+ RHSValNoInfo = RHS.getValNumInfo(0);
+ }
+ } else {
+ // It was defined as a copy from the LHS, find out what value # it is.
+ unsigned ValInst = RHS.getInstForValNum(0);
+ RHSValID = LHS.getLiveRangeContaining(ValInst-1)->ValId;
+ RHSValNoInfo = LHS.getValNumInfo(RHSValID);
+ }
+
+ LHSValNoAssignments.resize(LHS.getNumValNums(), -1);
+ RHSValNoAssignments.resize(RHS.getNumValNums(), -1);
+ ValueNumberInfo.resize(LHS.getNumValNums());
+
+ // Okay, *all* of the values in LHS that are defined as a copy from RHS
+ // should now get updated.
+ for (unsigned VN = 0, e = LHS.getNumValNums(); VN != e; ++VN) {
+ if (unsigned LHSSrcReg = LHS.getSrcRegForValNum(VN)) {
+ if (rep(LHSSrcReg) != RHS.reg) {
+ // If this is not a copy from the RHS, its value number will be
+ // unmodified by the coalescing.
+ ValueNumberInfo[VN] = LHS.getValNumInfo(VN);
+ LHSValNoAssignments[VN] = VN;
+ } else if (RHSValID == -1) {
+ // Otherwise, it is a copy from the RHS, and we don't already have a
+ // value# for it. Keep the current value number, but remember it.
+ LHSValNoAssignments[VN] = RHSValID = VN;
+ ValueNumberInfo[VN] = RHSValNoInfo;
+ } else {
+ // Otherwise, use the specified value #.
+ LHSValNoAssignments[VN] = RHSValID;
+ if (VN != (unsigned)RHSValID)
+ ValueNumberInfo[VN].first = ~1U;
+ else
+ ValueNumberInfo[VN] = RHSValNoInfo;
+ }
+ } else {
+ ValueNumberInfo[VN] = LHS.getValNumInfo(VN);
+ LHSValNoAssignments[VN] = VN;
+ }
+ }
+
+ assert(RHSValID != -1 && "Didn't find value #?");
+ RHSValNoAssignments[0] = RHSValID;
+
+ } else {
+ // Loop over the value numbers of the LHS, seeing if any are defined from
+ // the RHS.
+ SmallVector<int, 16> LHSValsDefinedFromRHS;
+ LHSValsDefinedFromRHS.resize(LHS.getNumValNums(), -1);
+ for (unsigned VN = 0, e = LHS.getNumValNums(); VN != e; ++VN) {
+ unsigned ValSrcReg = LHS.getSrcRegForValNum(VN);
+ if (ValSrcReg == 0) // Src not defined by a copy?
+ continue;
+
+ // DstReg is known to be a register in the LHS interval. If the src is
+ // from the RHS interval, we can use its value #.
+ if (rep(ValSrcReg) != RHS.reg)
+ continue;
+
+ // Figure out the value # from the RHS.
+ unsigned ValInst = LHS.getInstForValNum(VN);
+ LHSValsDefinedFromRHS[VN] = RHS.getLiveRangeContaining(ValInst-1)->ValId;
+ }
+
+ // Loop over the value numbers of the RHS, seeing if any are defined from
+ // the LHS.
+ SmallVector<int, 16> RHSValsDefinedFromLHS;
+ RHSValsDefinedFromLHS.resize(RHS.getNumValNums(), -1);
+ for (unsigned VN = 0, e = RHS.getNumValNums(); VN != e; ++VN) {
+ unsigned ValSrcReg = RHS.getSrcRegForValNum(VN);
+ if (ValSrcReg == 0) // Src not defined by a copy?
+ continue;
+
+ // DstReg is known to be a register in the RHS interval. If the src is
+ // from the LHS interval, we can use its value #.
+ if (rep(ValSrcReg) != LHS.reg)
+ continue;
+
+ // Figure out the value # from the LHS.
+ unsigned ValInst = RHS.getInstForValNum(VN);
+ RHSValsDefinedFromLHS[VN] = LHS.getLiveRangeContaining(ValInst-1)->ValId;
+ }
+
+ LHSValNoAssignments.resize(LHS.getNumValNums(), -1);
+ RHSValNoAssignments.resize(RHS.getNumValNums(), -1);
+ ValueNumberInfo.reserve(LHS.getNumValNums() + RHS.getNumValNums());
+
+ for (unsigned VN = 0, e = LHS.getNumValNums(); VN != e; ++VN) {
+ if (LHSValNoAssignments[VN] >= 0 || LHS.getInstForValNum(VN) == ~2U)
+ continue;
+ ComputeUltimateVN(VN, ValueNumberInfo,
+ LHSValsDefinedFromRHS, RHSValsDefinedFromLHS,
+ LHSValNoAssignments, RHSValNoAssignments, LHS, RHS);
+ }
+ for (unsigned VN = 0, e = RHS.getNumValNums(); VN != e; ++VN) {
+ if (RHSValNoAssignments[VN] >= 0 || RHS.getInstForValNum(VN) == ~2U)
+ continue;
+ // If this value number isn't a copy from the LHS, it's a new number.
+ if (RHSValsDefinedFromLHS[VN] == -1) {
+ ValueNumberInfo.push_back(RHS.getValNumInfo(VN));
+ RHSValNoAssignments[VN] = ValueNumberInfo.size()-1;
+ continue;
+ }
+
+ ComputeUltimateVN(VN, ValueNumberInfo,
+ RHSValsDefinedFromLHS, LHSValsDefinedFromRHS,
+ RHSValNoAssignments, LHSValNoAssignments, RHS, LHS);
+ }
+ }
+
+ // Armed with the mappings of LHS/RHS values to ultimate values, walk the
+ // interval lists to see if these intervals are coalescable.
+ LiveInterval::const_iterator I = LHS.begin();
+ LiveInterval::const_iterator IE = LHS.end();
+ LiveInterval::const_iterator J = RHS.begin();
+ LiveInterval::const_iterator JE = RHS.end();
+
+ // Skip ahead until the first place of potential sharing.
+ if (I->start < J->start) {
+ I = std::upper_bound(I, IE, J->start);
+ if (I != LHS.begin()) --I;
+ } else if (J->start < I->start) {
+ J = std::upper_bound(J, JE, I->start);
+ if (J != RHS.begin()) --J;
+ }
+
+ while (1) {
+ // Determine if these two live ranges overlap.
+ bool Overlaps;
+ if (I->start < J->start) {
+ Overlaps = I->end > J->start;
+ } else {
+ Overlaps = J->end > I->start;
+ }
+
+ // If so, check value # info to determine if they are really different.
+ if (Overlaps) {
+ // If the live range overlap will map to the same value number in the
+ // result liverange, we can still coalesce them. If not, we can't.
+ if (LHSValNoAssignments[I->ValId] != RHSValNoAssignments[J->ValId])
+ return false;
+ }
+
+ if (I->end < J->end) {
+ ++I;
+ if (I == IE) break;
+ } else {
+ ++J;
+ if (J == JE) break;
+ }
+ }
+
+ // If we get here, we know that we can coalesce the live ranges. Ask the
+ // intervals to coalesce themselves now.
+ LHS.join(RHS, &LHSValNoAssignments[0], &RHSValNoAssignments[0],
+ ValueNumberInfo);
+ return true;
+}
+
+namespace {
+ // DepthMBBCompare - Comparison predicate that sort first based on the loop
+ // depth of the basic block (the unsigned), and then on the MBB number.
+ struct DepthMBBCompare {
+ typedef std::pair<unsigned, MachineBasicBlock*> DepthMBBPair;
+ bool operator()(const DepthMBBPair &LHS, const DepthMBBPair &RHS) const {
+ if (LHS.first > RHS.first) return true; // Deeper loops first
+ return LHS.first == RHS.first &&
+ LHS.second->getNumber() < RHS.second->getNumber();
+ }
+ };
+}
+
+void SimpleRegisterCoalescing::CopyCoalesceInMBB(MachineBasicBlock *MBB,
+ std::vector<CopyRec> *TryAgain, bool PhysOnly) {
+ DOUT << ((Value*)MBB->getBasicBlock())->getName() << ":\n";
+
+ for (MachineBasicBlock::iterator MII = MBB->begin(), E = MBB->end();
+ MII != E;) {
+ MachineInstr *Inst = MII++;
+
+ // If this isn't a copy, we can't join intervals.
+ unsigned SrcReg, DstReg;
+ if (!tii_->isMoveInstr(*Inst, SrcReg, DstReg)) continue;
+
+ if (TryAgain && !JoinCopy(Inst, SrcReg, DstReg, PhysOnly))
+ TryAgain->push_back(getCopyRec(Inst, SrcReg, DstReg));
+ }
+}
+
+void SimpleRegisterCoalescing::joinIntervals() {
+ DOUT << "********** JOINING INTERVALS ***********\n";
+
+ JoinedLIs.resize(li_->getNumIntervals());
+ JoinedLIs.reset();
+
+ std::vector<CopyRec> TryAgainList;
+ const LoopInfo &LI = getAnalysis<LoopInfo>();
+ if (LI.begin() == LI.end()) {
+ // If there are no loops in the function, join intervals in function order.
+ for (MachineFunction::iterator I = mf_->begin(), E = mf_->end();
+ I != E; ++I)
+ CopyCoalesceInMBB(I, &TryAgainList);
+ } else {
+ // Otherwise, join intervals in inner loops before other intervals.
+ // Unfortunately we can't just iterate over loop hierarchy here because
+ // there may be more MBB's than BB's. Collect MBB's for sorting.
+
+ // Join intervals in the function prolog first. We want to join physical
+ // registers with virtual registers before the intervals got too long.
+ std::vector<std::pair<unsigned, MachineBasicBlock*> > MBBs;
+ for (MachineFunction::iterator I = mf_->begin(), E = mf_->end(); I != E;++I)
+ MBBs.push_back(std::make_pair(LI.getLoopDepth(I->getBasicBlock()), I));
+
+ // Sort by loop depth.
+ std::sort(MBBs.begin(), MBBs.end(), DepthMBBCompare());
+
+ // Finally, join intervals in loop nest order.
+ for (unsigned i = 0, e = MBBs.size(); i != e; ++i)
+ CopyCoalesceInMBB(MBBs[i].second, NULL, true);
+ for (unsigned i = 0, e = MBBs.size(); i != e; ++i)
+ CopyCoalesceInMBB(MBBs[i].second, &TryAgainList, false);
+ }
+
+ // Joining intervals can allow other intervals to be joined. Iteratively join
+ // until we make no progress.
+ bool ProgressMade = true;
+ while (ProgressMade) {
+ ProgressMade = false;
+
+ for (unsigned i = 0, e = TryAgainList.size(); i != e; ++i) {
+ CopyRec &TheCopy = TryAgainList[i];
+ if (TheCopy.MI &&
+ JoinCopy(TheCopy.MI, TheCopy.SrcReg, TheCopy.DstReg)) {
+ TheCopy.MI = 0; // Mark this one as done.
+ ProgressMade = true;
+ }
+ }
+ }
+
+ // Some live range has been lengthened due to colaescing, eliminate the
+ // unnecessary kills.
+ int RegNum = JoinedLIs.find_first();
+ while (RegNum != -1) {
+ unsigned Reg = RegNum + MRegisterInfo::FirstVirtualRegister;
+ unsigned repReg = rep(Reg);
+ LiveInterval &LI = li_->getInterval(repReg);
+ LiveVariables::VarInfo& svi = lv_->getVarInfo(Reg);
+ for (unsigned i = 0, e = svi.Kills.size(); i != e; ++i) {
+ MachineInstr *Kill = svi.Kills[i];
+ // Suppose vr1 = op vr2, x
+ // and vr1 and vr2 are coalesced. vr2 should still be marked kill
+ // unless it is a two-address operand.
+ if (li_->isRemoved(Kill) || hasRegisterDef(Kill, repReg))
+ continue;
+ if (LI.liveAt(li_->getInstructionIndex(Kill) + InstrSlots::NUM))
+ unsetRegisterKill(Kill, repReg);
+ }
+ RegNum = JoinedLIs.find_next(RegNum);
+ }
+
+ DOUT << "*** Register mapping ***\n";
+ for (int i = 0, e = r2rMap_.size(); i != e; ++i)
+ if (r2rMap_[i]) {
+ DOUT << " reg " << i << " -> ";
+ DEBUG(printRegName(r2rMap_[i]));
+ DOUT << "\n";
+ }
+}
+
+/// Return true if the two specified registers belong to different register
+/// classes. The registers may be either phys or virt regs.
+bool SimpleRegisterCoalescing::differingRegisterClasses(unsigned RegA,
+ unsigned RegB) const {
+
+ // Get the register classes for the first reg.
+ if (MRegisterInfo::isPhysicalRegister(RegA)) {
+ assert(MRegisterInfo::isVirtualRegister(RegB) &&
+ "Shouldn't consider two physregs!");
+ return !mf_->getSSARegMap()->getRegClass(RegB)->contains(RegA);
+ }
+
+ // Compare against the regclass for the second reg.
+ const TargetRegisterClass *RegClass = mf_->getSSARegMap()->getRegClass(RegA);
+ if (MRegisterInfo::isVirtualRegister(RegB))
+ return RegClass != mf_->getSSARegMap()->getRegClass(RegB);
+ else
+ return !RegClass->contains(RegB);
+}
+
+/// lastRegisterUse - Returns the last use of the specific register between
+/// cycles Start and End. It also returns the use operand by reference. It
+/// returns NULL if there are no uses.
+MachineInstr *
+SimpleRegisterCoalescing::lastRegisterUse(unsigned Start, unsigned End, unsigned Reg,
+ MachineOperand *&MOU) {
+ int e = (End-1) / InstrSlots::NUM * InstrSlots::NUM;
+ int s = Start;
+ while (e >= s) {
+ // Skip deleted instructions
+ MachineInstr *MI = li_->getInstructionFromIndex(e);
+ while ((e - InstrSlots::NUM) >= s && !MI) {
+ e -= InstrSlots::NUM;
+ MI = li_->getInstructionFromIndex(e);
+ }
+ if (e < s || MI == NULL)
+ return NULL;
+
+ for (unsigned i = 0, NumOps = MI->getNumOperands(); i != NumOps; ++i) {
+ MachineOperand &MO = MI->getOperand(i);
+ if (MO.isReg() && MO.isUse() && MO.getReg() &&
+ mri_->regsOverlap(rep(MO.getReg()), Reg)) {
+ MOU = &MO;
+ return MI;
+ }
+ }
+
+ e -= InstrSlots::NUM;
+ }
+
+ return NULL;
+}
+
+
+/// findDefOperand - Returns the MachineOperand that is a def of the specific
+/// register. It returns NULL if the def is not found.
+MachineOperand *SimpleRegisterCoalescing::findDefOperand(MachineInstr *MI, unsigned Reg) {
+ for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
+ MachineOperand &MO = MI->getOperand(i);
+ if (MO.isReg() && MO.isDef() &&
+ mri_->regsOverlap(rep(MO.getReg()), Reg))
+ return &MO;
+ }
+ return NULL;
+}
+
+/// unsetRegisterKill - Unset IsKill property of all uses of specific register
+/// of the specific instruction.
+void SimpleRegisterCoalescing::unsetRegisterKill(MachineInstr *MI, unsigned Reg) {
+ for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
+ MachineOperand &MO = MI->getOperand(i);
+ if (MO.isReg() && MO.isUse() && MO.isKill() && MO.getReg() &&
+ mri_->regsOverlap(rep(MO.getReg()), Reg))
+ MO.unsetIsKill();
+ }
+}
+
+/// unsetRegisterKills - Unset IsKill property of all uses of specific register
+/// between cycles Start and End.
+void SimpleRegisterCoalescing::unsetRegisterKills(unsigned Start, unsigned End,
+ unsigned Reg) {
+ int e = (End-1) / InstrSlots::NUM * InstrSlots::NUM;
+ int s = Start;
+ while (e >= s) {
+ // Skip deleted instructions
+ MachineInstr *MI = li_->getInstructionFromIndex(e);
+ while ((e - InstrSlots::NUM) >= s && !MI) {
+ e -= InstrSlots::NUM;
+ MI = li_->getInstructionFromIndex(e);
+ }
+ if (e < s || MI == NULL)
+ return;
+
+ for (unsigned i = 0, NumOps = MI->getNumOperands(); i != NumOps; ++i) {
+ MachineOperand &MO = MI->getOperand(i);
+ if (MO.isReg() && MO.isUse() && MO.isKill() && MO.getReg() &&
+ mri_->regsOverlap(rep(MO.getReg()), Reg)) {
+ MO.unsetIsKill();
+ }
+ }
+
+ e -= InstrSlots::NUM;
+ }
+}
+
+/// hasRegisterDef - True if the instruction defines the specific register.
+///
+bool SimpleRegisterCoalescing::hasRegisterDef(MachineInstr *MI, unsigned Reg) {
+ for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
+ MachineOperand &MO = MI->getOperand(i);
+ if (MO.isReg() && MO.isDef() &&
+ mri_->regsOverlap(rep(MO.getReg()), Reg))
+ return true;
+ }
+ return false;
+}
+
+void SimpleRegisterCoalescing::printRegName(unsigned reg) const {
+ if (MRegisterInfo::isPhysicalRegister(reg))
+ cerr << mri_->getName(reg);
+ else
+ cerr << "%reg" << reg;
+}
+
+void SimpleRegisterCoalescing::releaseMemory() {
+ r2rMap_.clear();
+ JoinedLIs.clear();
+}
+
+static bool isZeroLengthInterval(LiveInterval *li) {
+ for (LiveInterval::Ranges::const_iterator
+ i = li->ranges.begin(), e = li->ranges.end(); i != e; ++i)
+ if (i->end - i->start > LiveIntervals::InstrSlots::NUM)
+ return false;
+ return true;
+}
+
+bool SimpleRegisterCoalescing::runOnMachineFunction(MachineFunction &fn) {
+ mf_ = &fn;
+ tm_ = &fn.getTarget();
+ mri_ = tm_->getRegisterInfo();
+ tii_ = tm_->getInstrInfo();
+ li_ = &getAnalysis<LiveIntervals>();
+ lv_ = &getAnalysis<LiveVariables>();
+
+ DOUT << "********** SIMPLE REGISTER COALESCING **********\n"
+ << "********** Function: "
+ << ((Value*)mf_->getFunction())->getName() << '\n';
+
+ allocatableRegs_ = mri_->getAllocatableSet(fn);
+ for (MRegisterInfo::regclass_iterator I = mri_->regclass_begin(),
+ E = mri_->regclass_end(); I != E; ++I)
+ allocatableRCRegs_.insert(std::make_pair(*I,mri_->getAllocatableSet(fn, *I)));
+
+ r2rMap_.grow(mf_->getSSARegMap()->getLastVirtReg());
+
+ // Join (coalesce) intervals if requested.
+ if (EnableJoining) {
+ joinIntervals();
+ DOUT << "********** INTERVALS POST JOINING **********\n";
+ for (LiveIntervals::iterator I = li_->begin(), E = li_->end(); I != E; ++I) {
+ I->second.print(DOUT, mri_);
+ DOUT << "\n";
+ }
+ }
+
+ // perform a final pass over the instructions and compute spill
+ // weights, coalesce virtual registers and remove identity moves.
+ const LoopInfo &loopInfo = getAnalysis<LoopInfo>();
+
+ for (MachineFunction::iterator mbbi = mf_->begin(), mbbe = mf_->end();
+ mbbi != mbbe; ++mbbi) {
+ MachineBasicBlock* mbb = mbbi;
+ unsigned loopDepth = loopInfo.getLoopDepth(mbb->getBasicBlock());
+
+ for (MachineBasicBlock::iterator mii = mbb->begin(), mie = mbb->end();
+ mii != mie; ) {
+ // if the move will be an identity move delete it
+ unsigned srcReg, dstReg, RegRep;
+ if (tii_->isMoveInstr(*mii, srcReg, dstReg) &&
+ (RegRep = rep(srcReg)) == rep(dstReg)) {
+ // remove from def list
+ LiveInterval &RegInt = li_->getOrCreateInterval(RegRep);
+ MachineOperand *MO = mii->findRegisterDefOperand(dstReg);
+ // If def of this move instruction is dead, remove its live range from
+ // the dstination register's live interval.
+ if (MO->isDead()) {
+ unsigned MoveIdx = li_->getDefIndex(li_->getInstructionIndex(mii));
+ LiveInterval::iterator MLR = RegInt.FindLiveRangeContaining(MoveIdx);
+ RegInt.removeRange(MLR->start, MoveIdx+1);
+ if (RegInt.empty())
+ li_->removeInterval(RegRep);
+ }
+ li_->RemoveMachineInstrFromMaps(mii);
+ mii = mbbi->erase(mii);
+ ++numPeep;
+ } else {
+ SmallSet<unsigned, 4> UniqueUses;
+ for (unsigned i = 0, e = mii->getNumOperands(); i != e; ++i) {
+ const MachineOperand &mop = mii->getOperand(i);
+ if (mop.isRegister() && mop.getReg() &&
+ MRegisterInfo::isVirtualRegister(mop.getReg())) {
+ // replace register with representative register
+ unsigned reg = rep(mop.getReg());
+ mii->getOperand(i).setReg(reg);
+
+ // Multiple uses of reg by the same instruction. It should not
+ // contribute to spill weight again.
+ if (UniqueUses.count(reg) != 0)
+ continue;
+ LiveInterval &RegInt = li_->getInterval(reg);
+ float w = (mop.isUse()+mop.isDef()) * powf(10.0F, (float)loopDepth);
+ // If the definition instruction is re-materializable, its spill
+ // weight is half of what it would have been normally unless it's
+ // a load from fixed stack slot.
+ int Dummy;
+ if (RegInt.remat && !tii_->isLoadFromStackSlot(RegInt.remat, Dummy))
+ w /= 2;
+ RegInt.weight += w;
+ UniqueUses.insert(reg);
+ }
+ }
+ ++mii;
+ }
+ }
+ }
+
+ for (LiveIntervals::iterator I = li_->begin(), E = li_->end(); I != E; ++I) {
+ LiveInterval &LI = I->second;
+ if (MRegisterInfo::isVirtualRegister(LI.reg)) {
+ // If the live interval length is essentially zero, i.e. in every live
+ // range the use follows def immediately, it doesn't make sense to spill
+ // it and hope it will be easier to allocate for this li.
+ if (isZeroLengthInterval(&LI))
+ LI.weight = HUGE_VALF;
+
+ // Slightly prefer live interval that has been assigned a preferred reg.
+ if (LI.preference)
+ LI.weight *= 1.01F;
+
+ // Divide the weight of the interval by its size. This encourages
+ // spilling of intervals that are large and have few uses, and
+ // discourages spilling of small intervals with many uses.
+ LI.weight /= LI.getSize();
+ }
+ }
+
+ DEBUG(dump());
+ return true;
+}
+
+/// print - Implement the dump method.
+void SimpleRegisterCoalescing::print(std::ostream &O, const Module* m) const {
+ li_->print(O, m);
+}
diff --git a/lib/CodeGen/TwoAddressInstructionPass.cpp b/lib/CodeGen/TwoAddressInstructionPass.cpp
new file mode 100644
index 0000000..372b1b3
--- /dev/null
+++ b/lib/CodeGen/TwoAddressInstructionPass.cpp
@@ -0,0 +1,228 @@
+//===-- TwoAddressInstructionPass.cpp - Two-Address instruction pass ------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file was developed by the LLVM research group and is distributed under
+// the University of Illinois Open Source License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements the TwoAddress instruction pass which is used
+// by most register allocators. Two-Address instructions are rewritten
+// from:
+//
+// A = B op C
+//
+// to:
+//
+// A = B
+// A op= C
+//
+// Note that if a register allocator chooses to use this pass, that it
+// has to be capable of handling the non-SSA nature of these rewritten
+// virtual registers.
+//
+// It is also worth noting that the duplicate operand of the two
+// address instruction is removed.
+//
+//===----------------------------------------------------------------------===//
+
+#define DEBUG_TYPE "twoaddrinstr"
+#include "llvm/CodeGen/Passes.h"
+#include "llvm/Function.h"
+#include "llvm/CodeGen/LiveVariables.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/MachineInstr.h"
+#include "llvm/CodeGen/SSARegMap.h"
+#include "llvm/Target/MRegisterInfo.h"
+#include "llvm/Target/TargetInstrInfo.h"
+#include "llvm/Target/TargetMachine.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/Compiler.h"
+#include "llvm/ADT/Statistic.h"
+#include "llvm/ADT/STLExtras.h"
+using namespace llvm;
+
+STATISTIC(NumTwoAddressInstrs, "Number of two-address instructions");
+STATISTIC(NumCommuted , "Number of instructions commuted to coalesce");
+STATISTIC(NumConvertedTo3Addr, "Number of instructions promoted to 3-address");
+
+namespace {
+ struct VISIBILITY_HIDDEN TwoAddressInstructionPass
+ : public MachineFunctionPass {
+ static char ID; // Pass identification, replacement for typeid
+ TwoAddressInstructionPass() : MachineFunctionPass((intptr_t)&ID) {}
+
+ virtual void getAnalysisUsage(AnalysisUsage &AU) const;
+
+ /// runOnMachineFunction - pass entry point
+ bool runOnMachineFunction(MachineFunction&);
+ };
+
+ char TwoAddressInstructionPass::ID = 0;
+ RegisterPass<TwoAddressInstructionPass>
+ X("twoaddressinstruction", "Two-Address instruction pass");
+}
+
+const PassInfo *llvm::TwoAddressInstructionPassID = X.getPassInfo();
+
+void TwoAddressInstructionPass::getAnalysisUsage(AnalysisUsage &AU) const {
+ AU.addRequired<LiveVariables>();
+ AU.addPreserved<LiveVariables>();
+ AU.addPreservedID(PHIEliminationID);
+ MachineFunctionPass::getAnalysisUsage(AU);
+}
+
+/// runOnMachineFunction - Reduce two-address instructions to two
+/// operands.
+///
+bool TwoAddressInstructionPass::runOnMachineFunction(MachineFunction &MF) {
+ DOUT << "Machine Function\n";
+ const TargetMachine &TM = MF.getTarget();
+ const TargetInstrInfo &TII = *TM.getInstrInfo();
+ const MRegisterInfo &MRI = *TM.getRegisterInfo();
+ LiveVariables &LV = getAnalysis<LiveVariables>();
+
+ bool MadeChange = false;
+
+ DOUT << "********** REWRITING TWO-ADDR INSTRS **********\n";
+ DOUT << "********** Function: " << MF.getFunction()->getName() << '\n';
+
+ for (MachineFunction::iterator mbbi = MF.begin(), mbbe = MF.end();
+ mbbi != mbbe; ++mbbi) {
+ for (MachineBasicBlock::iterator mi = mbbi->begin(), me = mbbi->end();
+ mi != me; ++mi) {
+ const TargetInstrDescriptor *TID = mi->getInstrDescriptor();
+
+ bool FirstTied = true;
+ for (unsigned si = 1, e = TID->numOperands; si < e; ++si) {
+ int ti = TID->getOperandConstraint(si, TOI::TIED_TO);
+ if (ti == -1)
+ continue;
+
+ if (FirstTied) {
+ ++NumTwoAddressInstrs;
+ DOUT << '\t'; DEBUG(mi->print(*cerr.stream(), &TM));
+ }
+ FirstTied = false;
+
+ assert(mi->getOperand(si).isRegister() && mi->getOperand(si).getReg() &&
+ mi->getOperand(si).isUse() && "two address instruction invalid");
+
+ // if the two operands are the same we just remove the use
+ // and mark the def as def&use, otherwise we have to insert a copy.
+ if (mi->getOperand(ti).getReg() != mi->getOperand(si).getReg()) {
+ // rewrite:
+ // a = b op c
+ // to:
+ // a = b
+ // a = a op c
+ unsigned regA = mi->getOperand(ti).getReg();
+ unsigned regB = mi->getOperand(si).getReg();
+
+ assert(MRegisterInfo::isVirtualRegister(regA) &&
+ MRegisterInfo::isVirtualRegister(regB) &&
+ "cannot update physical register live information");
+
+#ifndef NDEBUG
+ // First, verify that we don't have a use of a in the instruction (a =
+ // b + a for example) because our transformation will not work. This
+ // should never occur because we are in SSA form.
+ for (unsigned i = 0; i != mi->getNumOperands(); ++i)
+ assert((int)i == ti ||
+ !mi->getOperand(i).isRegister() ||
+ mi->getOperand(i).getReg() != regA);
+#endif
+
+ // If this instruction is not the killing user of B, see if we can
+ // rearrange the code to make it so. Making it the killing user will
+ // allow us to coalesce A and B together, eliminating the copy we are
+ // about to insert.
+ if (!LV.KillsRegister(mi, regB)) {
+ // If this instruction is commutative, check to see if C dies. If
+ // so, swap the B and C operands. This makes the live ranges of A
+ // and C joinable.
+ // FIXME: This code also works for A := B op C instructions.
+ if ((TID->Flags & M_COMMUTABLE) && mi->getNumOperands() == 3) {
+ assert(mi->getOperand(3-si).isRegister() &&
+ "Not a proper commutative instruction!");
+ unsigned regC = mi->getOperand(3-si).getReg();
+ if (LV.KillsRegister(mi, regC)) {
+ DOUT << "2addr: COMMUTING : " << *mi;
+ MachineInstr *NewMI = TII.commuteInstruction(mi);
+ if (NewMI == 0) {
+ DOUT << "2addr: COMMUTING FAILED!\n";
+ } else {
+ DOUT << "2addr: COMMUTED TO: " << *NewMI;
+ // If the instruction changed to commute it, update livevar.
+ if (NewMI != mi) {
+ LV.instructionChanged(mi, NewMI); // Update live variables
+ mbbi->insert(mi, NewMI); // Insert the new inst
+ mbbi->erase(mi); // Nuke the old inst.
+ mi = NewMI;
+ }
+
+ ++NumCommuted;
+ regB = regC;
+ goto InstructionRearranged;
+ }
+ }
+ }
+
+ // If this instruction is potentially convertible to a true
+ // three-address instruction,
+ if (TID->Flags & M_CONVERTIBLE_TO_3_ADDR)
+ // FIXME: This assumes there are no more operands which are tied
+ // to another register.
+#ifndef NDEBUG
+ for (unsigned i = si+1, e = TID->numOperands; i < e; ++i)
+ assert(TID->getOperandConstraint(i, TOI::TIED_TO) == -1);
+#endif
+
+ if (MachineInstr *New = TII.convertToThreeAddress(mbbi, mi, LV)) {
+ DOUT << "2addr: CONVERTING 2-ADDR: " << *mi;
+ DOUT << "2addr: TO 3-ADDR: " << *New;
+ mbbi->erase(mi); // Nuke the old inst.
+ mi = New;
+ ++NumConvertedTo3Addr;
+ // Done with this instruction.
+ break;
+ }
+ }
+
+ InstructionRearranged:
+ const TargetRegisterClass* rc = MF.getSSARegMap()->getRegClass(regA);
+ MRI.copyRegToReg(*mbbi, mi, regA, regB, rc);
+
+ MachineBasicBlock::iterator prevMi = prior(mi);
+ DOUT << "\t\tprepend:\t"; DEBUG(prevMi->print(*cerr.stream(), &TM));
+
+ // Update live variables for regA
+ LiveVariables::VarInfo& varInfo = LV.getVarInfo(regA);
+ varInfo.DefInst = prevMi;
+
+ if (LV.removeVirtualRegisterKilled(regB, mbbi, mi))
+ LV.addVirtualRegisterKilled(regB, prevMi);
+
+ if (LV.removeVirtualRegisterDead(regB, mbbi, mi))
+ LV.addVirtualRegisterDead(regB, prevMi);
+
+ // replace all occurences of regB with regA
+ for (unsigned i = 0, e = mi->getNumOperands(); i != e; ++i) {
+ if (mi->getOperand(i).isRegister() &&
+ mi->getOperand(i).getReg() == regB)
+ mi->getOperand(i).setReg(regA);
+ }
+ }
+
+ assert(mi->getOperand(ti).isDef() && mi->getOperand(si).isUse());
+ mi->getOperand(ti).setReg(mi->getOperand(si).getReg());
+ MadeChange = true;
+
+ DOUT << "\t\trewrite to:\t"; DEBUG(mi->print(*cerr.stream(), &TM));
+ }
+ }
+ }
+
+ return MadeChange;
+}
diff --git a/lib/CodeGen/UnreachableBlockElim.cpp b/lib/CodeGen/UnreachableBlockElim.cpp
new file mode 100644
index 0000000..0a611ab
--- /dev/null
+++ b/lib/CodeGen/UnreachableBlockElim.cpp
@@ -0,0 +1,81 @@
+//===-- UnreachableBlockElim.cpp - Remove unreachable blocks for codegen --===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file was developed by the LLVM research group and is distributed under
+// the University of Illinois Open Source License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This pass is an extremely simple version of the SimplifyCFG pass. Its sole
+// job is to delete LLVM basic blocks that are not reachable from the entry
+// node. To do this, it performs a simple depth first traversal of the CFG,
+// then deletes any unvisited nodes.
+//
+// Note that this pass is really a hack. In particular, the instruction
+// selectors for various targets should just not generate code for unreachable
+// blocks. Until LLVM has a more systematic way of defining instruction
+// selectors, however, we cannot really expect them to handle additional
+// complexity.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/CodeGen/Passes.h"
+#include "llvm/Constant.h"
+#include "llvm/Instructions.h"
+#include "llvm/Function.h"
+#include "llvm/Pass.h"
+#include "llvm/Type.h"
+#include "llvm/Support/CFG.h"
+#include "llvm/Support/Compiler.h"
+#include "llvm/ADT/DepthFirstIterator.h"
+using namespace llvm;
+
+namespace {
+ class VISIBILITY_HIDDEN UnreachableBlockElim : public FunctionPass {
+ virtual bool runOnFunction(Function &F);
+ public:
+ static char ID; // Pass identification, replacement for typeid
+ UnreachableBlockElim() : FunctionPass((intptr_t)&ID) {}
+ };
+ char UnreachableBlockElim::ID = 0;
+ RegisterPass<UnreachableBlockElim>
+ X("unreachableblockelim", "Remove unreachable blocks from the CFG");
+}
+
+FunctionPass *llvm::createUnreachableBlockEliminationPass() {
+ return new UnreachableBlockElim();
+}
+
+bool UnreachableBlockElim::runOnFunction(Function &F) {
+ std::set<BasicBlock*> Reachable;
+
+ // Mark all reachable blocks.
+ for (df_ext_iterator<Function*> I = df_ext_begin(&F, Reachable),
+ E = df_ext_end(&F, Reachable); I != E; ++I)
+ /* Mark all reachable blocks */;
+
+ // Loop over all dead blocks, remembering them and deleting all instructions
+ // in them.
+ std::vector<BasicBlock*> DeadBlocks;
+ for (Function::iterator I = F.begin(), E = F.end(); I != E; ++I)
+ if (!Reachable.count(I)) {
+ BasicBlock *BB = I;
+ DeadBlocks.push_back(BB);
+ while (PHINode *PN = dyn_cast<PHINode>(BB->begin())) {
+ PN->replaceAllUsesWith(Constant::getNullValue(PN->getType()));
+ BB->getInstList().pop_front();
+ }
+ for (succ_iterator SI = succ_begin(BB), E = succ_end(BB); SI != E; ++SI)
+ (*SI)->removePredecessor(BB);
+ BB->dropAllReferences();
+ }
+
+ if (DeadBlocks.empty()) return false;
+
+ // Actually remove the blocks now.
+ for (unsigned i = 0, e = DeadBlocks.size(); i != e; ++i)
+ F.getBasicBlockList().erase(DeadBlocks[i]);
+
+ return true;
+}
diff --git a/lib/CodeGen/VirtRegMap.cpp b/lib/CodeGen/VirtRegMap.cpp
new file mode 100644
index 0000000..196e849
--- /dev/null
+++ b/lib/CodeGen/VirtRegMap.cpp
@@ -0,0 +1,1118 @@
+//===-- llvm/CodeGen/VirtRegMap.cpp - Virtual Register Map ----------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file was developed by the LLVM research group and is distributed under
+// the University of Illinois Open Source License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements the VirtRegMap class.
+//
+// It also contains implementations of the the Spiller interface, which, given a
+// virtual register map and a machine function, eliminates all virtual
+// references by replacing them with physical register references - adding spill
+// code as necessary.
+//
+//===----------------------------------------------------------------------===//
+
+#define DEBUG_TYPE "spiller"
+#include "VirtRegMap.h"
+#include "llvm/Function.h"
+#include "llvm/CodeGen/MachineFrameInfo.h"
+#include "llvm/CodeGen/MachineFunction.h"
+#include "llvm/CodeGen/SSARegMap.h"
+#include "llvm/Target/TargetMachine.h"
+#include "llvm/Target/TargetInstrInfo.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/Compiler.h"
+#include "llvm/ADT/BitVector.h"
+#include "llvm/ADT/Statistic.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/SmallSet.h"
+#include <algorithm>
+using namespace llvm;
+
+STATISTIC(NumSpills, "Number of register spills");
+STATISTIC(NumReMats, "Number of re-materialization");
+STATISTIC(NumStores, "Number of stores added");
+STATISTIC(NumLoads , "Number of loads added");
+STATISTIC(NumReused, "Number of values reused");
+STATISTIC(NumDSE , "Number of dead stores elided");
+STATISTIC(NumDCE , "Number of copies elided");
+
+namespace {
+ enum SpillerName { simple, local };
+
+ static cl::opt<SpillerName>
+ SpillerOpt("spiller",
+ cl::desc("Spiller to use: (default: local)"),
+ cl::Prefix,
+ cl::values(clEnumVal(simple, " simple spiller"),
+ clEnumVal(local, " local spiller"),
+ clEnumValEnd),
+ cl::init(local));
+}
+
+//===----------------------------------------------------------------------===//
+// VirtRegMap implementation
+//===----------------------------------------------------------------------===//
+
+VirtRegMap::VirtRegMap(MachineFunction &mf)
+ : TII(*mf.getTarget().getInstrInfo()), MF(mf),
+ Virt2PhysMap(NO_PHYS_REG), Virt2StackSlotMap(NO_STACK_SLOT),
+ ReMatId(MAX_STACK_SLOT+1) {
+ grow();
+}
+
+void VirtRegMap::grow() {
+ Virt2PhysMap.grow(MF.getSSARegMap()->getLastVirtReg());
+ Virt2StackSlotMap.grow(MF.getSSARegMap()->getLastVirtReg());
+}
+
+int VirtRegMap::assignVirt2StackSlot(unsigned virtReg) {
+ assert(MRegisterInfo::isVirtualRegister(virtReg));
+ assert(Virt2StackSlotMap[virtReg] == NO_STACK_SLOT &&
+ "attempt to assign stack slot to already spilled register");
+ const TargetRegisterClass* RC = MF.getSSARegMap()->getRegClass(virtReg);
+ int frameIndex = MF.getFrameInfo()->CreateStackObject(RC->getSize(),
+ RC->getAlignment());
+ Virt2StackSlotMap[virtReg] = frameIndex;
+ ++NumSpills;
+ return frameIndex;
+}
+
+void VirtRegMap::assignVirt2StackSlot(unsigned virtReg, int frameIndex) {
+ assert(MRegisterInfo::isVirtualRegister(virtReg));
+ assert(Virt2StackSlotMap[virtReg] == NO_STACK_SLOT &&
+ "attempt to assign stack slot to already spilled register");
+ assert((frameIndex >= 0 ||
+ (frameIndex >= MF.getFrameInfo()->getObjectIndexBegin())) &&
+ "illegal fixed frame index");
+ Virt2StackSlotMap[virtReg] = frameIndex;
+}
+
+int VirtRegMap::assignVirtReMatId(unsigned virtReg) {
+ assert(MRegisterInfo::isVirtualRegister(virtReg));
+ assert(Virt2StackSlotMap[virtReg] == NO_STACK_SLOT &&
+ "attempt to assign re-mat id to already spilled register");
+ const MachineInstr *DefMI = getReMaterializedMI(virtReg);
+ int FrameIdx;
+ if (TII.isLoadFromStackSlot((MachineInstr*)DefMI, FrameIdx)) {
+ // Load from stack slot is re-materialize as reload from the stack slot!
+ Virt2StackSlotMap[virtReg] = FrameIdx;
+ return FrameIdx;
+ }
+ Virt2StackSlotMap[virtReg] = ReMatId;
+ return ReMatId++;
+}
+
+void VirtRegMap::virtFolded(unsigned VirtReg, MachineInstr *OldMI,
+ unsigned OpNo, MachineInstr *NewMI) {
+ // Move previous memory references folded to new instruction.
+ MI2VirtMapTy::iterator IP = MI2VirtMap.lower_bound(NewMI);
+ for (MI2VirtMapTy::iterator I = MI2VirtMap.lower_bound(OldMI),
+ E = MI2VirtMap.end(); I != E && I->first == OldMI; ) {
+ MI2VirtMap.insert(IP, std::make_pair(NewMI, I->second));
+ MI2VirtMap.erase(I++);
+ }
+
+ ModRef MRInfo;
+ const TargetInstrDescriptor *TID = OldMI->getInstrDescriptor();
+ if (TID->getOperandConstraint(OpNo, TOI::TIED_TO) != -1 ||
+ TID->findTiedToSrcOperand(OpNo) != -1) {
+ // Folded a two-address operand.
+ MRInfo = isModRef;
+ } else if (OldMI->getOperand(OpNo).isDef()) {
+ MRInfo = isMod;
+ } else {
+ MRInfo = isRef;
+ }
+
+ // add new memory reference
+ MI2VirtMap.insert(IP, std::make_pair(NewMI, std::make_pair(VirtReg, MRInfo)));
+}
+
+void VirtRegMap::print(std::ostream &OS) const {
+ const MRegisterInfo* MRI = MF.getTarget().getRegisterInfo();
+
+ OS << "********** REGISTER MAP **********\n";
+ for (unsigned i = MRegisterInfo::FirstVirtualRegister,
+ e = MF.getSSARegMap()->getLastVirtReg(); i <= e; ++i) {
+ if (Virt2PhysMap[i] != (unsigned)VirtRegMap::NO_PHYS_REG)
+ OS << "[reg" << i << " -> " << MRI->getName(Virt2PhysMap[i]) << "]\n";
+
+ }
+
+ for (unsigned i = MRegisterInfo::FirstVirtualRegister,
+ e = MF.getSSARegMap()->getLastVirtReg(); i <= e; ++i)
+ if (Virt2StackSlotMap[i] != VirtRegMap::NO_STACK_SLOT)
+ OS << "[reg" << i << " -> fi#" << Virt2StackSlotMap[i] << "]\n";
+ OS << '\n';
+}
+
+void VirtRegMap::dump() const {
+ print(DOUT);
+}
+
+
+//===----------------------------------------------------------------------===//
+// Simple Spiller Implementation
+//===----------------------------------------------------------------------===//
+
+Spiller::~Spiller() {}
+
+namespace {
+ struct VISIBILITY_HIDDEN SimpleSpiller : public Spiller {
+ bool runOnMachineFunction(MachineFunction& mf, VirtRegMap &VRM);
+ };
+}
+
+bool SimpleSpiller::runOnMachineFunction(MachineFunction &MF, VirtRegMap &VRM) {
+ DOUT << "********** REWRITE MACHINE CODE **********\n";
+ DOUT << "********** Function: " << MF.getFunction()->getName() << '\n';
+ const TargetMachine &TM = MF.getTarget();
+ const MRegisterInfo &MRI = *TM.getRegisterInfo();
+
+ // LoadedRegs - Keep track of which vregs are loaded, so that we only load
+ // each vreg once (in the case where a spilled vreg is used by multiple
+ // operands). This is always smaller than the number of operands to the
+ // current machine instr, so it should be small.
+ std::vector<unsigned> LoadedRegs;
+
+ for (MachineFunction::iterator MBBI = MF.begin(), E = MF.end();
+ MBBI != E; ++MBBI) {
+ DOUT << MBBI->getBasicBlock()->getName() << ":\n";
+ MachineBasicBlock &MBB = *MBBI;
+ for (MachineBasicBlock::iterator MII = MBB.begin(),
+ E = MBB.end(); MII != E; ++MII) {
+ MachineInstr &MI = *MII;
+ for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
+ MachineOperand &MO = MI.getOperand(i);
+ if (MO.isRegister() && MO.getReg())
+ if (MRegisterInfo::isVirtualRegister(MO.getReg())) {
+ unsigned VirtReg = MO.getReg();
+ unsigned PhysReg = VRM.getPhys(VirtReg);
+ if (VRM.hasStackSlot(VirtReg)) {
+ int StackSlot = VRM.getStackSlot(VirtReg);
+ const TargetRegisterClass* RC =
+ MF.getSSARegMap()->getRegClass(VirtReg);
+
+ if (MO.isUse() &&
+ std::find(LoadedRegs.begin(), LoadedRegs.end(), VirtReg)
+ == LoadedRegs.end()) {
+ MRI.loadRegFromStackSlot(MBB, &MI, PhysReg, StackSlot, RC);
+ LoadedRegs.push_back(VirtReg);
+ ++NumLoads;
+ DOUT << '\t' << *prior(MII);
+ }
+
+ if (MO.isDef()) {
+ MRI.storeRegToStackSlot(MBB, next(MII), PhysReg, StackSlot, RC);
+ ++NumStores;
+ }
+ }
+ MF.setPhysRegUsed(PhysReg);
+ MI.getOperand(i).setReg(PhysReg);
+ } else {
+ MF.setPhysRegUsed(MO.getReg());
+ }
+ }
+
+ DOUT << '\t' << MI;
+ LoadedRegs.clear();
+ }
+ }
+ return true;
+}
+
+//===----------------------------------------------------------------------===//
+// Local Spiller Implementation
+//===----------------------------------------------------------------------===//
+
+namespace {
+ /// LocalSpiller - This spiller does a simple pass over the machine basic
+ /// block to attempt to keep spills in registers as much as possible for
+ /// blocks that have low register pressure (the vreg may be spilled due to
+ /// register pressure in other blocks).
+ class VISIBILITY_HIDDEN LocalSpiller : public Spiller {
+ const MRegisterInfo *MRI;
+ const TargetInstrInfo *TII;
+ public:
+ bool runOnMachineFunction(MachineFunction &MF, VirtRegMap &VRM) {
+ MRI = MF.getTarget().getRegisterInfo();
+ TII = MF.getTarget().getInstrInfo();
+ DOUT << "\n**** Local spiller rewriting function '"
+ << MF.getFunction()->getName() << "':\n";
+
+ std::vector<MachineInstr *> ReMatedMIs;
+ for (MachineFunction::iterator MBB = MF.begin(), E = MF.end();
+ MBB != E; ++MBB)
+ RewriteMBB(*MBB, VRM, ReMatedMIs);
+ for (unsigned i = 0, e = ReMatedMIs.size(); i != e; ++i)
+ delete ReMatedMIs[i];
+ return true;
+ }
+ private:
+ void RewriteMBB(MachineBasicBlock &MBB, VirtRegMap &VRM,
+ std::vector<MachineInstr*> &ReMatedMIs);
+ };
+}
+
+/// AvailableSpills - As the local spiller is scanning and rewriting an MBB from
+/// top down, keep track of which spills slots are available in each register.
+///
+/// Note that not all physregs are created equal here. In particular, some
+/// physregs are reloads that we are allowed to clobber or ignore at any time.
+/// Other physregs are values that the register allocated program is using that
+/// we cannot CHANGE, but we can read if we like. We keep track of this on a
+/// per-stack-slot basis as the low bit in the value of the SpillSlotsAvailable
+/// entries. The predicate 'canClobberPhysReg()' checks this bit and
+/// addAvailable sets it if.
+namespace {
+class VISIBILITY_HIDDEN AvailableSpills {
+ const MRegisterInfo *MRI;
+ const TargetInstrInfo *TII;
+
+ // SpillSlotsAvailable - This map keeps track of all of the spilled virtual
+ // register values that are still available, due to being loaded or stored to,
+ // but not invalidated yet.
+ std::map<int, unsigned> SpillSlotsAvailable;
+
+ // PhysRegsAvailable - This is the inverse of SpillSlotsAvailable, indicating
+ // which stack slot values are currently held by a physreg. This is used to
+ // invalidate entries in SpillSlotsAvailable when a physreg is modified.
+ std::multimap<unsigned, int> PhysRegsAvailable;
+
+ void disallowClobberPhysRegOnly(unsigned PhysReg);
+
+ void ClobberPhysRegOnly(unsigned PhysReg);
+public:
+ AvailableSpills(const MRegisterInfo *mri, const TargetInstrInfo *tii)
+ : MRI(mri), TII(tii) {
+ }
+
+ const MRegisterInfo *getRegInfo() const { return MRI; }
+
+ /// getSpillSlotPhysReg - If the specified stack slot is available in a
+ /// physical register, return that PhysReg, otherwise return 0.
+ unsigned getSpillSlotPhysReg(int Slot) const {
+ std::map<int, unsigned>::const_iterator I = SpillSlotsAvailable.find(Slot);
+ if (I != SpillSlotsAvailable.end()) {
+ return I->second >> 1; // Remove the CanClobber bit.
+ }
+ return 0;
+ }
+
+ /// addAvailable - Mark that the specified stack slot is available in the
+ /// specified physreg. If CanClobber is true, the physreg can be modified at
+ /// any time without changing the semantics of the program.
+ void addAvailable(int Slot, MachineInstr *MI, unsigned Reg,
+ bool CanClobber = true) {
+ // If this stack slot is thought to be available in some other physreg,
+ // remove its record.
+ ModifyStackSlot(Slot);
+
+ PhysRegsAvailable.insert(std::make_pair(Reg, Slot));
+ SpillSlotsAvailable[Slot] = (Reg << 1) | (unsigned)CanClobber;
+
+ if (Slot > VirtRegMap::MAX_STACK_SLOT)
+ DOUT << "Remembering RM#" << Slot-VirtRegMap::MAX_STACK_SLOT-1;
+ else
+ DOUT << "Remembering SS#" << Slot;
+ DOUT << " in physreg " << MRI->getName(Reg) << "\n";
+ }
+
+ /// canClobberPhysReg - Return true if the spiller is allowed to change the
+ /// value of the specified stackslot register if it desires. The specified
+ /// stack slot must be available in a physreg for this query to make sense.
+ bool canClobberPhysReg(int Slot) const {
+ assert(SpillSlotsAvailable.count(Slot) && "Slot not available!");
+ return SpillSlotsAvailable.find(Slot)->second & 1;
+ }
+
+ /// disallowClobberPhysReg - Unset the CanClobber bit of the specified
+ /// stackslot register. The register is still available but is no longer
+ /// allowed to be modifed.
+ void disallowClobberPhysReg(unsigned PhysReg);
+
+ /// ClobberPhysReg - This is called when the specified physreg changes
+ /// value. We use this to invalidate any info about stuff we thing lives in
+ /// it and any of its aliases.
+ void ClobberPhysReg(unsigned PhysReg);
+
+ /// ModifyStackSlot - This method is called when the value in a stack slot
+ /// changes. This removes information about which register the previous value
+ /// for this slot lives in (as the previous value is dead now).
+ void ModifyStackSlot(int Slot);
+};
+}
+
+/// disallowClobberPhysRegOnly - Unset the CanClobber bit of the specified
+/// stackslot register. The register is still available but is no longer
+/// allowed to be modifed.
+void AvailableSpills::disallowClobberPhysRegOnly(unsigned PhysReg) {
+ std::multimap<unsigned, int>::iterator I =
+ PhysRegsAvailable.lower_bound(PhysReg);
+ while (I != PhysRegsAvailable.end() && I->first == PhysReg) {
+ int Slot = I->second;
+ I++;
+ assert((SpillSlotsAvailable[Slot] >> 1) == PhysReg &&
+ "Bidirectional map mismatch!");
+ SpillSlotsAvailable[Slot] &= ~1;
+ DOUT << "PhysReg " << MRI->getName(PhysReg)
+ << " copied, it is available for use but can no longer be modified\n";
+ }
+}
+
+/// disallowClobberPhysReg - Unset the CanClobber bit of the specified
+/// stackslot register and its aliases. The register and its aliases may
+/// still available but is no longer allowed to be modifed.
+void AvailableSpills::disallowClobberPhysReg(unsigned PhysReg) {
+ for (const unsigned *AS = MRI->getAliasSet(PhysReg); *AS; ++AS)
+ disallowClobberPhysRegOnly(*AS);
+ disallowClobberPhysRegOnly(PhysReg);
+}
+
+/// ClobberPhysRegOnly - This is called when the specified physreg changes
+/// value. We use this to invalidate any info about stuff we thing lives in it.
+void AvailableSpills::ClobberPhysRegOnly(unsigned PhysReg) {
+ std::multimap<unsigned, int>::iterator I =
+ PhysRegsAvailable.lower_bound(PhysReg);
+ while (I != PhysRegsAvailable.end() && I->first == PhysReg) {
+ int Slot = I->second;
+ PhysRegsAvailable.erase(I++);
+ assert((SpillSlotsAvailable[Slot] >> 1) == PhysReg &&
+ "Bidirectional map mismatch!");
+ SpillSlotsAvailable.erase(Slot);
+ DOUT << "PhysReg " << MRI->getName(PhysReg)
+ << " clobbered, invalidating ";
+ if (Slot > VirtRegMap::MAX_STACK_SLOT)
+ DOUT << "RM#" << Slot-VirtRegMap::MAX_STACK_SLOT-1 << "\n";
+ else
+ DOUT << "SS#" << Slot << "\n";
+ }
+}
+
+/// ClobberPhysReg - This is called when the specified physreg changes
+/// value. We use this to invalidate any info about stuff we thing lives in
+/// it and any of its aliases.
+void AvailableSpills::ClobberPhysReg(unsigned PhysReg) {
+ for (const unsigned *AS = MRI->getAliasSet(PhysReg); *AS; ++AS)
+ ClobberPhysRegOnly(*AS);
+ ClobberPhysRegOnly(PhysReg);
+}
+
+/// ModifyStackSlot - This method is called when the value in a stack slot
+/// changes. This removes information about which register the previous value
+/// for this slot lives in (as the previous value is dead now).
+void AvailableSpills::ModifyStackSlot(int Slot) {
+ std::map<int, unsigned>::iterator It = SpillSlotsAvailable.find(Slot);
+ if (It == SpillSlotsAvailable.end()) return;
+ unsigned Reg = It->second >> 1;
+ SpillSlotsAvailable.erase(It);
+
+ // This register may hold the value of multiple stack slots, only remove this
+ // stack slot from the set of values the register contains.
+ std::multimap<unsigned, int>::iterator I = PhysRegsAvailable.lower_bound(Reg);
+ for (; ; ++I) {
+ assert(I != PhysRegsAvailable.end() && I->first == Reg &&
+ "Map inverse broken!");
+ if (I->second == Slot) break;
+ }
+ PhysRegsAvailable.erase(I);
+}
+
+
+
+/// InvalidateKills - MI is going to be deleted. If any of its operands are
+/// marked kill, then invalidate the information.
+static void InvalidateKills(MachineInstr &MI, BitVector &RegKills,
+ std::vector<MachineOperand*> &KillOps) {
+ for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
+ MachineOperand &MO = MI.getOperand(i);
+ if (!MO.isReg() || !MO.isUse() || !MO.isKill())
+ continue;
+ unsigned Reg = MO.getReg();
+ if (KillOps[Reg] == &MO) {
+ RegKills.reset(Reg);
+ KillOps[Reg] = NULL;
+ }
+ }
+}
+
+/// UpdateKills - Track and update kill info. If a MI reads a register that is
+/// marked kill, then it must be due to register reuse. Transfer the kill info
+/// over.
+static void UpdateKills(MachineInstr &MI, BitVector &RegKills,
+ std::vector<MachineOperand*> &KillOps) {
+ const TargetInstrDescriptor *TID = MI.getInstrDescriptor();
+ for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
+ MachineOperand &MO = MI.getOperand(i);
+ if (!MO.isReg() || !MO.isUse())
+ continue;
+ unsigned Reg = MO.getReg();
+ if (Reg == 0)
+ continue;
+
+ if (RegKills[Reg]) {
+ // That can't be right. Register is killed but not re-defined and it's
+ // being reused. Let's fix that.
+ KillOps[Reg]->unsetIsKill();
+ if (i < TID->numOperands &&
+ TID->getOperandConstraint(i, TOI::TIED_TO) == -1)
+ // Unless it's a two-address operand, this is the new kill.
+ MO.setIsKill();
+ }
+
+ if (MO.isKill()) {
+ RegKills.set(Reg);
+ KillOps[Reg] = &MO;
+ }
+ }
+
+ for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
+ const MachineOperand &MO = MI.getOperand(i);
+ if (!MO.isReg() || !MO.isDef())
+ continue;
+ unsigned Reg = MO.getReg();
+ RegKills.reset(Reg);
+ KillOps[Reg] = NULL;
+ }
+}
+
+
+// ReusedOp - For each reused operand, we keep track of a bit of information, in
+// case we need to rollback upon processing a new operand. See comments below.
+namespace {
+ struct ReusedOp {
+ // The MachineInstr operand that reused an available value.
+ unsigned Operand;
+
+ // StackSlot - The spill slot of the value being reused.
+ unsigned StackSlot;
+
+ // PhysRegReused - The physical register the value was available in.
+ unsigned PhysRegReused;
+
+ // AssignedPhysReg - The physreg that was assigned for use by the reload.
+ unsigned AssignedPhysReg;
+
+ // VirtReg - The virtual register itself.
+ unsigned VirtReg;
+
+ ReusedOp(unsigned o, unsigned ss, unsigned prr, unsigned apr,
+ unsigned vreg)
+ : Operand(o), StackSlot(ss), PhysRegReused(prr), AssignedPhysReg(apr),
+ VirtReg(vreg) {}
+ };
+
+ /// ReuseInfo - This maintains a collection of ReuseOp's for each operand that
+ /// is reused instead of reloaded.
+ class VISIBILITY_HIDDEN ReuseInfo {
+ MachineInstr &MI;
+ std::vector<ReusedOp> Reuses;
+ BitVector PhysRegsClobbered;
+ public:
+ ReuseInfo(MachineInstr &mi, const MRegisterInfo *mri) : MI(mi) {
+ PhysRegsClobbered.resize(mri->getNumRegs());
+ }
+
+ bool hasReuses() const {
+ return !Reuses.empty();
+ }
+
+ /// addReuse - If we choose to reuse a virtual register that is already
+ /// available instead of reloading it, remember that we did so.
+ void addReuse(unsigned OpNo, unsigned StackSlot,
+ unsigned PhysRegReused, unsigned AssignedPhysReg,
+ unsigned VirtReg) {
+ // If the reload is to the assigned register anyway, no undo will be
+ // required.
+ if (PhysRegReused == AssignedPhysReg) return;
+
+ // Otherwise, remember this.
+ Reuses.push_back(ReusedOp(OpNo, StackSlot, PhysRegReused,
+ AssignedPhysReg, VirtReg));
+ }
+
+ void markClobbered(unsigned PhysReg) {
+ PhysRegsClobbered.set(PhysReg);
+ }
+
+ bool isClobbered(unsigned PhysReg) const {
+ return PhysRegsClobbered.test(PhysReg);
+ }
+
+ /// GetRegForReload - We are about to emit a reload into PhysReg. If there
+ /// is some other operand that is using the specified register, either pick
+ /// a new register to use, or evict the previous reload and use this reg.
+ unsigned GetRegForReload(unsigned PhysReg, MachineInstr *MI,
+ AvailableSpills &Spills,
+ std::map<int, MachineInstr*> &MaybeDeadStores,
+ SmallSet<unsigned, 8> &Rejected,
+ BitVector &RegKills,
+ std::vector<MachineOperand*> &KillOps) {
+ if (Reuses.empty()) return PhysReg; // This is most often empty.
+
+ for (unsigned ro = 0, e = Reuses.size(); ro != e; ++ro) {
+ ReusedOp &Op = Reuses[ro];
+ // If we find some other reuse that was supposed to use this register
+ // exactly for its reload, we can change this reload to use ITS reload
+ // register. That is, unless its reload register has already been
+ // considered and subsequently rejected because it has also been reused
+ // by another operand.
+ if (Op.PhysRegReused == PhysReg &&
+ Rejected.count(Op.AssignedPhysReg) == 0) {
+ // Yup, use the reload register that we didn't use before.
+ unsigned NewReg = Op.AssignedPhysReg;
+ Rejected.insert(PhysReg);
+ return GetRegForReload(NewReg, MI, Spills, MaybeDeadStores, Rejected,
+ RegKills, KillOps);
+ } else {
+ // Otherwise, we might also have a problem if a previously reused
+ // value aliases the new register. If so, codegen the previous reload
+ // and use this one.
+ unsigned PRRU = Op.PhysRegReused;
+ const MRegisterInfo *MRI = Spills.getRegInfo();
+ if (MRI->areAliases(PRRU, PhysReg)) {
+ // Okay, we found out that an alias of a reused register
+ // was used. This isn't good because it means we have
+ // to undo a previous reuse.
+ MachineBasicBlock *MBB = MI->getParent();
+ const TargetRegisterClass *AliasRC =
+ MBB->getParent()->getSSARegMap()->getRegClass(Op.VirtReg);
+
+ // Copy Op out of the vector and remove it, we're going to insert an
+ // explicit load for it.
+ ReusedOp NewOp = Op;
+ Reuses.erase(Reuses.begin()+ro);
+
+ // Ok, we're going to try to reload the assigned physreg into the
+ // slot that we were supposed to in the first place. However, that
+ // register could hold a reuse. Check to see if it conflicts or
+ // would prefer us to use a different register.
+ unsigned NewPhysReg = GetRegForReload(NewOp.AssignedPhysReg,
+ MI, Spills, MaybeDeadStores,
+ Rejected, RegKills, KillOps);
+
+ MRI->loadRegFromStackSlot(*MBB, MI, NewPhysReg,
+ NewOp.StackSlot, AliasRC);
+ Spills.ClobberPhysReg(NewPhysReg);
+ Spills.ClobberPhysReg(NewOp.PhysRegReused);
+
+ // Any stores to this stack slot are not dead anymore.
+ MaybeDeadStores.erase(NewOp.StackSlot);
+
+ MI->getOperand(NewOp.Operand).setReg(NewPhysReg);
+
+ Spills.addAvailable(NewOp.StackSlot, MI, NewPhysReg);
+ ++NumLoads;
+ MachineBasicBlock::iterator MII = MI;
+ --MII;
+ UpdateKills(*MII, RegKills, KillOps);
+ DOUT << '\t' << *MII;
+
+ DOUT << "Reuse undone!\n";
+ --NumReused;
+
+ // Finally, PhysReg is now available, go ahead and use it.
+ return PhysReg;
+ }
+ }
+ }
+ return PhysReg;
+ }
+
+ /// GetRegForReload - Helper for the above GetRegForReload(). Add a
+ /// 'Rejected' set to remember which registers have been considered and
+ /// rejected for the reload. This avoids infinite looping in case like
+ /// this:
+ /// t1 := op t2, t3
+ /// t2 <- assigned r0 for use by the reload but ended up reuse r1
+ /// t3 <- assigned r1 for use by the reload but ended up reuse r0
+ /// t1 <- desires r1
+ /// sees r1 is taken by t2, tries t2's reload register r0
+ /// sees r0 is taken by t3, tries t3's reload register r1
+ /// sees r1 is taken by t2, tries t2's reload register r0 ...
+ unsigned GetRegForReload(unsigned PhysReg, MachineInstr *MI,
+ AvailableSpills &Spills,
+ std::map<int, MachineInstr*> &MaybeDeadStores,
+ BitVector &RegKills,
+ std::vector<MachineOperand*> &KillOps) {
+ SmallSet<unsigned, 8> Rejected;
+ return GetRegForReload(PhysReg, MI, Spills, MaybeDeadStores, Rejected,
+ RegKills, KillOps);
+ }
+ };
+}
+
+
+/// rewriteMBB - Keep track of which spills are available even after the
+/// register allocator is done with them. If possible, avoid reloading vregs.
+void LocalSpiller::RewriteMBB(MachineBasicBlock &MBB, VirtRegMap &VRM,
+ std::vector<MachineInstr*> &ReMatedMIs) {
+ DOUT << MBB.getBasicBlock()->getName() << ":\n";
+
+ // Spills - Keep track of which spilled values are available in physregs so
+ // that we can choose to reuse the physregs instead of emitting reloads.
+ AvailableSpills Spills(MRI, TII);
+
+ // MaybeDeadStores - When we need to write a value back into a stack slot,
+ // keep track of the inserted store. If the stack slot value is never read
+ // (because the value was used from some available register, for example), and
+ // subsequently stored to, the original store is dead. This map keeps track
+ // of inserted stores that are not used. If we see a subsequent store to the
+ // same stack slot, the original store is deleted.
+ std::map<int, MachineInstr*> MaybeDeadStores;
+
+ // Keep track of kill information.
+ BitVector RegKills(MRI->getNumRegs());
+ std::vector<MachineOperand*> KillOps;
+ KillOps.resize(MRI->getNumRegs(), NULL);
+
+ MachineFunction &MF = *MBB.getParent();
+ for (MachineBasicBlock::iterator MII = MBB.begin(), E = MBB.end();
+ MII != E; ) {
+ MachineInstr &MI = *MII;
+ MachineBasicBlock::iterator NextMII = MII; ++NextMII;
+ VirtRegMap::MI2VirtMapTy::const_iterator I, End;
+
+ bool Erased = false;
+ bool BackTracked = false;
+
+ /// ReusedOperands - Keep track of operand reuse in case we need to undo
+ /// reuse.
+ ReuseInfo ReusedOperands(MI, MRI);
+
+ // Loop over all of the implicit defs, clearing them from our available
+ // sets.
+ const TargetInstrDescriptor *TID = MI.getInstrDescriptor();
+
+ // If this instruction is being rematerialized, just remove it!
+ int FrameIdx;
+ if (TII->isTriviallyReMaterializable(&MI) ||
+ TII->isLoadFromStackSlot(&MI, FrameIdx)) {
+ Erased = true;
+ for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
+ MachineOperand &MO = MI.getOperand(i);
+ if (!MO.isRegister() || MO.getReg() == 0)
+ continue; // Ignore non-register operands.
+ if (MO.isDef() && !VRM.isReMaterialized(MO.getReg())) {
+ Erased = false;
+ break;
+ }
+ }
+ if (Erased) {
+ VRM.RemoveFromFoldedVirtMap(&MI);
+ ReMatedMIs.push_back(MI.removeFromParent());
+ goto ProcessNextInst;
+ }
+ }
+
+ if (TID->ImplicitDefs) {
+ const unsigned *ImpDef = TID->ImplicitDefs;
+ for ( ; *ImpDef; ++ImpDef) {
+ MF.setPhysRegUsed(*ImpDef);
+ ReusedOperands.markClobbered(*ImpDef);
+ Spills.ClobberPhysReg(*ImpDef);
+ }
+ }
+
+ // Process all of the spilled uses and all non spilled reg references.
+ for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
+ MachineOperand &MO = MI.getOperand(i);
+ if (!MO.isRegister() || MO.getReg() == 0)
+ continue; // Ignore non-register operands.
+
+ if (MRegisterInfo::isPhysicalRegister(MO.getReg())) {
+ // Ignore physregs for spilling, but remember that it is used by this
+ // function.
+ MF.setPhysRegUsed(MO.getReg());
+ ReusedOperands.markClobbered(MO.getReg());
+ continue;
+ }
+
+ assert(MRegisterInfo::isVirtualRegister(MO.getReg()) &&
+ "Not a virtual or a physical register?");
+
+ unsigned VirtReg = MO.getReg();
+ if (!VRM.hasStackSlot(VirtReg)) {
+ // This virtual register was assigned a physreg!
+ unsigned Phys = VRM.getPhys(VirtReg);
+ MF.setPhysRegUsed(Phys);
+ if (MO.isDef())
+ ReusedOperands.markClobbered(Phys);
+ MI.getOperand(i).setReg(Phys);
+ continue;
+ }
+
+ // This virtual register is now known to be a spilled value.
+ if (!MO.isUse())
+ continue; // Handle defs in the loop below (handle use&def here though)
+
+ bool doReMat = VRM.isReMaterialized(VirtReg);
+ int StackSlot = VRM.getStackSlot(VirtReg);
+ unsigned PhysReg;
+
+ // Check to see if this stack slot is available.
+ if ((PhysReg = Spills.getSpillSlotPhysReg(StackSlot))) {
+ // This spilled operand might be part of a two-address operand. If this
+ // is the case, then changing it will necessarily require changing the
+ // def part of the instruction as well. However, in some cases, we
+ // aren't allowed to modify the reused register. If none of these cases
+ // apply, reuse it.
+ bool CanReuse = true;
+ int ti = TID->getOperandConstraint(i, TOI::TIED_TO);
+ if (ti != -1 &&
+ MI.getOperand(ti).isReg() &&
+ MI.getOperand(ti).getReg() == VirtReg) {
+ // Okay, we have a two address operand. We can reuse this physreg as
+ // long as we are allowed to clobber the value and there isn't an
+ // earlier def that has already clobbered the physreg.
+ CanReuse = Spills.canClobberPhysReg(StackSlot) &&
+ !ReusedOperands.isClobbered(PhysReg);
+ }
+
+ if (CanReuse) {
+ // If this stack slot value is already available, reuse it!
+ if (StackSlot > VirtRegMap::MAX_STACK_SLOT)
+ DOUT << "Reusing RM#" << StackSlot-VirtRegMap::MAX_STACK_SLOT-1;
+ else
+ DOUT << "Reusing SS#" << StackSlot;
+ DOUT << " from physreg "
+ << MRI->getName(PhysReg) << " for vreg"
+ << VirtReg <<" instead of reloading into physreg "
+ << MRI->getName(VRM.getPhys(VirtReg)) << "\n";
+ MI.getOperand(i).setReg(PhysReg);
+
+ // The only technical detail we have is that we don't know that
+ // PhysReg won't be clobbered by a reloaded stack slot that occurs
+ // later in the instruction. In particular, consider 'op V1, V2'.
+ // If V1 is available in physreg R0, we would choose to reuse it
+ // here, instead of reloading it into the register the allocator
+ // indicated (say R1). However, V2 might have to be reloaded
+ // later, and it might indicate that it needs to live in R0. When
+ // this occurs, we need to have information available that
+ // indicates it is safe to use R1 for the reload instead of R0.
+ //
+ // To further complicate matters, we might conflict with an alias,
+ // or R0 and R1 might not be compatible with each other. In this
+ // case, we actually insert a reload for V1 in R1, ensuring that
+ // we can get at R0 or its alias.
+ ReusedOperands.addReuse(i, StackSlot, PhysReg,
+ VRM.getPhys(VirtReg), VirtReg);
+ if (ti != -1)
+ // Only mark it clobbered if this is a use&def operand.
+ ReusedOperands.markClobbered(PhysReg);
+ ++NumReused;
+ continue;
+ }
+
+ // Otherwise we have a situation where we have a two-address instruction
+ // whose mod/ref operand needs to be reloaded. This reload is already
+ // available in some register "PhysReg", but if we used PhysReg as the
+ // operand to our 2-addr instruction, the instruction would modify
+ // PhysReg. This isn't cool if something later uses PhysReg and expects
+ // to get its initial value.
+ //
+ // To avoid this problem, and to avoid doing a load right after a store,
+ // we emit a copy from PhysReg into the designated register for this
+ // operand.
+ unsigned DesignatedReg = VRM.getPhys(VirtReg);
+ assert(DesignatedReg && "Must map virtreg to physreg!");
+
+ // Note that, if we reused a register for a previous operand, the
+ // register we want to reload into might not actually be
+ // available. If this occurs, use the register indicated by the
+ // reuser.
+ if (ReusedOperands.hasReuses())
+ DesignatedReg = ReusedOperands.GetRegForReload(DesignatedReg, &MI,
+ Spills, MaybeDeadStores, RegKills, KillOps);
+
+ // If the mapped designated register is actually the physreg we have
+ // incoming, we don't need to inserted a dead copy.
+ if (DesignatedReg == PhysReg) {
+ // If this stack slot value is already available, reuse it!
+ if (StackSlot > VirtRegMap::MAX_STACK_SLOT)
+ DOUT << "Reusing RM#" << StackSlot-VirtRegMap::MAX_STACK_SLOT-1;
+ else
+ DOUT << "Reusing SS#" << StackSlot;
+ DOUT << " from physreg " << MRI->getName(PhysReg) << " for vreg"
+ << VirtReg
+ << " instead of reloading into same physreg.\n";
+ MI.getOperand(i).setReg(PhysReg);
+ ReusedOperands.markClobbered(PhysReg);
+ ++NumReused;
+ continue;
+ }
+
+ const TargetRegisterClass* RC = MF.getSSARegMap()->getRegClass(VirtReg);
+ MF.setPhysRegUsed(DesignatedReg);
+ ReusedOperands.markClobbered(DesignatedReg);
+ MRI->copyRegToReg(MBB, &MI, DesignatedReg, PhysReg, RC);
+
+ MachineInstr *CopyMI = prior(MII);
+ UpdateKills(*CopyMI, RegKills, KillOps);
+
+ // This invalidates DesignatedReg.
+ Spills.ClobberPhysReg(DesignatedReg);
+
+ Spills.addAvailable(StackSlot, &MI, DesignatedReg);
+ MI.getOperand(i).setReg(DesignatedReg);
+ DOUT << '\t' << *prior(MII);
+ ++NumReused;
+ continue;
+ }
+
+ // Otherwise, reload it and remember that we have it.
+ PhysReg = VRM.getPhys(VirtReg);
+ assert(PhysReg && "Must map virtreg to physreg!");
+ const TargetRegisterClass* RC = MF.getSSARegMap()->getRegClass(VirtReg);
+
+ // Note that, if we reused a register for a previous operand, the
+ // register we want to reload into might not actually be
+ // available. If this occurs, use the register indicated by the
+ // reuser.
+ if (ReusedOperands.hasReuses())
+ PhysReg = ReusedOperands.GetRegForReload(PhysReg, &MI,
+ Spills, MaybeDeadStores, RegKills, KillOps);
+
+ MF.setPhysRegUsed(PhysReg);
+ ReusedOperands.markClobbered(PhysReg);
+ if (doReMat) {
+ MRI->reMaterialize(MBB, &MI, PhysReg, VRM.getReMaterializedMI(VirtReg));
+ ++NumReMats;
+ } else {
+ MRI->loadRegFromStackSlot(MBB, &MI, PhysReg, StackSlot, RC);
+ ++NumLoads;
+ }
+ // This invalidates PhysReg.
+ Spills.ClobberPhysReg(PhysReg);
+
+ // Any stores to this stack slot are not dead anymore.
+ if (!doReMat)
+ MaybeDeadStores.erase(StackSlot);
+ Spills.addAvailable(StackSlot, &MI, PhysReg);
+ // Assumes this is the last use. IsKill will be unset if reg is reused
+ // unless it's a two-address operand.
+ if (TID->getOperandConstraint(i, TOI::TIED_TO) == -1)
+ MI.getOperand(i).setIsKill();
+ MI.getOperand(i).setReg(PhysReg);
+ UpdateKills(*prior(MII), RegKills, KillOps);
+ DOUT << '\t' << *prior(MII);
+ }
+
+ DOUT << '\t' << MI;
+
+ // If we have folded references to memory operands, make sure we clear all
+ // physical registers that may contain the value of the spilled virtual
+ // register
+ for (tie(I, End) = VRM.getFoldedVirts(&MI); I != End; ++I) {
+ DOUT << "Folded vreg: " << I->second.first << " MR: "
+ << I->second.second;
+ unsigned VirtReg = I->second.first;
+ VirtRegMap::ModRef MR = I->second.second;
+ if (!VRM.hasStackSlot(VirtReg)) {
+ DOUT << ": No stack slot!\n";
+ continue;
+ }
+ int SS = VRM.getStackSlot(VirtReg);
+ DOUT << " - StackSlot: " << SS << "\n";
+
+ // If this folded instruction is just a use, check to see if it's a
+ // straight load from the virt reg slot.
+ if ((MR & VirtRegMap::isRef) && !(MR & VirtRegMap::isMod)) {
+ int FrameIdx;
+ if (unsigned DestReg = TII->isLoadFromStackSlot(&MI, FrameIdx)) {
+ if (FrameIdx == SS) {
+ // If this spill slot is available, turn it into a copy (or nothing)
+ // instead of leaving it as a load!
+ if (unsigned InReg = Spills.getSpillSlotPhysReg(SS)) {
+ DOUT << "Promoted Load To Copy: " << MI;
+ if (DestReg != InReg) {
+ MRI->copyRegToReg(MBB, &MI, DestReg, InReg,
+ MF.getSSARegMap()->getRegClass(VirtReg));
+ // Revisit the copy so we make sure to notice the effects of the
+ // operation on the destreg (either needing to RA it if it's
+ // virtual or needing to clobber any values if it's physical).
+ NextMII = &MI;
+ --NextMII; // backtrack to the copy.
+ BackTracked = true;
+ } else
+ DOUT << "Removing now-noop copy: " << MI;
+
+ VRM.RemoveFromFoldedVirtMap(&MI);
+ MBB.erase(&MI);
+ Erased = true;
+ goto ProcessNextInst;
+ }
+ }
+ }
+ }
+
+ // If this reference is not a use, any previous store is now dead.
+ // Otherwise, the store to this stack slot is not dead anymore.
+ std::map<int, MachineInstr*>::iterator MDSI = MaybeDeadStores.find(SS);
+ if (MDSI != MaybeDeadStores.end()) {
+ if (MR & VirtRegMap::isRef) // Previous store is not dead.
+ MaybeDeadStores.erase(MDSI);
+ else {
+ // If we get here, the store is dead, nuke it now.
+ assert(VirtRegMap::isMod && "Can't be modref!");
+ DOUT << "Removed dead store:\t" << *MDSI->second;
+ InvalidateKills(*MDSI->second, RegKills, KillOps);
+ MBB.erase(MDSI->second);
+ VRM.RemoveFromFoldedVirtMap(MDSI->second);
+ MaybeDeadStores.erase(MDSI);
+ ++NumDSE;
+ }
+ }
+
+ // If the spill slot value is available, and this is a new definition of
+ // the value, the value is not available anymore.
+ if (MR & VirtRegMap::isMod) {
+ // Notice that the value in this stack slot has been modified.
+ Spills.ModifyStackSlot(SS);
+
+ // If this is *just* a mod of the value, check to see if this is just a
+ // store to the spill slot (i.e. the spill got merged into the copy). If
+ // so, realize that the vreg is available now, and add the store to the
+ // MaybeDeadStore info.
+ int StackSlot;
+ if (!(MR & VirtRegMap::isRef)) {
+ if (unsigned SrcReg = TII->isStoreToStackSlot(&MI, StackSlot)) {
+ assert(MRegisterInfo::isPhysicalRegister(SrcReg) &&
+ "Src hasn't been allocated yet?");
+ // Okay, this is certainly a store of SrcReg to [StackSlot]. Mark
+ // this as a potentially dead store in case there is a subsequent
+ // store into the stack slot without a read from it.
+ MaybeDeadStores[StackSlot] = &MI;
+
+ // If the stack slot value was previously available in some other
+ // register, change it now. Otherwise, make the register available,
+ // in PhysReg.
+ Spills.addAvailable(StackSlot, &MI, SrcReg, false/*don't clobber*/);
+ }
+ }
+ }
+ }
+
+ // Process all of the spilled defs.
+ for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
+ MachineOperand &MO = MI.getOperand(i);
+ if (MO.isRegister() && MO.getReg() && MO.isDef()) {
+ unsigned VirtReg = MO.getReg();
+
+ if (!MRegisterInfo::isVirtualRegister(VirtReg)) {
+ // Check to see if this is a noop copy. If so, eliminate the
+ // instruction before considering the dest reg to be changed.
+ unsigned Src, Dst;
+ if (TII->isMoveInstr(MI, Src, Dst) && Src == Dst) {
+ ++NumDCE;
+ DOUT << "Removing now-noop copy: " << MI;
+ MBB.erase(&MI);
+ Erased = true;
+ VRM.RemoveFromFoldedVirtMap(&MI);
+ Spills.disallowClobberPhysReg(VirtReg);
+ goto ProcessNextInst;
+ }
+
+ // If it's not a no-op copy, it clobbers the value in the destreg.
+ Spills.ClobberPhysReg(VirtReg);
+ ReusedOperands.markClobbered(VirtReg);
+
+ // Check to see if this instruction is a load from a stack slot into
+ // a register. If so, this provides the stack slot value in the reg.
+ int FrameIdx;
+ if (unsigned DestReg = TII->isLoadFromStackSlot(&MI, FrameIdx)) {
+ assert(DestReg == VirtReg && "Unknown load situation!");
+
+ // Otherwise, if it wasn't available, remember that it is now!
+ Spills.addAvailable(FrameIdx, &MI, DestReg);
+ goto ProcessNextInst;
+ }
+
+ continue;
+ }
+
+ // The only vregs left are stack slot definitions.
+ int StackSlot = VRM.getStackSlot(VirtReg);
+ const TargetRegisterClass *RC = MF.getSSARegMap()->getRegClass(VirtReg);
+
+ // If this def is part of a two-address operand, make sure to execute
+ // the store from the correct physical register.
+ unsigned PhysReg;
+ int TiedOp = MI.getInstrDescriptor()->findTiedToSrcOperand(i);
+ if (TiedOp != -1)
+ PhysReg = MI.getOperand(TiedOp).getReg();
+ else {
+ PhysReg = VRM.getPhys(VirtReg);
+ if (ReusedOperands.isClobbered(PhysReg)) {
+ // Another def has taken the assigned physreg. It must have been a
+ // use&def which got it due to reuse. Undo the reuse!
+ PhysReg = ReusedOperands.GetRegForReload(PhysReg, &MI,
+ Spills, MaybeDeadStores, RegKills, KillOps);
+ }
+ }
+
+ MF.setPhysRegUsed(PhysReg);
+ ReusedOperands.markClobbered(PhysReg);
+ MRI->storeRegToStackSlot(MBB, next(MII), PhysReg, StackSlot, RC);
+ DOUT << "Store:\t" << *next(MII);
+ MI.getOperand(i).setReg(PhysReg);
+
+ // If there is a dead store to this stack slot, nuke it now.
+ MachineInstr *&LastStore = MaybeDeadStores[StackSlot];
+ if (LastStore) {
+ DOUT << "Removed dead store:\t" << *LastStore;
+ ++NumDSE;
+ InvalidateKills(*LastStore, RegKills, KillOps);
+ MBB.erase(LastStore);
+ VRM.RemoveFromFoldedVirtMap(LastStore);
+ }
+ LastStore = next(MII);
+
+ // If the stack slot value was previously available in some other
+ // register, change it now. Otherwise, make the register available,
+ // in PhysReg.
+ Spills.ModifyStackSlot(StackSlot);
+ Spills.ClobberPhysReg(PhysReg);
+ Spills.addAvailable(StackSlot, LastStore, PhysReg);
+ ++NumStores;
+
+ // Check to see if this is a noop copy. If so, eliminate the
+ // instruction before considering the dest reg to be changed.
+ {
+ unsigned Src, Dst;
+ if (TII->isMoveInstr(MI, Src, Dst) && Src == Dst) {
+ ++NumDCE;
+ DOUT << "Removing now-noop copy: " << MI;
+ MBB.erase(&MI);
+ Erased = true;
+ VRM.RemoveFromFoldedVirtMap(&MI);
+ UpdateKills(*LastStore, RegKills, KillOps);
+ goto ProcessNextInst;
+ }
+ }
+ }
+ }
+ ProcessNextInst:
+ if (!Erased && !BackTracked)
+ for (MachineBasicBlock::iterator II = MI; II != NextMII; ++II)
+ UpdateKills(*II, RegKills, KillOps);
+ MII = NextMII;
+ }
+}
+
+
+llvm::Spiller* llvm::createSpiller() {
+ switch (SpillerOpt) {
+ default: assert(0 && "Unreachable!");
+ case local:
+ return new LocalSpiller();
+ case simple:
+ return new SimpleSpiller();
+ }
+}
diff --git a/lib/CodeGen/VirtRegMap.h b/lib/CodeGen/VirtRegMap.h
new file mode 100644
index 0000000..b7cbe51c
--- /dev/null
+++ b/lib/CodeGen/VirtRegMap.h
@@ -0,0 +1,211 @@
+//===-- llvm/CodeGen/VirtRegMap.h - Virtual Register Map -*- C++ -*--------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file was developed by the LLVM research group and is distributed under
+// the University of Illinois Open Source License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements a virtual register map. This maps virtual registers to
+// physical registers and virtual registers to stack slots. It is created and
+// updated by a register allocator and then used by a machine code rewriter that
+// adds spill code and rewrites virtual into physical register references.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CODEGEN_VIRTREGMAP_H
+#define LLVM_CODEGEN_VIRTREGMAP_H
+
+#include "llvm/Target/MRegisterInfo.h"
+#include "llvm/ADT/BitVector.h"
+#include "llvm/ADT/IndexedMap.h"
+#include "llvm/Support/Streams.h"
+#include <map>
+
+namespace llvm {
+ class MachineInstr;
+ class TargetInstrInfo;
+
+ class VirtRegMap {
+ public:
+ enum {
+ NO_PHYS_REG = 0,
+ NO_STACK_SLOT = (1L << 30)-1,
+ MAX_STACK_SLOT = (1L << 18)-1
+ };
+
+ enum ModRef { isRef = 1, isMod = 2, isModRef = 3 };
+ typedef std::multimap<MachineInstr*,
+ std::pair<unsigned, ModRef> > MI2VirtMapTy;
+
+ private:
+ const TargetInstrInfo &TII;
+
+ MachineFunction &MF;
+ /// Virt2PhysMap - This is a virtual to physical register
+ /// mapping. Each virtual register is required to have an entry in
+ /// it; even spilled virtual registers (the register mapped to a
+ /// spilled register is the temporary used to load it from the
+ /// stack).
+ IndexedMap<unsigned, VirtReg2IndexFunctor> Virt2PhysMap;
+ /// Virt2StackSlotMap - This is virtual register to stack slot
+ /// mapping. Each spilled virtual register has an entry in it
+ /// which corresponds to the stack slot this register is spilled
+ /// at.
+ IndexedMap<int, VirtReg2IndexFunctor> Virt2StackSlotMap;
+ /// MI2VirtMap - This is MachineInstr to virtual register
+ /// mapping. In the case of memory spill code being folded into
+ /// instructions, we need to know which virtual register was
+ /// read/written by this instruction.
+ MI2VirtMapTy MI2VirtMap;
+
+ /// ReMatMap - This is virtual register to re-materialized instruction
+ /// mapping. Each virtual register whose definition is going to be
+ /// re-materialized has an entry in it.
+ std::map<unsigned, const MachineInstr*> ReMatMap;
+
+ /// ReMatId - Instead of assigning a stack slot to a to be rematerialized
+ /// virtual register, an unique id is being assigned. This keeps track of
+ /// the highest id used so far. Note, this starts at (1<<18) to avoid
+ /// conflicts with stack slot numbers.
+ int ReMatId;
+
+ VirtRegMap(const VirtRegMap&); // DO NOT IMPLEMENT
+ void operator=(const VirtRegMap&); // DO NOT IMPLEMENT
+
+ public:
+ VirtRegMap(MachineFunction &mf);
+
+ void grow();
+
+ /// @brief returns true if the specified virtual register is
+ /// mapped to a physical register
+ bool hasPhys(unsigned virtReg) const {
+ return getPhys(virtReg) != NO_PHYS_REG;
+ }
+
+ /// @brief returns the physical register mapped to the specified
+ /// virtual register
+ unsigned getPhys(unsigned virtReg) const {
+ assert(MRegisterInfo::isVirtualRegister(virtReg));
+ return Virt2PhysMap[virtReg];
+ }
+
+ /// @brief creates a mapping for the specified virtual register to
+ /// the specified physical register
+ void assignVirt2Phys(unsigned virtReg, unsigned physReg) {
+ assert(MRegisterInfo::isVirtualRegister(virtReg) &&
+ MRegisterInfo::isPhysicalRegister(physReg));
+ assert(Virt2PhysMap[virtReg] == NO_PHYS_REG &&
+ "attempt to assign physical register to already mapped "
+ "virtual register");
+ Virt2PhysMap[virtReg] = physReg;
+ }
+
+ /// @brief clears the specified virtual register's, physical
+ /// register mapping
+ void clearVirt(unsigned virtReg) {
+ assert(MRegisterInfo::isVirtualRegister(virtReg));
+ assert(Virt2PhysMap[virtReg] != NO_PHYS_REG &&
+ "attempt to clear a not assigned virtual register");
+ Virt2PhysMap[virtReg] = NO_PHYS_REG;
+ }
+
+ /// @brief clears all virtual to physical register mappings
+ void clearAllVirt() {
+ Virt2PhysMap.clear();
+ grow();
+ }
+
+ /// @brief returns true is the specified virtual register is
+ /// mapped to a stack slot
+ bool hasStackSlot(unsigned virtReg) const {
+ return getStackSlot(virtReg) != NO_STACK_SLOT;
+ }
+
+ /// @brief returns the stack slot mapped to the specified virtual
+ /// register
+ int getStackSlot(unsigned virtReg) const {
+ assert(MRegisterInfo::isVirtualRegister(virtReg));
+ return Virt2StackSlotMap[virtReg];
+ }
+
+ /// @brief create a mapping for the specifed virtual register to
+ /// the next available stack slot
+ int assignVirt2StackSlot(unsigned virtReg);
+ /// @brief create a mapping for the specified virtual register to
+ /// the specified stack slot
+ void assignVirt2StackSlot(unsigned virtReg, int frameIndex);
+
+ /// @brief assign an unique re-materialization id to the specified
+ /// virtual register.
+ int assignVirtReMatId(unsigned virtReg);
+
+ /// @brief returns true if the specified virtual register is being
+ /// re-materialized.
+ bool isReMaterialized(unsigned virtReg) const {
+ return ReMatMap.count(virtReg) != 0;
+ }
+
+ /// @brief returns the original machine instruction being re-issued
+ /// to re-materialize the specified virtual register.
+ const MachineInstr *getReMaterializedMI(unsigned virtReg) {
+ return ReMatMap[virtReg];
+ }
+
+ /// @brief records the specified virtual register will be
+ /// re-materialized and the original instruction which will be re-issed
+ /// for this purpose.
+ void setVirtIsReMaterialized(unsigned virtReg, MachineInstr *def) {
+ ReMatMap[virtReg] = def;
+ }
+
+ /// @brief Updates information about the specified virtual register's value
+ /// folded into newMI machine instruction. The OpNum argument indicates the
+ /// operand number of OldMI that is folded.
+ void virtFolded(unsigned VirtReg, MachineInstr *OldMI, unsigned OpNum,
+ MachineInstr *NewMI);
+
+ /// @brief returns the virtual registers' values folded in memory
+ /// operands of this instruction
+ std::pair<MI2VirtMapTy::const_iterator, MI2VirtMapTy::const_iterator>
+ getFoldedVirts(MachineInstr* MI) const {
+ return MI2VirtMap.equal_range(MI);
+ }
+
+ /// RemoveFromFoldedVirtMap - If the specified machine instruction is in
+ /// the folded instruction map, remove its entry from the map.
+ void RemoveFromFoldedVirtMap(MachineInstr *MI) {
+ MI2VirtMap.erase(MI);
+ }
+
+ void print(std::ostream &OS) const;
+ void print(std::ostream *OS) const { if (OS) print(*OS); }
+ void dump() const;
+ };
+
+ inline std::ostream *operator<<(std::ostream *OS, const VirtRegMap &VRM) {
+ VRM.print(OS);
+ return OS;
+ }
+ inline std::ostream &operator<<(std::ostream &OS, const VirtRegMap &VRM) {
+ VRM.print(OS);
+ return OS;
+ }
+
+ /// Spiller interface: Implementations of this interface assign spilled
+ /// virtual registers to stack slots, rewriting the code.
+ struct Spiller {
+ virtual ~Spiller();
+ virtual bool runOnMachineFunction(MachineFunction &MF,
+ VirtRegMap &VRM) = 0;
+ };
+
+ /// createSpiller - Create an return a spiller object, as specified on the
+ /// command line.
+ Spiller* createSpiller();
+
+} // End llvm namespace
+
+#endif