diff --git a/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
index 5cb13e3..2df363e 100644
--- a/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ b/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -22,6 +22,7 @@
 #include "llvm/Target/TargetData.h"
 #include "llvm/Target/TargetMachine.h"
 #include "llvm/Target/TargetOptions.h"
+#include "llvm/Target/TargetSubtarget.h"
 #include "llvm/CallingConv.h"
 #include "llvm/Constants.h"
 #include "llvm/DerivedTypes.h"
@@ -2842,123 +2843,6 @@
       break;
     }
     break;
-  case ISD::MEMSET:
-  case ISD::MEMCPY:
-  case ISD::MEMMOVE: {
-    Tmp1 = LegalizeOp(Node->getOperand(0));      // Chain
-    Tmp2 = LegalizeOp(Node->getOperand(1));      // Pointer
-
-    if (Node->getOpcode() == ISD::MEMSET) {      // memset = ubyte
-      switch (getTypeAction(Node->getOperand(2).getValueType())) {
-      case Expand: assert(0 && "Cannot expand a byte!");
-      case Legal:
-        Tmp3 = LegalizeOp(Node->getOperand(2));
-        break;
-      case Promote:
-        Tmp3 = PromoteOp(Node->getOperand(2));
-        break;
-      }
-    } else {
-      Tmp3 = LegalizeOp(Node->getOperand(2));    // memcpy/move = pointer,
-    }
-
-    SDOperand Tmp4;
-    switch (getTypeAction(Node->getOperand(3).getValueType())) {
-    case Expand: {
-      // Length is too big, just take the lo-part of the length.
-      SDOperand HiPart;
-      ExpandOp(Node->getOperand(3), Tmp4, HiPart);
-      break;
-    }
-    case Legal:
-      Tmp4 = LegalizeOp(Node->getOperand(3));
-      break;
-    case Promote:
-      Tmp4 = PromoteOp(Node->getOperand(3));
-      break;
-    }
-
-    SDOperand Tmp5;
-    switch (getTypeAction(Node->getOperand(4).getValueType())) {  // uint
-    case Expand: assert(0 && "Cannot expand this yet!");
-    case Legal:
-      Tmp5 = LegalizeOp(Node->getOperand(4));
-      break;
-    case Promote:
-      Tmp5 = PromoteOp(Node->getOperand(4));
-      break;
-    }
-
-    SDOperand Tmp6;
-    switch (getTypeAction(Node->getOperand(5).getValueType())) {  // bool
-    case Expand: assert(0 && "Cannot expand this yet!");
-    case Legal:
-      Tmp6 = LegalizeOp(Node->getOperand(5));
-      break;
-    case Promote:
-      Tmp6 = PromoteOp(Node->getOperand(5));
-      break;
-    }
-
-    switch (TLI.getOperationAction(Node->getOpcode(), MVT::Other)) {
-    default: assert(0 && "This action not implemented for this operation!");
-    case TargetLowering::Custom:
-      isCustom = true;
-      // FALLTHROUGH
-    case TargetLowering::Legal: {
-      SDOperand Ops[] = { Tmp1, Tmp2, Tmp3, Tmp4, Tmp5, Tmp6 };
-      Result = DAG.UpdateNodeOperands(Result, Ops, 6);
-      if (isCustom) {
-        Tmp1 = TLI.LowerOperation(Result, DAG);
-        if (Tmp1.Val) Result = Tmp1;
-      }
-      break;
-    }
-    case TargetLowering::Expand: {
-      // Otherwise, the target does not support this operation.  Lower the
-      // operation to an explicit libcall as appropriate.
-      MVT::ValueType IntPtr = TLI.getPointerTy();
-      const Type *IntPtrTy = TLI.getTargetData()->getIntPtrType();
-      TargetLowering::ArgListTy Args;
-      TargetLowering::ArgListEntry Entry;
-
-      const char *FnName = 0;
-      if (Node->getOpcode() == ISD::MEMSET) {
-        Entry.Node = Tmp2; Entry.Ty = IntPtrTy;
-        Args.push_back(Entry);
-        // Extend the (previously legalized) ubyte argument to be an int value
-        // for the call.
-        if (Tmp3.getValueType() > MVT::i32)
-          Tmp3 = DAG.getNode(ISD::TRUNCATE, MVT::i32, Tmp3);
-        else
-          Tmp3 = DAG.getNode(ISD::ZERO_EXTEND, MVT::i32, Tmp3);
-        Entry.Node = Tmp3; Entry.Ty = Type::Int32Ty; Entry.isSExt = true;
-        Args.push_back(Entry);
-        Entry.Node = Tmp4; Entry.Ty = IntPtrTy; Entry.isSExt = false;
-        Args.push_back(Entry);
-
-        FnName = "memset";
-      } else if (Node->getOpcode() == ISD::MEMCPY ||
-                 Node->getOpcode() == ISD::MEMMOVE) {
-        Entry.Ty = IntPtrTy;
-        Entry.Node = Tmp2; Args.push_back(Entry);
-        Entry.Node = Tmp3; Args.push_back(Entry);
-        Entry.Node = Tmp4; Args.push_back(Entry);
-        FnName = Node->getOpcode() == ISD::MEMMOVE ? "memmove" : "memcpy";
-      } else {
-        assert(0 && "Unknown op!");
-      }
-
-      std::pair<SDOperand,SDOperand> CallResult =
-        TLI.LowerCallTo(Tmp1, Type::VoidTy,
-                        false, false, false, CallingConv::C, false,
-                        DAG.getExternalSymbol(FnName, IntPtr), Args, DAG);
-      Result = CallResult.second;
-      break;
-    }
-    }
-    break;
-  }
 
   case ISD::SHL_PARTS:
   case ISD::SRA_PARTS:
diff --git a/lib/CodeGen/SelectionDAG/LegalizeTypes.cpp b/lib/CodeGen/SelectionDAG/LegalizeTypes.cpp
index 6511cff..380c422 100644
--- a/lib/CodeGen/SelectionDAG/LegalizeTypes.cpp
+++ b/lib/CodeGen/SelectionDAG/LegalizeTypes.cpp
@@ -439,51 +439,6 @@
   return DAG.getLoad(DestVT, Store, FIPtr, NULL, 0);
 }
 
-/// HandleMemIntrinsic - This handles memcpy/memset/memmove with invalid
-/// operands.  This promotes or expands the operands as required.
-SDOperand DAGTypeLegalizer::HandleMemIntrinsic(SDNode *N) {
-  // The chain and pointer [operands #0 and #1] are always valid types.
-  SDOperand Chain = N->getOperand(0);
-  SDOperand Ptr   = N->getOperand(1);
-  SDOperand Op2   = N->getOperand(2);
-  
-  // Op #2 is either a value (memset) or a pointer.  Promote it if required.
-  switch (getTypeAction(Op2.getValueType())) {
-  default: assert(0 && "Unknown action for pointer/value operand");
-  case Legal: break;
-  case Promote: Op2 = GetPromotedOp(Op2); break;
-  }
-
-  // The length could have any action required.
-  SDOperand Length = N->getOperand(3);
-  switch (getTypeAction(Length.getValueType())) {
-  default: assert(0 && "Unknown action for memop operand");
-  case Legal: break;
-  case Promote: Length = GetPromotedZExtOp(Length); break;
-  case Expand:
-    SDOperand Dummy;  // discard the high part.
-    GetExpandedOp(Length, Length, Dummy);
-    break;
-  }
-
-  SDOperand Align = N->getOperand(4);
-  switch (getTypeAction(Align.getValueType())) {
-  default: assert(0 && "Unknown action for memop operand");
-  case Legal: break;
-  case Promote: Align = GetPromotedZExtOp(Align); break;
-  }
-
-  SDOperand AlwaysInline = N->getOperand(5);
-  switch (getTypeAction(AlwaysInline.getValueType())) {
-  default: assert(0 && "Unknown action for memop operand");
-  case Legal: break;
-  case Promote: AlwaysInline = GetPromotedZExtOp(AlwaysInline); break;
-  }
-
-  SDOperand Ops[] = { Chain, Ptr, Op2, Length, Align, AlwaysInline };
-  return DAG.UpdateNodeOperands(SDOperand(N, 0), Ops, 6);
-}
-
 /// JoinIntegers - Build an integer with low bits Lo and high bits Hi.
 SDOperand DAGTypeLegalizer::JoinIntegers(SDOperand Lo, SDOperand Hi) {
   MVT::ValueType LVT = Lo.getValueType();
diff --git a/lib/CodeGen/SelectionDAG/LegalizeTypes.h b/lib/CodeGen/SelectionDAG/LegalizeTypes.h
index 7d245ab..5b98793 100644
--- a/lib/CodeGen/SelectionDAG/LegalizeTypes.h
+++ b/lib/CodeGen/SelectionDAG/LegalizeTypes.h
@@ -165,7 +165,6 @@
   // Common routines.
   SDOperand BitConvertToInteger(SDOperand Op);
   SDOperand CreateStackStoreLoad(SDOperand Op, MVT::ValueType DestVT);
-  SDOperand HandleMemIntrinsic(SDNode *N);
   SDOperand JoinIntegers(SDOperand Lo, SDOperand Hi);
   void SplitInteger(SDOperand Op, SDOperand &Lo, SDOperand &Hi);
   void SplitInteger(SDOperand Op, MVT::ValueType LoVT, MVT::ValueType HiVT,
diff --git a/lib/CodeGen/SelectionDAG/LegalizeTypesExpand.cpp b/lib/CodeGen/SelectionDAG/LegalizeTypesExpand.cpp
index b872a44..fcde8f3 100644
--- a/lib/CodeGen/SelectionDAG/LegalizeTypesExpand.cpp
+++ b/lib/CodeGen/SelectionDAG/LegalizeTypesExpand.cpp
@@ -946,9 +946,6 @@
     case ISD::STORE:
       Res = ExpandOperand_STORE(cast<StoreSDNode>(N), OpNo);
       break;
-    case ISD::MEMSET:
-    case ISD::MEMCPY:
-    case ISD::MEMMOVE:     Res = HandleMemIntrinsic(N); break;
 
     case ISD::BUILD_VECTOR: Res = ExpandOperand_BUILD_VECTOR(N); break;
     }
diff --git a/lib/CodeGen/SelectionDAG/LegalizeTypesPromote.cpp b/lib/CodeGen/SelectionDAG/LegalizeTypesPromote.cpp
index b8118eb..93c8c60 100644
--- a/lib/CodeGen/SelectionDAG/LegalizeTypesPromote.cpp
+++ b/lib/CodeGen/SelectionDAG/LegalizeTypesPromote.cpp
@@ -447,9 +447,6 @@
 
   case ISD::STORE:       Res = PromoteOperand_STORE(cast<StoreSDNode>(N),
                                                     OpNo); break;
-  case ISD::MEMSET:
-  case ISD::MEMCPY:
-  case ISD::MEMMOVE:     Res = HandleMemIntrinsic(N); break;
 
   case ISD::BUILD_VECTOR: Res = PromoteOperand_BUILD_VECTOR(N); break;
   case ISD::INSERT_VECTOR_ELT:
diff --git a/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index f096c70..327a8fe 100644
--- a/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -17,6 +17,7 @@
 #include "llvm/Intrinsics.h"
 #include "llvm/DerivedTypes.h"
 #include "llvm/Assembly/Writer.h"
+#include "llvm/CallingConv.h"
 #include "llvm/CodeGen/MachineBasicBlock.h"
 #include "llvm/CodeGen/MachineConstantPool.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
@@ -2385,28 +2386,357 @@
   return getNode(Opcode, VT, Ops, 5);
 }
 
-SDOperand SelectionDAG::getMemcpy(SDOperand Chain, SDOperand Dest,
-                                  SDOperand Src, SDOperand Size,
-                                  SDOperand Align,
-                                  SDOperand AlwaysInline) {
-  SDOperand Ops[] = { Chain, Dest, Src, Size, Align, AlwaysInline };
-  return getNode(ISD::MEMCPY, MVT::Other, Ops, 6);
+/// getMemsetValue - Vectorized representation of the memset value
+/// operand.
+static SDOperand getMemsetValue(SDOperand Value, MVT::ValueType VT,
+                                SelectionDAG &DAG) {
+  MVT::ValueType CurVT = VT;
+  if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Value)) {
+    uint64_t Val   = C->getValue() & 255;
+    unsigned Shift = 8;
+    while (CurVT != MVT::i8) {
+      Val = (Val << Shift) | Val;
+      Shift <<= 1;
+      CurVT = (MVT::ValueType)((unsigned)CurVT - 1);
+    }
+    return DAG.getConstant(Val, VT);
+  } else {
+    Value = DAG.getNode(ISD::ZERO_EXTEND, VT, Value);
+    unsigned Shift = 8;
+    while (CurVT != MVT::i8) {
+      Value =
+        DAG.getNode(ISD::OR, VT,
+                    DAG.getNode(ISD::SHL, VT, Value,
+                                DAG.getConstant(Shift, MVT::i8)), Value);
+      Shift <<= 1;
+      CurVT = (MVT::ValueType)((unsigned)CurVT - 1);
+    }
+
+    return Value;
+  }
 }
 
-SDOperand SelectionDAG::getMemmove(SDOperand Chain, SDOperand Dest,
-                                  SDOperand Src, SDOperand Size,
-                                  SDOperand Align,
-                                  SDOperand AlwaysInline) {
-  SDOperand Ops[] = { Chain, Dest, Src, Size, Align, AlwaysInline };
-  return getNode(ISD::MEMMOVE, MVT::Other, Ops, 6);
+/// getMemsetStringVal - Similar to getMemsetValue. Except this is only
+/// used when a memcpy is turned into a memset when the source is a constant
+/// string ptr.
+static SDOperand getMemsetStringVal(MVT::ValueType VT,
+                                    SelectionDAG &DAG,
+                                    const TargetLowering &TLI,
+                                    std::string &Str, unsigned Offset) {
+  uint64_t Val = 0;
+  unsigned MSB = MVT::getSizeInBits(VT) / 8;
+  if (TLI.isLittleEndian())
+    Offset = Offset + MSB - 1;
+  for (unsigned i = 0; i != MSB; ++i) {
+    Val = (Val << 8) | (unsigned char)Str[Offset];
+    Offset += TLI.isLittleEndian() ? -1 : 1;
+  }
+  return DAG.getConstant(Val, VT);
 }
 
-SDOperand SelectionDAG::getMemset(SDOperand Chain, SDOperand Dest,
+/// getMemBasePlusOffset - Returns base and offset node for the 
+static SDOperand getMemBasePlusOffset(SDOperand Base, unsigned Offset,
+                                      SelectionDAG &DAG) {
+  MVT::ValueType VT = Base.getValueType();
+  return DAG.getNode(ISD::ADD, VT, Base, DAG.getConstant(Offset, VT));
+}
+
+/// MeetsMaxMemopRequirement - Determines if the number of memory ops required
+/// to replace the memset / memcpy is below the threshold. It also returns the
+/// types of the sequence of memory ops to perform memset / memcpy.
+static bool MeetsMaxMemopRequirement(std::vector<MVT::ValueType> &MemOps,
+                                     unsigned Limit, uint64_t Size,
+                                     unsigned Align,
+                                     const TargetLowering &TLI) {
+  MVT::ValueType VT;
+
+  if (TLI.allowsUnalignedMemoryAccesses()) {
+    VT = MVT::i64;
+  } else {
+    switch (Align & 7) {
+    case 0:
+      VT = MVT::i64;
+      break;
+    case 4:
+      VT = MVT::i32;
+      break;
+    case 2:
+      VT = MVT::i16;
+      break;
+    default:
+      VT = MVT::i8;
+      break;
+    }
+  }
+
+  MVT::ValueType LVT = MVT::i64;
+  while (!TLI.isTypeLegal(LVT))
+    LVT = (MVT::ValueType)((unsigned)LVT - 1);
+  assert(MVT::isInteger(LVT));
+
+  if (VT > LVT)
+    VT = LVT;
+
+  unsigned NumMemOps = 0;
+  while (Size != 0) {
+    unsigned VTSize = MVT::getSizeInBits(VT) / 8;
+    while (VTSize > Size) {
+      VT = (MVT::ValueType)((unsigned)VT - 1);
+      VTSize >>= 1;
+    }
+    assert(MVT::isInteger(VT));
+
+    if (++NumMemOps > Limit)
+      return false;
+    MemOps.push_back(VT);
+    Size -= VTSize;
+  }
+
+  return true;
+}
+
+static SDOperand getMemcpyLoadsAndStores(SelectionDAG &DAG,
+                                         SDOperand Chain, SDOperand Dst,
+                                         SDOperand Src, uint64_t Size,
+                                         unsigned Align,
+                                         bool AlwaysInline,
+                                         Value *DstSV, uint64_t DstOff,
+                                         Value *SrcSV, uint64_t SrcOff) {
+  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
+
+  // Expand memcpy to a series of store ops if the size operand falls below
+  // a certain threshold.
+  std::vector<MVT::ValueType> MemOps;
+  uint64_t Limit = -1;
+  if (!AlwaysInline)
+    Limit = TLI.getMaxStoresPerMemcpy();
+  if (!MeetsMaxMemopRequirement(MemOps, Limit, Size, Align, TLI))
+    return SDOperand();
+
+  SmallVector<SDOperand, 8> OutChains;
+
+  unsigned NumMemOps = MemOps.size();
+  unsigned SrcDelta = 0;
+  GlobalAddressSDNode *G = NULL;
+  std::string Str;
+  bool CopyFromStr = false;
+
+  if (Src.getOpcode() == ISD::GlobalAddress)
+    G = cast<GlobalAddressSDNode>(Src);
+  else if (Src.getOpcode() == ISD::ADD &&
+           Src.getOperand(0).getOpcode() == ISD::GlobalAddress &&
+           Src.getOperand(1).getOpcode() == ISD::Constant) {
+    G = cast<GlobalAddressSDNode>(Src.getOperand(0));
+    SrcDelta = cast<ConstantSDNode>(Src.getOperand(1))->getValue();
+  }
+  if (G) {
+    GlobalVariable *GV = dyn_cast<GlobalVariable>(G->getGlobal());
+    if (GV && GV->isConstant()) {
+      Str = GV->getStringValue(false);
+      if (!Str.empty()) {
+        CopyFromStr = true;
+        SrcOff += SrcDelta;
+      }
+    }
+  }
+
+  for (unsigned i = 0; i < NumMemOps; i++) {
+    MVT::ValueType VT = MemOps[i];
+    unsigned VTSize = MVT::getSizeInBits(VT) / 8;
+    SDOperand Value, Store;
+
+    if (CopyFromStr) {
+      Value = getMemsetStringVal(VT, DAG, TLI, Str, SrcOff);
+      Store =
+        DAG.getStore(Chain, Value,
+                     getMemBasePlusOffset(Dst, DstOff, DAG),
+                     DstSV, DstOff);
+    } else {
+      Value = DAG.getLoad(VT, Chain,
+                          getMemBasePlusOffset(Src, SrcOff, DAG),
+                          SrcSV, SrcOff, false, Align);
+      Store =
+        DAG.getStore(Chain, Value,
+                     getMemBasePlusOffset(Dst, DstOff, DAG),
+                     DstSV, DstOff, false, Align);
+    }
+    OutChains.push_back(Store);
+    SrcOff += VTSize;
+    DstOff += VTSize;
+  }
+
+  return DAG.getNode(ISD::TokenFactor, MVT::Other,
+                     &OutChains[0], OutChains.size());
+}
+
+static SDOperand getMemsetStores(SelectionDAG &DAG,
+                                 SDOperand Chain, SDOperand Dst,
+                                 SDOperand Src, uint64_t Size,
+                                 unsigned Align,
+                                 Value *DstSV, uint64_t DstOff) {
+  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
+
+  // Expand memset to a series of load/store ops if the size operand
+  // falls below a certain threshold.
+  std::vector<MVT::ValueType> MemOps;
+  if (!MeetsMaxMemopRequirement(MemOps, TLI.getMaxStoresPerMemset(),
+                                Size, Align, TLI))
+    return SDOperand();
+
+  SmallVector<SDOperand, 8> OutChains;
+
+  unsigned NumMemOps = MemOps.size();
+  for (unsigned i = 0; i < NumMemOps; i++) {
+    MVT::ValueType VT = MemOps[i];
+    unsigned VTSize = MVT::getSizeInBits(VT) / 8;
+    SDOperand Value = getMemsetValue(Src, VT, DAG);
+    SDOperand Store = DAG.getStore(Chain, Value,
+                                   getMemBasePlusOffset(Dst, DstOff, DAG),
+                                   DstSV, DstOff);
+    OutChains.push_back(Store);
+    DstOff += VTSize;
+  }
+
+  return DAG.getNode(ISD::TokenFactor, MVT::Other,
+                     &OutChains[0], OutChains.size());
+}
+
+SDOperand SelectionDAG::getMemcpy(SDOperand Chain, SDOperand Dst,
                                   SDOperand Src, SDOperand Size,
-                                  SDOperand Align,
-                                  SDOperand AlwaysInline) {
-  SDOperand Ops[] = { Chain, Dest, Src, Size, Align, AlwaysInline };
-  return getNode(ISD::MEMSET, MVT::Other, Ops, 6);
+                                  unsigned Align, bool AlwaysInline,
+                                  Value *DstSV, uint64_t DstOff,
+                                  Value *SrcSV, uint64_t SrcOff) {
+
+  // Check to see if we should lower the memcpy to loads and stores first.
+  // For cases within the target-specified limits, this is the best choice.
+  ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
+  if (ConstantSize) {
+    // Memcpy with size zero? Just return the original chain.
+    if (ConstantSize->isNullValue())
+      return Chain;
+
+    SDOperand Result =
+      getMemcpyLoadsAndStores(*this, Chain, Dst, Src, ConstantSize->getValue(),
+                              Align, false, DstSV, DstOff, SrcSV, SrcOff);
+    if (Result.Val)
+      return Result;
+  }
+
+  // Then check to see if we should lower the memcpy with target-specific
+  // code. If the target chooses to do this, this is the next best.
+  SDOperand Result =
+    TLI.EmitTargetCodeForMemcpy(*this, Chain, Dst, Src, Size, Align,
+                                AlwaysInline,
+                                DstSV, DstOff, SrcSV, SrcOff);
+  if (Result.Val)
+    return Result;
+
+  // If we really need inline code and the target declined to provide it,
+  // use a (potentially long) sequence of loads and stores.
+  if (AlwaysInline) {
+    assert(ConstantSize && "AlwaysInline requires a constant size!");
+    return getMemcpyLoadsAndStores(*this, Chain, Dst, Src,
+                                   ConstantSize->getValue(), Align, true,
+                                   DstSV, DstOff, SrcSV, SrcOff);
+  }
+
+  // Emit a library call.
+  TargetLowering::ArgListTy Args;
+  TargetLowering::ArgListEntry Entry;
+  Entry.Ty = TLI.getTargetData()->getIntPtrType();
+  Entry.Node = Dst; Args.push_back(Entry);
+  Entry.Node = Src; Args.push_back(Entry);
+  Entry.Node = Size; Args.push_back(Entry);
+  std::pair<SDOperand,SDOperand> CallResult =
+    TLI.LowerCallTo(Chain, Type::VoidTy,
+                    false, false, false, CallingConv::C, false,
+                    getExternalSymbol("memcpy", TLI.getPointerTy()),
+                    Args, *this);
+  return CallResult.second;
+}
+
+SDOperand SelectionDAG::getMemmove(SDOperand Chain, SDOperand Dst,
+                                   SDOperand Src, SDOperand Size,
+                                   unsigned Align,
+                                   Value *DstSV, uint64_t DstOff,
+                                   Value *SrcSV, uint64_t SrcOff) {
+
+  // TODO: Optimize small memmove cases with simple loads and stores,
+  // ensuring that all loads precede all stores. This can cause severe
+  // register pressure, so targets should be careful with the size limit.
+
+  // Then check to see if we should lower the memmove with target-specific
+  // code. If the target chooses to do this, this is the next best.
+  SDOperand Result =
+    TLI.EmitTargetCodeForMemmove(*this, Chain, Dst, Src, Size, Align,
+                                 DstSV, DstOff, SrcSV, SrcOff);
+  if (Result.Val)
+    return Result;
+
+  // Emit a library call.
+  TargetLowering::ArgListTy Args;
+  TargetLowering::ArgListEntry Entry;
+  Entry.Ty = TLI.getTargetData()->getIntPtrType();
+  Entry.Node = Dst; Args.push_back(Entry);
+  Entry.Node = Src; Args.push_back(Entry);
+  Entry.Node = Size; Args.push_back(Entry);
+  std::pair<SDOperand,SDOperand> CallResult =
+    TLI.LowerCallTo(Chain, Type::VoidTy,
+                    false, false, false, CallingConv::C, false,
+                    getExternalSymbol("memmove", TLI.getPointerTy()),
+                    Args, *this);
+  return CallResult.second;
+}
+
+SDOperand SelectionDAG::getMemset(SDOperand Chain, SDOperand Dst,
+                                  SDOperand Src, SDOperand Size,
+                                  unsigned Align,
+                                  Value *DstSV, uint64_t DstOff) {
+
+  // Check to see if we should lower the memset to stores first.
+  // For cases within the target-specified limits, this is the best choice.
+  ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
+  if (ConstantSize) {
+    // Memset with size zero? Just return the original chain.
+    if (ConstantSize->isNullValue())
+      return Chain;
+
+    SDOperand Result =
+      getMemsetStores(*this, Chain, Dst, Src, ConstantSize->getValue(), Align,
+                      DstSV, DstOff);
+    if (Result.Val)
+      return Result;
+  }
+
+  // Then check to see if we should lower the memset with target-specific
+  // code. If the target chooses to do this, this is the next best.
+  SDOperand Result =
+    TLI.EmitTargetCodeForMemset(*this, Chain, Dst, Src, Size, Align,
+                                DstSV, DstOff);
+  if (Result.Val)
+    return Result;
+
+  // Emit a library call.
+  const Type *IntPtrTy = TLI.getTargetData()->getIntPtrType();
+  TargetLowering::ArgListTy Args;
+  TargetLowering::ArgListEntry Entry;
+  Entry.Node = Dst; Entry.Ty = IntPtrTy;
+  Args.push_back(Entry);
+  // Extend or truncate the argument to be an i32 value for the call.
+  if (Src.getValueType() > MVT::i32)
+    Src = getNode(ISD::TRUNCATE, MVT::i32, Src);
+  else
+    Src = getNode(ISD::ZERO_EXTEND, MVT::i32, Src);
+  Entry.Node = Src; Entry.Ty = Type::Int32Ty; Entry.isSExt = true;
+  Args.push_back(Entry);
+  Entry.Node = Size; Entry.Ty = IntPtrTy; Entry.isSExt = false;
+  Args.push_back(Entry);
+  std::pair<SDOperand,SDOperand> CallResult =
+    TLI.LowerCallTo(Chain, Type::VoidTy,
+                    false, false, false, CallingConv::C, false,
+                    getExternalSymbol("memset", TLI.getPointerTy()),
+                    Args, *this);
+  return CallResult.second;
 }
 
 SDOperand SelectionDAG::getAtomic(unsigned Opcode, SDOperand Chain, 
@@ -4009,11 +4339,6 @@
   case ISD::STACKRESTORE:       return "stackrestore";
   case ISD::TRAP:               return "trap";
 
-  // Block memory operations.
-  case ISD::MEMSET:  return "memset";
-  case ISD::MEMCPY:  return "memcpy";
-  case ISD::MEMMOVE: return "memmove";
-
   // Bit manipulation
   case ISD::BSWAP:   return "bswap";
   case ISD::CTPOP:   return "ctpop";
diff --git a/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp b/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
index cfef9ac..ac5cfd2 100644
--- a/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
+++ b/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
@@ -647,8 +647,6 @@
   void visitVAEnd(CallInst &I);
   void visitVACopy(CallInst &I);
 
-  void visitMemIntrinsic(CallInst &I, unsigned Op);
-
   void visitGetResult(GetResultInst &I);
 
   void visitUserOp1(Instruction &I) {
@@ -2737,18 +2735,48 @@
     return "_longjmp"+!TLI.usesUnderscoreLongJmp();
     break;
   case Intrinsic::memcpy_i32:
-  case Intrinsic::memcpy_i64:
-    visitMemIntrinsic(I, ISD::MEMCPY);
+  case Intrinsic::memcpy_i64: {
+    SDOperand Op1 = getValue(I.getOperand(1));
+    SDOperand Op2 = getValue(I.getOperand(2));
+    SDOperand Op3 = getValue(I.getOperand(3));
+    unsigned Align = cast<ConstantInt>(I.getOperand(4))->getZExtValue();
+    DAG.setRoot(DAG.getMemcpy(getRoot(), Op1, Op2, Op3, Align, false,
+                              I.getOperand(1), 0, I.getOperand(2), 0));
     return 0;
+  }
   case Intrinsic::memset_i32:
-  case Intrinsic::memset_i64:
-    visitMemIntrinsic(I, ISD::MEMSET);
+  case Intrinsic::memset_i64: {
+    SDOperand Op1 = getValue(I.getOperand(1));
+    SDOperand Op2 = getValue(I.getOperand(2));
+    SDOperand Op3 = getValue(I.getOperand(3));
+    unsigned Align = cast<ConstantInt>(I.getOperand(4))->getZExtValue();
+    DAG.setRoot(DAG.getMemset(getRoot(), Op1, Op2, Op3, Align,
+                              I.getOperand(1), 0));
     return 0;
+  }
   case Intrinsic::memmove_i32:
-  case Intrinsic::memmove_i64:
-    visitMemIntrinsic(I, ISD::MEMMOVE);
+  case Intrinsic::memmove_i64: {
+    SDOperand Op1 = getValue(I.getOperand(1));
+    SDOperand Op2 = getValue(I.getOperand(2));
+    SDOperand Op3 = getValue(I.getOperand(3));
+    unsigned Align = cast<ConstantInt>(I.getOperand(4))->getZExtValue();
+
+    // If the source and destination are known to not be aliases, we can
+    // lower memmove as memcpy.
+    uint64_t Size = -1ULL;
+    if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op3))
+      Size = C->getValue();
+    if (AA.alias(I.getOperand(1), Size, I.getOperand(2), Size) ==
+        AliasAnalysis::NoAlias) {
+      DAG.setRoot(DAG.getMemcpy(getRoot(), Op1, Op2, Op3, Align, false,
+                                I.getOperand(1), 0, I.getOperand(2), 0));
+      return 0;
+    }
+
+    DAG.setRoot(DAG.getMemmove(getRoot(), Op1, Op2, Op3, Align,
+                               I.getOperand(1), 0, I.getOperand(2), 0));
     return 0;
-    
+  }
   case Intrinsic::dbg_stoppoint: {
     MachineModuleInfo *MMI = DAG.getMachineModuleInfo();
     DbgStopPointInst &SPI = cast<DbgStopPointInst>(I);
@@ -4342,242 +4370,6 @@
   return SDOperand();
 }
 
-/// getMemsetValue - Vectorized representation of the memset value
-/// operand.
-static SDOperand getMemsetValue(SDOperand Value, MVT::ValueType VT,
-                                SelectionDAG &DAG) {
-  MVT::ValueType CurVT = VT;
-  if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Value)) {
-    uint64_t Val   = C->getValue() & 255;
-    unsigned Shift = 8;
-    while (CurVT != MVT::i8) {
-      Val = (Val << Shift) | Val;
-      Shift <<= 1;
-      CurVT = (MVT::ValueType)((unsigned)CurVT - 1);
-    }
-    return DAG.getConstant(Val, VT);
-  } else {
-    Value = DAG.getNode(ISD::ZERO_EXTEND, VT, Value);
-    unsigned Shift = 8;
-    while (CurVT != MVT::i8) {
-      Value =
-        DAG.getNode(ISD::OR, VT,
-                    DAG.getNode(ISD::SHL, VT, Value,
-                                DAG.getConstant(Shift, MVT::i8)), Value);
-      Shift <<= 1;
-      CurVT = (MVT::ValueType)((unsigned)CurVT - 1);
-    }
-
-    return Value;
-  }
-}
-
-/// getMemsetStringVal - Similar to getMemsetValue. Except this is only
-/// used when a memcpy is turned into a memset when the source is a constant
-/// string ptr.
-static SDOperand getMemsetStringVal(MVT::ValueType VT,
-                                    SelectionDAG &DAG, TargetLowering &TLI,
-                                    std::string &Str, unsigned Offset) {
-  uint64_t Val = 0;
-  unsigned MSB = MVT::getSizeInBits(VT) / 8;
-  if (TLI.isLittleEndian())
-    Offset = Offset + MSB - 1;
-  for (unsigned i = 0; i != MSB; ++i) {
-    Val = (Val << 8) | (unsigned char)Str[Offset];
-    Offset += TLI.isLittleEndian() ? -1 : 1;
-  }
-  return DAG.getConstant(Val, VT);
-}
-
-/// getMemBasePlusOffset - Returns base and offset node for the 
-static SDOperand getMemBasePlusOffset(SDOperand Base, unsigned Offset,
-                                      SelectionDAG &DAG, TargetLowering &TLI) {
-  MVT::ValueType VT = Base.getValueType();
-  return DAG.getNode(ISD::ADD, VT, Base, DAG.getConstant(Offset, VT));
-}
-
-/// MeetsMaxMemopRequirement - Determines if the number of memory ops required
-/// to replace the memset / memcpy is below the threshold. It also returns the
-/// types of the sequence of  memory ops to perform memset / memcpy.
-static bool MeetsMaxMemopRequirement(std::vector<MVT::ValueType> &MemOps,
-                                     unsigned Limit, uint64_t Size,
-                                     unsigned Align, TargetLowering &TLI) {
-  MVT::ValueType VT;
-
-  if (TLI.allowsUnalignedMemoryAccesses()) {
-    VT = MVT::i64;
-  } else {
-    switch (Align & 7) {
-    case 0:
-      VT = MVT::i64;
-      break;
-    case 4:
-      VT = MVT::i32;
-      break;
-    case 2:
-      VT = MVT::i16;
-      break;
-    default:
-      VT = MVT::i8;
-      break;
-    }
-  }
-
-  MVT::ValueType LVT = MVT::i64;
-  while (!TLI.isTypeLegal(LVT))
-    LVT = (MVT::ValueType)((unsigned)LVT - 1);
-  assert(MVT::isInteger(LVT));
-
-  if (VT > LVT)
-    VT = LVT;
-
-  unsigned NumMemOps = 0;
-  while (Size != 0) {
-    unsigned VTSize = MVT::getSizeInBits(VT) / 8;
-    while (VTSize > Size) {
-      VT = (MVT::ValueType)((unsigned)VT - 1);
-      VTSize >>= 1;
-    }
-    assert(MVT::isInteger(VT));
-
-    if (++NumMemOps > Limit)
-      return false;
-    MemOps.push_back(VT);
-    Size -= VTSize;
-  }
-
-  return true;
-}
-
-void SelectionDAGLowering::visitMemIntrinsic(CallInst &I, unsigned Op) {
-  SDOperand Op1 = getValue(I.getOperand(1));
-  SDOperand Op2 = getValue(I.getOperand(2));
-  SDOperand Op3 = getValue(I.getOperand(3));
-  SDOperand Op4 = getValue(I.getOperand(4));
-  unsigned Align = (unsigned)cast<ConstantSDNode>(Op4)->getValue();
-  if (Align == 0) Align = 1;
-
-  // If the source and destination are known to not be aliases, we can
-  // lower memmove as memcpy.
-  if (Op == ISD::MEMMOVE) {
-    uint64_t Size = -1ULL;
-    if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op3))
-      Size = C->getValue();
-    if (AA.alias(I.getOperand(1), Size, I.getOperand(2), Size) ==
-        AliasAnalysis::NoAlias)
-      Op = ISD::MEMCPY;
-  }
-
-  if (ConstantSDNode *Size = dyn_cast<ConstantSDNode>(Op3)) {
-    std::vector<MVT::ValueType> MemOps;
-
-    // Expand memset / memcpy to a series of load / store ops
-    // if the size operand falls below a certain threshold.
-    SmallVector<SDOperand, 8> OutChains;
-    switch (Op) {
-    default: break;  // Do nothing for now.
-    case ISD::MEMSET: {
-      if (MeetsMaxMemopRequirement(MemOps, TLI.getMaxStoresPerMemset(),
-                                   Size->getValue(), Align, TLI)) {
-        unsigned NumMemOps = MemOps.size();
-        unsigned Offset = 0;
-        for (unsigned i = 0; i < NumMemOps; i++) {
-          MVT::ValueType VT = MemOps[i];
-          unsigned VTSize = MVT::getSizeInBits(VT) / 8;
-          SDOperand Value = getMemsetValue(Op2, VT, DAG);
-          SDOperand Store = DAG.getStore(getRoot(), Value,
-                                    getMemBasePlusOffset(Op1, Offset, DAG, TLI),
-                                         I.getOperand(1), Offset);
-          OutChains.push_back(Store);
-          Offset += VTSize;
-        }
-      }
-      break;
-    }
-    case ISD::MEMCPY: {
-      if (MeetsMaxMemopRequirement(MemOps, TLI.getMaxStoresPerMemcpy(),
-                                   Size->getValue(), Align, TLI)) {
-        unsigned NumMemOps = MemOps.size();
-        unsigned SrcOff = 0, DstOff = 0, SrcDelta = 0;
-        GlobalAddressSDNode *G = NULL;
-        std::string Str;
-        bool CopyFromStr = false;
-
-        if (Op2.getOpcode() == ISD::GlobalAddress)
-          G = cast<GlobalAddressSDNode>(Op2);
-        else if (Op2.getOpcode() == ISD::ADD &&
-                 Op2.getOperand(0).getOpcode() == ISD::GlobalAddress &&
-                 Op2.getOperand(1).getOpcode() == ISD::Constant) {
-          G = cast<GlobalAddressSDNode>(Op2.getOperand(0));
-          SrcDelta = cast<ConstantSDNode>(Op2.getOperand(1))->getValue();
-        }
-        if (G) {
-          GlobalVariable *GV = dyn_cast<GlobalVariable>(G->getGlobal());
-          if (GV && GV->isConstant()) {
-            Str = GV->getStringValue(false);
-            if (!Str.empty()) {
-              CopyFromStr = true;
-              SrcOff += SrcDelta;
-            }
-          }
-        }
-
-        for (unsigned i = 0; i < NumMemOps; i++) {
-          MVT::ValueType VT = MemOps[i];
-          unsigned VTSize = MVT::getSizeInBits(VT) / 8;
-          SDOperand Value, Chain, Store;
-
-          if (CopyFromStr) {
-            Value = getMemsetStringVal(VT, DAG, TLI, Str, SrcOff);
-            Chain = getRoot();
-            Store =
-              DAG.getStore(Chain, Value,
-                           getMemBasePlusOffset(Op1, DstOff, DAG, TLI),
-                           I.getOperand(1), DstOff);
-          } else {
-            Value = DAG.getLoad(VT, getRoot(),
-                                getMemBasePlusOffset(Op2, SrcOff, DAG, TLI),
-                                I.getOperand(2), SrcOff, false, Align);
-            Chain = Value.getValue(1);
-            Store =
-              DAG.getStore(Chain, Value,
-                           getMemBasePlusOffset(Op1, DstOff, DAG, TLI),
-                           I.getOperand(1), DstOff, false, Align);
-          }
-          OutChains.push_back(Store);
-          SrcOff += VTSize;
-          DstOff += VTSize;
-        }
-      }
-      break;
-    }
-    }
-
-    if (!OutChains.empty()) {
-      DAG.setRoot(DAG.getNode(ISD::TokenFactor, MVT::Other,
-                  &OutChains[0], OutChains.size()));
-      return;
-    }
-  }
-
-  SDOperand AlwaysInline = DAG.getConstant(0, MVT::i1);
-  SDOperand Node;
-  switch(Op) {
-    default:
-      assert(0 && "Unknown Op");
-    case ISD::MEMCPY:
-      Node = DAG.getMemcpy(getRoot(), Op1, Op2, Op3, Op4, AlwaysInline);
-      break;
-    case ISD::MEMMOVE:
-      Node = DAG.getMemmove(getRoot(), Op1, Op2, Op3, Op4, AlwaysInline);
-      break;
-    case ISD::MEMSET:
-      Node = DAG.getMemset(getRoot(), Op1, Op2, Op3, Op4, AlwaysInline);
-      break;
-  }
-  DAG.setRoot(Node);
-}
-
 //===----------------------------------------------------------------------===//
 // SelectionDAGISel code
 //===----------------------------------------------------------------------===//
diff --git a/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/lib/CodeGen/SelectionDAG/TargetLowering.cpp
index a0894dd..f69f046 100644
--- a/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ b/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -17,7 +17,7 @@
 #include "llvm/Target/TargetData.h"
 #include "llvm/Target/TargetMachine.h"
 #include "llvm/Target/TargetRegisterInfo.h"
-#include "llvm/CallingConv.h"
+#include "llvm/GlobalVariable.h"
 #include "llvm/DerivedTypes.h"
 #include "llvm/CodeGen/SelectionDAG.h"
 #include "llvm/ADT/StringExtras.h"
@@ -234,59 +234,6 @@
 
 TargetLowering::~TargetLowering() {}
 
-
-SDOperand TargetLowering::LowerMEMCPY(SDOperand Op, SelectionDAG &DAG) {
-  assert(getSubtarget() && "Subtarget not defined");
-  SDOperand ChainOp = Op.getOperand(0);
-  SDOperand DestOp = Op.getOperand(1);
-  SDOperand SourceOp = Op.getOperand(2);
-  SDOperand CountOp = Op.getOperand(3);
-  SDOperand AlignOp = Op.getOperand(4);
-  SDOperand AlwaysInlineOp = Op.getOperand(5);
-
-  bool AlwaysInline = (bool)cast<ConstantSDNode>(AlwaysInlineOp)->getValue();
-  unsigned Align = (unsigned)cast<ConstantSDNode>(AlignOp)->getValue();
-  if (Align == 0) Align = 1;
-
-  // If size is unknown, call memcpy.
-  ConstantSDNode *I = dyn_cast<ConstantSDNode>(CountOp);
-  if (!I) {
-    assert(!AlwaysInline && "Cannot inline copy of unknown size");
-    return LowerMEMCPYCall(ChainOp, DestOp, SourceOp, CountOp, DAG);
-  }
-
-  // If not DWORD aligned or if size is more than threshold, then call memcpy.
-  // The libc version is likely to be faster for the following cases. It can
-  // use the address value and run time information about the CPU.
-  // With glibc 2.6.1 on a core 2, coping an array of 100M longs was 30% faster
-  unsigned Size = I->getValue();
-  if (AlwaysInline ||
-      (Size <= getSubtarget()->getMaxInlineSizeThreshold() &&
-       (Align & 3) == 0))
-    return LowerMEMCPYInline(ChainOp, DestOp, SourceOp, Size, Align, DAG);
-  return LowerMEMCPYCall(ChainOp, DestOp, SourceOp, CountOp, DAG);
-}
-
-
-SDOperand TargetLowering::LowerMEMCPYCall(SDOperand Chain,
-                                          SDOperand Dest,
-                                          SDOperand Source,
-                                          SDOperand Count,
-                                          SelectionDAG &DAG) {
-  MVT::ValueType IntPtr = getPointerTy();
-  TargetLowering::ArgListTy Args;
-  TargetLowering::ArgListEntry Entry;
-  Entry.Ty = getTargetData()->getIntPtrType();
-  Entry.Node = Dest; Args.push_back(Entry);
-  Entry.Node = Source; Args.push_back(Entry);
-  Entry.Node = Count; Args.push_back(Entry);
-  std::pair<SDOperand,SDOperand> CallResult =
-      LowerCallTo(Chain, Type::VoidTy, false, false, false, CallingConv::C,
-                  false, DAG.getExternalSymbol("memcpy", IntPtr), Args, DAG);
-  return CallResult.second;
-}
-
-
 /// computeRegisterProperties - Once all of the register classes are added,
 /// this allows us to compute derived properties we expose.
 void TargetLowering::computeRegisterProperties() {