Put the invalid flag of OwningResult into the Action pointer.
This shrinks OwningResult by one pointer. Since it is no longer larger than OwningPtr, merge the two.
This leads to simpler client code and speeds up my benchmark by 2.7%.
For some reason, this exposes a previously hidden bug, causing a regression in SemaCXX/condition.cpp.

git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@63867 91177308-0d34-0410-b5e6-96231b3b80d8
diff --git a/Driver/PrintParserCallbacks.cpp b/Driver/PrintParserCallbacks.cpp
index 90cdf45..77d7ef1 100644
--- a/Driver/PrintParserCallbacks.cpp
+++ b/Driver/PrintParserCallbacks.cpp
@@ -497,7 +497,7 @@
     virtual OwningExprResult ActOnParenExpr(SourceLocation L, SourceLocation R,
                                             ExprArg Val) {
       llvm::cout << __FUNCTION__ << "\n";
-      return move_res(Val);  // Default impl returns operand.
+      return move(Val);  // Default impl returns operand.
     }
 
     // Postfix Expressions.
diff --git a/include/clang/Parse/Action.h b/include/clang/Parse/Action.h
index 28e63b3..271a3a8 100644
--- a/include/clang/Parse/Action.h
+++ b/include/clang/Parse/Action.h
@@ -94,9 +94,15 @@
   // is complete.
 
   /// Single expressions or statements as arguments.
+#if !defined(DISABLE_SMART_POINTERS)
+  typedef ASTOwningResult<&ActionBase::DeleteExpr> ExprArg;
+  typedef ASTOwningResult<&ActionBase::DeleteStmt> StmtArg;
+  typedef ASTOwningResult<&ActionBase::DeleteTemplateArg> TemplateArgArg;
+#else
   typedef ASTOwningPtr<&ActionBase::DeleteExpr> ExprArg;
   typedef ASTOwningPtr<&ActionBase::DeleteStmt> StmtArg;
   typedef ASTOwningPtr<&ActionBase::DeleteTemplateArg> TemplateArgArg;
+#endif
 
   /// Multiple expressions or statements as arguments.
   typedef ASTMultiPtr<&ActionBase::DeleteExpr> MultiExprArg;
@@ -602,7 +608,7 @@
 
   virtual OwningExprResult ActOnParenExpr(SourceLocation L, SourceLocation R,
                                           ExprArg Val) {
-    return move_res(Val);  // Default impl returns operand.
+    return move(Val);  // Default impl returns operand.
   }
 
   // Postfix Expressions.
diff --git a/include/clang/Parse/Ownership.h b/include/clang/Parse/Ownership.h
index 704d6b5..6017db7 100644
--- a/include/clang/Parse/Ownership.h
+++ b/include/clang/Parse/Ownership.h
@@ -105,6 +105,8 @@
 // Flip this switch to measure performance impact of the smart pointers.
 //#define DISABLE_SMART_POINTERS
 
+#include "llvm/ADT/PointerIntPair.h"
+
 namespace clang
 {
   // Basic
@@ -228,9 +230,6 @@
   /// has an extra flag to indicate an additional success status.
   template <ASTDestroyer Destroyer> class ASTOwningResult;
 
-  /// ASTOwningPtr - A moveable smart pointer for AST nodes.
-  template <ASTDestroyer Destroyer> class ASTOwningPtr;
-
   /// ASTMultiPtr - A moveable smart pointer to multiple AST nodes. Only owns
   /// the individual pointers, not the array holding them.
   template <ASTDestroyer Destroyer> class ASTMultiPtr;
@@ -250,19 +249,6 @@
       ASTOwningResult<Destroyer> * operator ->() { return &Moved; }
     };
 
-    /// Move emulation helper for ASTOwningPtr. NEVER EVER use this class
-    /// directly if you don't know what you're doing.
-    template <ASTDestroyer Destroyer>
-    class ASTPtrMover
-    {
-      ASTOwningPtr<Destroyer> &Moved;
-
-    public:
-      ASTPtrMover(ASTOwningPtr<Destroyer> &moved) : Moved(moved) {}
-
-      ASTOwningPtr<Destroyer> * operator ->() { return &Moved; }
-    };
-
     /// Move emulation helper for ASTMultiPtr. NEVER EVER use this class
     /// directly if you don't know what you're doing.
     template <ASTDestroyer Destroyer>
@@ -279,91 +265,40 @@
       void release();
     };
   }
-#endif
+#else
 
+  /// Kept only as a type-safe wrapper for a void pointer, when smart pointers
+  /// are disabled. When they are enabled, ASTOwningResult takes over.
   template <ASTDestroyer Destroyer>
   class ASTOwningPtr
   {
-#if !defined(DISABLE_SMART_POINTERS)
-    ActionBase *Actions;
-#endif
     void *Node;
 
-#if !defined(DISABLE_SMART_POINTERS)
-    friend class moving::ASTPtrMover<Destroyer>;
-
-    ASTOwningPtr(ASTOwningPtr&); // DO NOT IMPLEMENT
-    ASTOwningPtr& operator =(ASTOwningPtr&); // DO NOT IMPLEMENT
-
-    void destroy() {
-      if (Node) {
-        assert(Actions && "Owning pointer without Action owns node.");
-        (Actions->*Destroyer)(Node);
-      }
-    }
-#endif
-
   public:
-#if !defined(DISABLE_SMART_POINTERS)
-    explicit ASTOwningPtr(ActionBase &actions)
-      : Actions(&actions), Node(0) {}
-    ASTOwningPtr(ActionBase &actions, void *node)
-      : Actions(&actions), Node(node) {}
-    /// Move from another owning pointer
-    ASTOwningPtr(moving::ASTPtrMover<Destroyer> mover)
-      : Actions(mover->Actions), Node(mover->take()) {}
-
-    /// Move assignment from another owning pointer
-    ASTOwningPtr & operator =(moving::ASTPtrMover<Destroyer> mover) {
-      Actions = mover->Actions;
-      Node = mover->take();
-      return *this;
-    }
-
-    /// Assignment from a raw pointer. Takes ownership - beware!
-    ASTOwningPtr & operator =(void *raw) {
-      assert((Actions || !raw) && "Cannot assign non-null raw without Action");
-      Node = raw;
-      return *this;
-    }
-#else // Different set if smart pointers are disabled
     explicit ASTOwningPtr(ActionBase &) : Node(0) {}
     ASTOwningPtr(ActionBase &, void *node) : Node(node) {}
     // Normal copying operators are defined implicitly.
-    explicit ASTOwningPtr(void *ptr) : Node(ptr) {}
+    ASTOwningPtr(const ASTOwningResult<Destroyer> &o);
 
     ASTOwningPtr & operator =(void *raw) {
       Node = raw;
       return *this;
     }
-#endif
 
     /// Access to the raw pointer.
     void * get() const { return Node; }
 
     /// Release the raw pointer.
     void * take() {
-#if !defined(DISABLE_SMART_POINTERS)
-      void *tmp = Node;
-      Node = 0;
-      return tmp;
-#else
       return Node;
-#endif
     }
 
     /// Alias for interface familiarity with unique_ptr.
     void * release() {
       return take();
     }
-
-#if !defined(DISABLE_SMART_POINTERS)
-    /// Move hook
-    operator moving::ASTPtrMover<Destroyer>() {
-      return moving::ASTPtrMover<Destroyer>(*this);
-    }
-#endif
   };
+#endif
 
   // Important: There are two different implementations of
   // ASTOwningResult below, depending on whether
@@ -374,74 +309,84 @@
   template <ASTDestroyer Destroyer>
   class ASTOwningResult
   {
-    ASTOwningPtr<Destroyer> Ptr;
-    bool Invalid;
+    llvm::PointerIntPair<ActionBase*, 1, bool> ActionInv;
+    void *Ptr;
 
     friend class moving::ASTResultMover<Destroyer>;
 
     ASTOwningResult(ASTOwningResult&); // DO NOT IMPLEMENT
     ASTOwningResult& operator =(ASTOwningResult&); // DO NOT IMPLEMENT
 
+    void destroy() {
+      if (Ptr) {
+        assert(ActionInv.getPointer() &&
+               "Smart pointer has node but no action.");
+        (ActionInv.getPointer()->*Destroyer)(Ptr);
+        Ptr = 0;
+      }
+    }
+
   public:
     typedef ActionBase::ActionResult<DestroyerToUID<Destroyer>::UID> DumbResult;
 
     explicit ASTOwningResult(ActionBase &actions, bool invalid = false)
-      : Ptr(actions, 0), Invalid(invalid) {}
+      : ActionInv(&actions, invalid), Ptr(0) {}
     ASTOwningResult(ActionBase &actions, void *node)
-      : Ptr(actions, node), Invalid(false) {}
+      : ActionInv(&actions, false), Ptr(node) {}
     ASTOwningResult(ActionBase &actions, const DumbResult &res)
-      : Ptr(actions, res.get()), Invalid(res.isInvalid()) {}
+      : ActionInv(&actions, res.isInvalid()), Ptr(res.get()) {}
     /// Move from another owning result
     ASTOwningResult(moving::ASTResultMover<Destroyer> mover)
-      : Ptr(moving::ASTPtrMover<Destroyer>(mover->Ptr)),
-        Invalid(mover->Invalid) {}
-    /// Move from an owning pointer
-    ASTOwningResult(moving::ASTPtrMover<Destroyer> mover)
-      : Ptr(mover), Invalid(false) {}
+      : ActionInv(mover->ActionInv),
+        Ptr(mover->Ptr) {
+      mover->Ptr = 0;
+    }
+
+    ~ASTOwningResult() {
+      destroy();
+    }
 
     /// Move assignment from another owning result
     ASTOwningResult & operator =(moving::ASTResultMover<Destroyer> mover) {
-      Ptr = move(mover->Ptr);
-      Invalid = mover->Invalid;
-      return *this;
-    }
-
-    /// Move assignment from an owning ptr
-    ASTOwningResult & operator =(moving::ASTPtrMover<Destroyer> mover) {
-      Ptr = mover;
-      Invalid = false;
+      destroy();
+      ActionInv = mover->ActionInv;
+      Ptr = mover->Ptr;
+      mover->Ptr = 0;
       return *this;
     }
 
     /// Assignment from a raw pointer. Takes ownership - beware!
-    ASTOwningResult & operator =(void *raw)
-    {
+    ASTOwningResult & operator =(void *raw) {
+      destroy();
       Ptr = raw;
-      Invalid = false;
+      ActionInv.setInt(false);
       return *this;
     }
 
     /// Assignment from an ActionResult. Takes ownership - beware!
     ASTOwningResult & operator =(const DumbResult &res) {
+      destroy();
       Ptr = res.get();
-      Invalid = res.isInvalid();
+      ActionInv.setInt(res.isInvalid());
       return *this;
     }
 
     /// Access to the raw pointer.
-    void * get() const { return Ptr.get(); }
+    void * get() const { return Ptr; }
 
-    bool isInvalid() const { return Invalid; }
+    bool isInvalid() const { return ActionInv.getInt(); }
 
     /// Does this point to a usable AST node? To be usable, the node must be
     /// valid and non-null.
-    bool isUsable() const { return !Invalid && get(); }
+    bool isUsable() const { return !isInvalid() && get(); }
 
     /// Take outside ownership of the raw pointer.
     void * take() {
-      if (Invalid)
+      if (isInvalid())
         return 0;
-      return Ptr.take();
+      void *tmp = Ptr;
+      Ptr = 0;
+      return tmp;
     }
 
     /// Alias for interface familiarity with unique_ptr.
@@ -449,20 +394,15 @@
 
     /// Pass ownership to a classical ActionResult.
     DumbResult result() {
-      if (Invalid)
+      if (isInvalid())
         return true;
-      return Ptr.take();
+      return take();
     }
 
     /// Move hook
     operator moving::ASTResultMover<Destroyer>() {
       return moving::ASTResultMover<Destroyer>(*this);
     }
-
-    /// Special function for moving to an OwningPtr.
-    moving::ASTPtrMover<Destroyer> ptr_move() {
-      return moving::ASTPtrMover<Destroyer>(Ptr);
-    }
   };
 #else
   template <ASTDestroyer Destroyer>
@@ -480,8 +420,7 @@
     ASTOwningResult(ActionBase &actions, void *node) : Result(node) { }
     ASTOwningResult(ActionBase &actions, const DumbResult &res) : Result(res) { }
     // Normal copying semantics are defined implicitly.
-    // The fake movers need this:
-    explicit ASTOwningResult(void *ptr) : Result(ptr) { }
+    ASTOwningResult(const ASTOwningPtr<Destroyer> &o) : Result(o.get()) { }
 
     /// Assignment from a raw pointer. Takes ownership - beware!
     ASTOwningResult & operator =(void *raw)
@@ -616,29 +555,17 @@
   }
 
   template <ASTDestroyer Destroyer> inline
-  ASTOwningPtr<Destroyer> move(ASTOwningPtr<Destroyer> &ptr) {
-    return ASTOwningPtr<Destroyer>(moving::ASTPtrMover<Destroyer>(ptr));
-  }
-
-  template <ASTDestroyer Destroyer> inline
   ASTMultiPtr<Destroyer> move(ASTMultiPtr<Destroyer> &ptr) {
     return ASTMultiPtr<Destroyer>(moving::ASTMultiMover<Destroyer>(ptr));
   }
 
-  // These are necessary because of ambiguity problems.
-
-  template <ASTDestroyer Destroyer> inline
-  ASTOwningPtr<Destroyer> move_arg(ASTOwningResult<Destroyer> &ptr) {
-    return ASTOwningPtr<Destroyer>(ptr.ptr_move());
-  }
-
-  template <ASTDestroyer Destroyer> inline
-  ASTOwningResult<Destroyer> move_res(ASTOwningPtr<Destroyer> &ptr) {
-    return ASTOwningResult<Destroyer>(moving::ASTPtrMover<Destroyer>(ptr));
-  }
-
 #else
 
+  template <ASTDestroyer Destroyer> inline
+  ASTOwningPtr<Destroyer>::ASTOwningPtr(const ASTOwningResult<Destroyer> &o)
+    : Node(o.get())
+  {}
+
   // These versions are hopefully no-ops.
   template <ASTDestroyer Destroyer> inline
   ASTOwningResult<Destroyer>& move(ASTOwningResult<Destroyer> &ptr) {
@@ -655,16 +582,6 @@
     return ptr;
   }
 
-  template <ASTDestroyer Destroyer> inline
-  ASTOwningPtr<Destroyer> move_arg(ASTOwningResult<Destroyer> &ptr) {
-    return ASTOwningPtr<Destroyer>(ptr.take());
-  }
-
-  template <ASTDestroyer Destroyer> inline
-  ASTOwningResult<Destroyer> move_res(ASTOwningPtr<Destroyer> &ptr) {
-    return ASTOwningResult<Destroyer>(ptr.get());
-  }
-
 #endif
 
 }
diff --git a/lib/Parse/ParseDecl.cpp b/lib/Parse/ParseDecl.cpp
index e2601b6..bbd5d4b 100644
--- a/lib/Parse/ParseDecl.cpp
+++ b/lib/Parse/ParseDecl.cpp
@@ -312,7 +312,7 @@
         SkipUntil(tok::semi);
         return 0;
       }
-      Actions.AddInitializerToDecl(LastDeclInGroup, move_arg(Init));
+      Actions.AddInitializerToDecl(LastDeclInGroup, move(Init));
     } else if (Tok.is(tok::l_paren)) {
       // Parse C++ direct initializer: '(' expression-list ')'
       SourceLocation LParenLoc = ConsumeParen();
diff --git a/lib/Parse/ParseExpr.cpp b/lib/Parse/ParseExpr.cpp
index d20f294..597c50c 100644
--- a/lib/Parse/ParseExpr.cpp
+++ b/lib/Parse/ParseExpr.cpp
@@ -205,7 +205,7 @@
   if (LHS.isInvalid()) return move(LHS);
 
   LHS = Actions.ActOnUnaryOp(CurScope, ExtLoc, tok::kw___extension__,
-                             move_arg(LHS));
+                             move(LHS));
   if (LHS.isInvalid()) return move(LHS);
 
   return ParseRHSOfBinaryExpression(move(LHS), prec::Comma);
@@ -334,12 +334,11 @@
       // Combine the LHS and RHS into the LHS (e.g. build AST).
       if (TernaryMiddle.isInvalid())
         LHS = Actions.ActOnBinOp(CurScope, OpToken.getLocation(),
-                                 OpToken.getKind(), move_arg(LHS),
-                                 move_arg(RHS));
+                                 OpToken.getKind(), move(LHS), move(RHS));
       else
         LHS = Actions.ActOnConditionalOp(OpToken.getLocation(), ColonLoc,
-                                         move_arg(LHS), move_arg(TernaryMiddle),
-                                         move_arg(RHS));
+                                         move(LHS), move(TernaryMiddle),
+                                         move(RHS));
     }
   }
 }
@@ -493,8 +492,7 @@
       // TODO: For cast expression with CastTy.
       Res = ParseCastExpression(false);
       if (!Res.isInvalid())
-        Res = Actions.ActOnCastExpr(LParenLoc, CastTy, RParenLoc,
-                                    move_arg(Res));
+        Res = Actions.ActOnCastExpr(LParenLoc, CastTy, RParenLoc, move(Res));
       return move(Res);
     }
 
@@ -570,7 +568,7 @@
     SourceLocation SavedLoc = ConsumeToken();
     Res = ParseCastExpression(true);
     if (!Res.isInvalid())
-      Res = Actions.ActOnUnaryOp(CurScope, SavedLoc, SavedKind, move_arg(Res));
+      Res = Actions.ActOnUnaryOp(CurScope, SavedLoc, SavedKind, move(Res));
     return move(Res);
   }
   case tok::amp: {         // unary-expression: '&' cast-expression
@@ -578,7 +576,7 @@
     SourceLocation SavedLoc = ConsumeToken();
     Res = ParseCastExpression(false, true);
     if (!Res.isInvalid())
-      Res = Actions.ActOnUnaryOp(CurScope, SavedLoc, SavedKind, move_arg(Res));
+      Res = Actions.ActOnUnaryOp(CurScope, SavedLoc, SavedKind, move(Res));
     return move(Res);
   }
 
@@ -592,7 +590,7 @@
     SourceLocation SavedLoc = ConsumeToken();
     Res = ParseCastExpression(false);
     if (!Res.isInvalid())
-      Res = Actions.ActOnUnaryOp(CurScope, SavedLoc, SavedKind, move_arg(Res));
+      Res = Actions.ActOnUnaryOp(CurScope, SavedLoc, SavedKind, move(Res));
     return move(Res);
   }
 
@@ -602,7 +600,7 @@
     SourceLocation SavedLoc = ConsumeToken();
     Res = ParseCastExpression(false);
     if (!Res.isInvalid())
-      Res = Actions.ActOnUnaryOp(CurScope, SavedLoc, SavedKind, move_arg(Res));
+      Res = Actions.ActOnUnaryOp(CurScope, SavedLoc, SavedKind, move(Res));
     return move(Res);
   }
   case tok::kw_sizeof:     // unary-expression: 'sizeof' unary-expression
@@ -765,8 +763,8 @@
       SourceLocation RLoc = Tok.getLocation();
 
       if (!LHS.isInvalid() && !Idx.isInvalid() && Tok.is(tok::r_square)) {
-        LHS = Actions.ActOnArraySubscriptExpr(CurScope, move_arg(LHS), Loc,
-                                              move_arg(Idx), RLoc);
+        LHS = Actions.ActOnArraySubscriptExpr(CurScope, move(LHS), Loc,
+                                              move(Idx), RLoc);
       } else
         LHS = ExprError();
 
@@ -792,7 +790,7 @@
       if (!LHS.isInvalid() && Tok.is(tok::r_paren)) {
         assert((ArgExprs.size() == 0 || ArgExprs.size()-1 == CommaLocs.size())&&
                "Unexpected number of commas!");
-        LHS = Actions.ActOnCallExpr(CurScope, move_arg(LHS), Loc,
+        LHS = Actions.ActOnCallExpr(CurScope, move(LHS), Loc,
                                     move_arg(ArgExprs), &CommaLocs[0],
                                     Tok.getLocation());
       }
@@ -811,7 +809,7 @@
       }
 
       if (!LHS.isInvalid()) {
-        LHS = Actions.ActOnMemberReferenceExpr(CurScope, move_arg(LHS), OpLoc,
+        LHS = Actions.ActOnMemberReferenceExpr(CurScope, move(LHS), OpLoc,
                                                OpKind, Tok.getLocation(),
                                                *Tok.getIdentifierInfo());
       }
@@ -822,7 +820,7 @@
     case tok::minusminus:  // postfix-expression: postfix-expression '--'
       if (!LHS.isInvalid()) {
         LHS = Actions.ActOnPostfixUnaryOp(CurScope, Tok.getLocation(), 
-                                          Tok.getKind(), move_arg(LHS));
+                                          Tok.getKind(), move(LHS));
       }
       ConsumeToken();
       break;
@@ -1128,7 +1126,7 @@
       ExprType = CompoundLiteral;
       if (!Result.isInvalid())
         return Actions.ActOnCompoundLiteral(OpenLoc, Ty, RParenLoc,
-                                            move_arg(Result));
+                                            move(Result));
       return move(Result);
     }
 
@@ -1146,8 +1144,7 @@
     Result = ParseExpression();
     ExprType = SimpleExpr;
     if (!Result.isInvalid() && Tok.is(tok::r_paren))
-      Result = Actions.ActOnParenExpr(OpenLoc, Tok.getLocation(),
-                                      move_arg(Result));
+      Result = Actions.ActOnParenExpr(OpenLoc, Tok.getLocation(), move(Result));
   }
 
   // Match the ')'.
diff --git a/lib/Parse/ParseInit.cpp b/lib/Parse/ParseInit.cpp
index ec8470c..0b34fea 100644
--- a/lib/Parse/ParseInit.cpp
+++ b/lib/Parse/ParseInit.cpp
@@ -174,7 +174,7 @@
 
       return ParseAssignmentExprWithObjCMessageExprStart(StartLoc,
                                                          SourceLocation(),
-                                                         0, move_arg(Idx));
+                                                         0, move(Idx));
     }
 
     // Create designation if we haven't already.
diff --git a/lib/Parse/ParseObjc.cpp b/lib/Parse/ParseObjc.cpp
index 5d7d7f1..f455626 100644
--- a/lib/Parse/ParseObjc.cpp
+++ b/lib/Parse/ParseObjc.cpp
@@ -1202,7 +1202,7 @@
     }
   }
   ConsumeToken(); // consume ';'
-  return Actions.ActOnObjCAtThrowStmt(atLoc, move_arg(Res));
+  return Actions.ActOnObjCAtThrowStmt(atLoc, move(Res));
 }
 
 /// objc-synchronized-statement:
@@ -1239,8 +1239,7 @@
   BodyScope.Exit();
   if (SynchBody.isInvalid())
     SynchBody = Actions.ActOnNullStmt(Tok.getLocation());
-  return Actions.ActOnObjCAtSynchronizedStmt(atLoc, move_arg(Res),
-                                             move_arg(SynchBody));
+  return Actions.ActOnObjCAtSynchronizedStmt(atLoc, move(Res), move(SynchBody));
 }
 
 ///  objc-try-catch-statement:
@@ -1313,8 +1312,8 @@
         if (CatchBody.isInvalid())
           CatchBody = Actions.ActOnNullStmt(Tok.getLocation());
         CatchStmts = Actions.ActOnObjCAtCatchStmt(AtCatchFinallyLoc,
-                        RParenLoc, move_arg(FirstPart), move_arg(CatchBody),
-                        move_arg(CatchStmts));
+                        RParenLoc, move(FirstPart), move(CatchBody),
+                        move(CatchStmts));
       } else {
         Diag(AtCatchFinallyLoc, diag::err_expected_lparen_after)
           << "@catch clause";
@@ -1334,7 +1333,7 @@
       if (FinallyBody.isInvalid())
         FinallyBody = Actions.ActOnNullStmt(Tok.getLocation());
       FinallyStmt = Actions.ActOnObjCAtFinallyStmt(AtCatchFinallyLoc,
-                                                   move_arg(FinallyBody));
+                                                   move(FinallyBody));
       catch_or_finally_seen = true;
       break;
     }
@@ -1343,9 +1342,8 @@
     Diag(atLoc, diag::err_missing_catch_finally);
     return StmtError();
   }
-  return Actions.ActOnObjCAtTryStmt(atLoc, move_arg(TryBody),
-                                    move_arg(CatchStmts),
-                                    move_arg(FinallyStmt));
+  return Actions.ActOnObjCAtTryStmt(atLoc, move(TryBody), move(CatchStmts),
+                                    move(FinallyStmt));
 }
 
 ///   objc-method-def: objc-method-proto ';'[opt] '{' body '}'
@@ -1387,7 +1385,7 @@
   BodyScope.Exit();
 
   // TODO: Pass argument information.
-  Actions.ActOnFinishFunctionBody(MDecl, move_arg(FnBody));
+  Actions.ActOnFinishFunctionBody(MDecl, move(FnBody));
   return MDecl;
 }
 
@@ -1408,7 +1406,7 @@
   }
   // Otherwise, eat the semicolon.
   ExpectAndConsume(tok::semi, diag::err_expected_semi_after_expr);
-  return Actions.ActOnExprStmt(move_arg(Res));
+  return Actions.ActOnExprStmt(move(Res));
 }
 
 Parser::OwningExprResult Parser::ParseObjCAtExpression(SourceLocation AtLoc) {
@@ -1459,7 +1457,7 @@
   }
 
   return ParseObjCMessageExpressionBody(LBracLoc, SourceLocation(),
-                                        0, move_arg(Res));
+                                        0, move(Res));
 }
 
 /// ParseObjCMessageExpressionBody - Having parsed "'[' objc-receiver", parse
diff --git a/lib/Parse/ParseStmt.cpp b/lib/Parse/ParseStmt.cpp
index 5ff0905..a830399 100644
--- a/lib/Parse/ParseStmt.cpp
+++ b/lib/Parse/ParseStmt.cpp
@@ -118,7 +118,7 @@
       }
       // Otherwise, eat the semicolon.
       ExpectAndConsume(tok::semi, diag::err_expected_semi_after_expr);
-      return Actions.ActOnExprStmt(move_arg(Expr));
+      return Actions.ActOnExprStmt(move(Expr));
     }
 
   case tok::kw_case:                // C99 6.8.1: labeled-statement
@@ -217,7 +217,7 @@
 
   return Actions.ActOnLabelStmt(IdentTok.getLocation(),
                                 IdentTok.getIdentifierInfo(),
-                                ColonLoc, move_arg(SubStmt));
+                                ColonLoc, move(SubStmt));
 }
 
 /// ParseCaseStatement
@@ -271,8 +271,8 @@
   if (SubStmt.isInvalid())
     SubStmt = Actions.ActOnNullStmt(ColonLoc);
 
-  return Actions.ActOnCaseStmt(CaseLoc, move_arg(LHS), DotDotDotLoc,
-                               move_arg(RHS), ColonLoc, move_arg(SubStmt));
+  return Actions.ActOnCaseStmt(CaseLoc, move(LHS), DotDotDotLoc,
+                               move(RHS), ColonLoc, move(SubStmt));
 }
 
 /// ParseDefaultStatement
@@ -303,7 +303,7 @@
     return StmtError();
 
   return Actions.ActOnDefaultStmt(DefaultLoc, ColonLoc,
-                                  move_arg(SubStmt), CurScope);
+                                  move(SubStmt), CurScope);
 }
 
 
@@ -393,7 +393,7 @@
         // Eat the semicolon at the end of stmt and convert the expr into a
         // statement.
         ExpectAndConsume(tok::semi, diag::err_expected_semi_after_expr);
-        R = Actions.ActOnExprStmt(move_arg(Res));
+        R = Actions.ActOnExprStmt(move(Res));
       }
     }
 
@@ -568,8 +568,8 @@
   if (ElseStmt.isInvalid())
     ElseStmt = Actions.ActOnNullStmt(ElseStmtLoc);
 
-  return Actions.ActOnIfStmt(IfLoc, move_arg(CondExp), move_arg(ThenStmt),
-                             ElseLoc, move_arg(ElseStmt));
+  return Actions.ActOnIfStmt(IfLoc, move(CondExp), move(ThenStmt),
+                             ElseLoc, move(ElseStmt));
 }
 
 /// ParseSwitchStatement
@@ -612,7 +612,7 @@
 
   OwningStmtResult Switch(Actions);
   if (!Cond.isInvalid())
-    Switch = Actions.ActOnStartOfSwitchStmt(move_arg(Cond));
+    Switch = Actions.ActOnStartOfSwitchStmt(move(Cond));
 
   // C99 6.8.4p3 - In C99, the body of the switch statement is a scope, even if
   // there is no compound stmt.  C90 does not have this clause.  We only do this
@@ -644,8 +644,7 @@
   if (Cond.isInvalid())
     return StmtError();
 
-  return Actions.ActOnFinishSwitchStmt(SwitchLoc, move_arg(Switch),
-                                       move_arg(Body));
+  return Actions.ActOnFinishSwitchStmt(SwitchLoc, move(Switch), move(Body));
 }
 
 /// ParseWhileStatement
@@ -714,7 +713,7 @@
   if (Cond.isInvalid() || Body.isInvalid())
     return StmtError();
 
-  return Actions.ActOnWhileStmt(WhileLoc, move_arg(Cond), move_arg(Body));
+  return Actions.ActOnWhileStmt(WhileLoc, move(Cond), move(Body));
 }
 
 /// ParseDoStatement
@@ -778,7 +777,7 @@
   if (Cond.isInvalid() || Body.isInvalid())
     return StmtError();
 
-  return Actions.ActOnDoStmt(DoLoc, move_arg(Body), WhileLoc, move_arg(Cond));
+  return Actions.ActOnDoStmt(DoLoc, move(Body), WhileLoc, move(Cond));
 }
 
 /// ParseForStatement
@@ -860,7 +859,7 @@
 
     // Turn the expression into a stmt.
     if (!Value.isInvalid())
-      FirstPart = Actions.ActOnExprStmt(move_arg(Value));
+      FirstPart = Actions.ActOnExprStmt(move(Value));
 
     if (Tok.is(tok::semi)) {
       ConsumeToken();
@@ -927,14 +926,14 @@
     return StmtError();
 
   if (!ForEach)
-    return Actions.ActOnForStmt(ForLoc, LParenLoc, move_arg(FirstPart),
-                              move_arg(SecondPart), move_arg(ThirdPart),
-                              RParenLoc, move_arg(Body));
+    return Actions.ActOnForStmt(ForLoc, LParenLoc, move(FirstPart),
+                              move(SecondPart), move(ThirdPart),
+                              RParenLoc, move(Body));
   else
     return Actions.ActOnObjCForCollectionStmt(ForLoc, LParenLoc,
-                                              move_arg(FirstPart),
-                                              move_arg(SecondPart),
-                                              RParenLoc, move_arg(Body));
+                                              move(FirstPart),
+                                              move(SecondPart),
+                                              RParenLoc, move(Body));
 }
 
 /// ParseGotoStatement
@@ -962,7 +961,7 @@
       SkipUntil(tok::semi, false, true);
       return StmtError();
     }
-    Res = Actions.ActOnIndirectGotoStmt(GotoLoc, StarLoc, move_arg(R));
+    Res = Actions.ActOnIndirectGotoStmt(GotoLoc, StarLoc, move(R));
   } else {
     Diag(Tok, diag::err_expected_ident);
     return StmtError();
@@ -1008,7 +1007,7 @@
       return StmtError();
     }
   }
-  return Actions.ActOnReturnStmt(ReturnLoc, move_arg(R));
+  return Actions.ActOnReturnStmt(ReturnLoc, move(R));
 }
 
 /// FuzzyParseMicrosoftAsmStatement. When -fms-extensions is enabled, this
@@ -1148,7 +1147,7 @@
   return Actions.ActOnAsmStmt(AsmLoc, isSimple, isVolatile,
                               NumOutputs, NumInputs, &Names[0],
                               move_arg(Constraints), move_arg(Exprs),
-                              move_arg(AsmString), move_arg(Clobbers),
+                              move(AsmString), move_arg(Clobbers),
                               RParenLoc);
 }
 
@@ -1233,7 +1232,7 @@
   if (FnBody.isInvalid())
     FnBody = Actions.ActOnCompoundStmt(L, R, MultiStmtArg(Actions), false);
 
-  return Actions.ActOnFinishFunctionBody(Decl, move_arg(FnBody));
+  return Actions.ActOnFinishFunctionBody(Decl, move(FnBody));
 }
 
 /// ParseCXXTryBlock - Parse a C++ try-block.
@@ -1267,8 +1266,7 @@
   if (Handlers.empty())
     return StmtError();
 
-  return Actions.ActOnCXXTryBlock(TryLoc, move_arg(TryBlock),
-                                  move_arg(Handlers));
+  return Actions.ActOnCXXTryBlock(TryLoc, move(TryBlock), move_arg(Handlers));
 }
 
 /// ParseCXXCatchBlock - Parse a C++ catch block, called handler in the standard
@@ -1319,5 +1317,5 @@
   if (Block.isInvalid())
     return move(Block);
 
-  return Actions.ActOnCXXCatchBlock(CatchLoc, ExceptionDecl, move_arg(Block));
+  return Actions.ActOnCXXCatchBlock(CatchLoc, ExceptionDecl, move(Block));
 }
diff --git a/lib/Parse/Parser.cpp b/lib/Parse/Parser.cpp
index 2f7e2ea..eb84a4a 100644
--- a/lib/Parse/Parser.cpp
+++ b/lib/Parse/Parser.cpp
@@ -357,7 +357,7 @@
                      "top-level asm block");
 
     if (!Result.isInvalid())
-      return Actions.ActOnFileScopeAsmDecl(Tok.getLocation(), move_arg(Result));
+      return Actions.ActOnFileScopeAsmDecl(Tok.getLocation(), move(Result));
     return 0;
   }
   case tok::at: