Refactor deoptimization support in debugger

This CL prepares breakpoint support for inlined methods where we'll have to
deoptimize everything.

We move deoptimization-related information to Dbg class only (deoptimization
request queue, full deoptimization event count and deoptimization lock). We
replace MethodInstrumentionRequest by DeoptimizationRequest. This is used to
know which kind of deoptimization is required for a particular event.

It also simplifies lock ordering a bit during event setup: we no longer need to
hold the deoptimization lock while holding the breakpoint lock. Moreover, the
deoptimization lock should be held only after the event list lock.

Bug: 12187616
Change-Id: Iff13f004adaeb25e5d609238bacce0b9720510e6
diff --git a/runtime/debugger.cc b/runtime/debugger.cc
index c18d5c6..62019fa 100644
--- a/runtime/debugger.cc
+++ b/runtime/debugger.cc
@@ -200,17 +200,9 @@
 size_t Dbg::alloc_record_count_ = 0;
 
 // Deoptimization support.
-struct MethodInstrumentationRequest {
-  bool deoptimize;
-
-  // Method for selective deoptimization. NULL means full deoptimization.
-  mirror::ArtMethod* method;
-
-  MethodInstrumentationRequest(bool deoptimize, mirror::ArtMethod* method)
-    : deoptimize(deoptimize), method(method) {}
-};
-// TODO we need to visit associated methods as roots.
-static std::vector<MethodInstrumentationRequest> gDeoptimizationRequests GUARDED_BY(Locks::deoptimization_lock_);
+Mutex* Dbg::deoptimization_lock_ = nullptr;
+std::vector<DeoptimizationRequest> Dbg::deoptimization_requests_;
+size_t Dbg::full_deoptimization_event_count_ = 0;
 
 // Breakpoints.
 static std::vector<Breakpoint> gBreakpoints GUARDED_BY(Locks::breakpoint_lock_);
@@ -238,6 +230,12 @@
   }
 }
 
+void DeoptimizationRequest::VisitRoots(RootCallback* callback, void* arg) {
+  if (method != nullptr) {
+    callback(reinterpret_cast<mirror::Object**>(&method), arg, 0, kRootDebugger);
+  }
+}
+
 static bool IsBreakpoint(const mirror::ArtMethod* m, uint32_t dex_pc)
     LOCKS_EXCLUDED(Locks::breakpoint_lock_)
     SHARED_LOCKS_REQUIRED(Locks::mutator_lock_) {
@@ -502,6 +500,7 @@
   gRegistry = new ObjectRegistry;
 
   alloc_tracker_lock_ = new Mutex("AllocTracker lock");
+  deoptimization_lock_ = new Mutex("deoptimization lock", kDeoptimizationLock);
   // Init JDWP if the debugger is enabled. This may connect out to a
   // debugger, passively listen for a debugger, or block waiting for a
   // debugger.
@@ -524,9 +523,17 @@
 }
 
 void Dbg::VisitRoots(RootCallback* callback, void* arg) {
-  MutexLock mu(Thread::Current(), *Locks::breakpoint_lock_);
-  for (Breakpoint& bp : gBreakpoints) {
-    bp.VisitRoots(callback, arg);
+  {
+    MutexLock mu(Thread::Current(), *Locks::breakpoint_lock_);
+    for (Breakpoint& bp : gBreakpoints) {
+      bp.VisitRoots(callback, arg);
+    }
+  }
+  if (deoptimization_lock_ != nullptr) {  // only true if the debugger is started.
+    MutexLock mu(Thread::Current(), *deoptimization_lock_);
+    for (DeoptimizationRequest& req : deoptimization_requests_) {
+      req.VisitRoots(callback, arg);
+    }
   }
 }
 
@@ -539,6 +546,8 @@
   gRegistry = nullptr;
   delete alloc_tracker_lock_;
   alloc_tracker_lock_ = nullptr;
+  delete deoptimization_lock_;
+  deoptimization_lock_ = nullptr;
 }
 
 void Dbg::GcDidFinish() {
@@ -605,8 +614,9 @@
   }
 
   {
-    MutexLock mu(Thread::Current(), *Locks::deoptimization_lock_);
-    CHECK_EQ(gDeoptimizationRequests.size(), 0U);
+    MutexLock mu(Thread::Current(), *deoptimization_lock_);
+    CHECK_EQ(deoptimization_requests_.size(), 0U);
+    CHECK_EQ(full_deoptimization_event_count_, 0U);
   }
 
   Runtime* runtime = Runtime::Current();
@@ -646,8 +656,9 @@
       // Since we're going to disable deoptimization, we clear the deoptimization requests queue.
       // This prevents us from having any pending deoptimization request when the debugger attaches
       // to us again while no event has been requested yet.
-      MutexLock mu(Thread::Current(), *Locks::deoptimization_lock_);
-      gDeoptimizationRequests.clear();
+      MutexLock mu(Thread::Current(), *deoptimization_lock_);
+      deoptimization_requests_.clear();
+      full_deoptimization_event_count_ = 0U;
     }
     runtime->GetInstrumentation()->RemoveListener(&gDebugInstrumentationListener,
                                                   instrumentation::Instrumentation::kMethodEntered |
@@ -2546,44 +2557,86 @@
   }
 }
 
-static void ProcessDeoptimizationRequests()
-    LOCKS_EXCLUDED(Locks::deoptimization_lock_)
-    EXCLUSIVE_LOCKS_REQUIRED(Locks::mutator_lock_) {
-  Locks::mutator_lock_->AssertExclusiveHeld(Thread::Current());
-  MutexLock mu(Thread::Current(), *Locks::deoptimization_lock_);
+// Process request while all mutator threads are suspended.
+void Dbg::ProcessDeoptimizationRequest(const DeoptimizationRequest& request) {
   instrumentation::Instrumentation* instrumentation = Runtime::Current()->GetInstrumentation();
-  for (const MethodInstrumentationRequest& request : gDeoptimizationRequests) {
-    mirror::ArtMethod* const method = request.method;
-    if (method != nullptr) {
-      // Selective deoptimization.
-      if (request.deoptimize) {
-        VLOG(jdwp) << "Deoptimize method " << PrettyMethod(method);
-        instrumentation->Deoptimize(method);
-      } else {
-        VLOG(jdwp) << "Undeoptimize method " << PrettyMethod(method);
-        instrumentation->Undeoptimize(method);
-      }
-    } else {
-      // Full deoptimization.
-      if (request.deoptimize) {
-        VLOG(jdwp) << "Deoptimize the world";
-        instrumentation->DeoptimizeEverything();
-      } else {
-        VLOG(jdwp) << "Undeoptimize the world";
-        instrumentation->UndeoptimizeEverything();
-      }
-    }
+  switch (request.kind) {
+    case DeoptimizationRequest::kNothing:
+      LOG(WARNING) << "Ignoring empty deoptimization request.";
+      break;
+    case DeoptimizationRequest::kFullDeoptimization:
+      VLOG(jdwp) << "Deoptimize the world";
+      instrumentation->DeoptimizeEverything();
+      break;
+    case DeoptimizationRequest::kFullUndeoptimization:
+      VLOG(jdwp) << "Undeoptimize the world";
+      instrumentation->UndeoptimizeEverything();
+      break;
+    case DeoptimizationRequest::kSelectiveDeoptimization:
+      VLOG(jdwp) << "Deoptimize method " << PrettyMethod(request.method);
+      instrumentation->Deoptimize(request.method);
+      break;
+    case DeoptimizationRequest::kSelectiveUndeoptimization:
+      VLOG(jdwp) << "Undeoptimize method " << PrettyMethod(request.method);
+      instrumentation->Undeoptimize(request.method);
+      break;
+    default:
+      LOG(FATAL) << "Unsupported deoptimization request kind " << request.kind;
+      break;
   }
-  gDeoptimizationRequests.clear();
 }
 
-// Process deoptimization requests after suspending all mutator threads.
+void Dbg::RequestDeoptimization(const DeoptimizationRequest& req) {
+  if (req.kind == DeoptimizationRequest::kNothing) {
+    // Nothing to do.
+    return;
+  }
+  MutexLock mu(Thread::Current(), *deoptimization_lock_);
+  switch (req.kind) {
+    case DeoptimizationRequest::kFullDeoptimization: {
+      DCHECK(req.method == nullptr);
+      if (full_deoptimization_event_count_ == 0) {
+        VLOG(jdwp) << "Request full deoptimization";
+        deoptimization_requests_.push_back(req);
+      }
+      ++full_deoptimization_event_count_;
+      break;
+    }
+    case DeoptimizationRequest::kFullUndeoptimization: {
+      DCHECK(req.method == nullptr);
+      DCHECK_GT(full_deoptimization_event_count_, 0U);
+      --full_deoptimization_event_count_;
+      if (full_deoptimization_event_count_ == 0) {
+        VLOG(jdwp) << "Request full undeoptimization";
+        deoptimization_requests_.push_back(req);
+      }
+      break;
+    }
+    case DeoptimizationRequest::kSelectiveDeoptimization: {
+      DCHECK(req.method != nullptr);
+      VLOG(jdwp) << "Request deoptimization of " << PrettyMethod(req.method);
+      deoptimization_requests_.push_back(req);
+      break;
+    }
+    case DeoptimizationRequest::kSelectiveUndeoptimization: {
+      DCHECK(req.method != nullptr);
+      VLOG(jdwp) << "Request undeoptimization of " << PrettyMethod(req.method);
+      deoptimization_requests_.push_back(req);
+      break;
+    }
+    default: {
+      LOG(FATAL) << "Unknown deoptimization request kind " << req.kind;
+      break;
+    }
+  }
+}
+
 void Dbg::ManageDeoptimization() {
   Thread* const self = Thread::Current();
   {
     // Avoid suspend/resume if there is no pending request.
-    MutexLock mu(self, *Locks::deoptimization_lock_);
-    if (gDeoptimizationRequests.empty()) {
+    MutexLock mu(self, *deoptimization_lock_);
+    if (deoptimization_requests_.empty()) {
       return;
     }
   }
@@ -2593,27 +2646,21 @@
   Runtime* const runtime = Runtime::Current();
   runtime->GetThreadList()->SuspendAll();
   const ThreadState old_state = self->SetStateUnsafe(kRunnable);
-  ProcessDeoptimizationRequests();
+  {
+    MutexLock mu(self, *deoptimization_lock_);
+    for (const DeoptimizationRequest& request : deoptimization_requests_) {
+      ProcessDeoptimizationRequest(request);
+    }
+    deoptimization_requests_.clear();
+  }
   CHECK_EQ(self->SetStateUnsafe(old_state), kRunnable);
   runtime->GetThreadList()->ResumeAll();
   self->TransitionFromSuspendedToRunnable();
 }
 
-// Enable full deoptimization.
-void Dbg::EnableFullDeoptimization() {
-  MutexLock mu(Thread::Current(), *Locks::deoptimization_lock_);
-  VLOG(jdwp) << "Request full deoptimization";
-  gDeoptimizationRequests.push_back(MethodInstrumentationRequest(true, nullptr));
-}
-
-// Disable full deoptimization.
-void Dbg::DisableFullDeoptimization() {
-  MutexLock mu(Thread::Current(), *Locks::deoptimization_lock_);
-  VLOG(jdwp) << "Request full undeoptimization";
-  gDeoptimizationRequests.push_back(MethodInstrumentationRequest(false, nullptr));
-}
-
-void Dbg::WatchLocation(const JDWP::JdwpLocation* location) {
+void Dbg::WatchLocation(const JDWP::JdwpLocation* location, DeoptimizationRequest* req) {
+  // TODO we don't need to deoptimize a method if it's not compiled since it already runs with the
+  // interpreter.
   bool need_deoptimization = true;
   mirror::ArtMethod* m = FromMethodId(location->method_id);
   {
@@ -2630,18 +2677,17 @@
     }
 
     gBreakpoints.push_back(Breakpoint(m, location->dex_pc));
-    VLOG(jdwp) << "Set breakpoint #" << (gBreakpoints.size() - 1) << ": " << gBreakpoints[gBreakpoints.size() - 1];
+    VLOG(jdwp) << "Set breakpoint #" << (gBreakpoints.size() - 1) << ": "
+               << gBreakpoints[gBreakpoints.size() - 1];
   }
 
   if (need_deoptimization) {
-    // Request its deoptimization. This will be done after updating the JDWP event list.
-    MutexLock mu(Thread::Current(), *Locks::deoptimization_lock_);
-    gDeoptimizationRequests.push_back(MethodInstrumentationRequest(true, m));
-    VLOG(jdwp) << "Request deoptimization of " << PrettyMethod(m);
+    req->kind = DeoptimizationRequest::kSelectiveDeoptimization;
+    req->method = m;
   }
 }
 
-void Dbg::UnwatchLocation(const JDWP::JdwpLocation* location) {
+void Dbg::UnwatchLocation(const JDWP::JdwpLocation* location, DeoptimizationRequest* req) {
   bool can_undeoptimize = true;
   mirror::ArtMethod* m = FromMethodId(location->method_id);
   DCHECK(Runtime::Current()->GetInstrumentation()->IsDeoptimized(m));
@@ -2666,9 +2712,8 @@
 
   if (can_undeoptimize) {
     // Request its undeoptimization. This will be done after updating the JDWP event list.
-    MutexLock mu(Thread::Current(), *Locks::deoptimization_lock_);
-    gDeoptimizationRequests.push_back(MethodInstrumentationRequest(false, m));
-    VLOG(jdwp) << "Request undeoptimization of " << PrettyMethod(m);
+    req->kind = DeoptimizationRequest::kSelectiveUndeoptimization;
+    req->method = m;
   }
 }