Check stacks for unmodifiable frames

We keep a generate a list of classes that have unmodifiable frames
during the zygote fork and check for them in IsModifiableClass.

Test: Start apps on aosp_marlin-userdebug phone with libartd.so

Change-Id: I6bbaa20d307c3803a5808fb4108638365895e802
diff --git a/runtime/Android.bp b/runtime/Android.bp
index d136aa1..b4c7b9c 100644
--- a/runtime/Android.bp
+++ b/runtime/Android.bp
@@ -171,6 +171,7 @@
         "native/org_apache_harmony_dalvik_ddmc_DdmServer.cc",
         "native/org_apache_harmony_dalvik_ddmc_DdmVmInternal.cc",
         "native/sun_misc_Unsafe.cc",
+        "non_debuggable_classes.cc",
         "oat.cc",
         "oat_file.cc",
         "oat_file_assistant.cc",
diff --git a/runtime/native/dalvik_system_ZygoteHooks.cc b/runtime/native/dalvik_system_ZygoteHooks.cc
index fd22d9e..100f476 100644
--- a/runtime/native/dalvik_system_ZygoteHooks.cc
+++ b/runtime/native/dalvik_system_ZygoteHooks.cc
@@ -26,9 +26,11 @@
 #include "jit/jit.h"
 #include "jni_internal.h"
 #include "JNIHelp.h"
+#include "non_debuggable_classes.h"
 #include "scoped_thread_state_change-inl.h"
 #include "ScopedUtfChars.h"
 #include "thread-inl.h"
+#include "thread_list.h"
 #include "trace.h"
 
 #if defined(__linux__)
@@ -39,6 +41,10 @@
 
 namespace art {
 
+// Set to true to always determine the non-debuggable classes even if we would not allow a debugger
+// to actually attach.
+static constexpr bool kAlwaysCollectNonDebuggableClasses = kIsDebugBuild;
+
 using android::base::StringPrintf;
 
 static void EnableDebugger() {
@@ -68,6 +74,39 @@
   }
 }
 
+static void DoCollectNonDebuggableCallback(Thread* thread, void* data ATTRIBUTE_UNUSED)
+    REQUIRES(Locks::mutator_lock_) {
+  class NonDebuggableStacksVisitor : public StackVisitor {
+   public:
+    explicit NonDebuggableStacksVisitor(Thread* t)
+        : StackVisitor(t, nullptr, StackVisitor::StackWalkKind::kIncludeInlinedFrames) {}
+
+    ~NonDebuggableStacksVisitor() OVERRIDE {}
+
+    bool VisitFrame() OVERRIDE REQUIRES(Locks::mutator_lock_) {
+      if (GetMethod()->IsRuntimeMethod()) {
+        return true;
+      }
+      NonDebuggableClasses::AddNonDebuggableClass(GetMethod()->GetDeclaringClass());
+      if (kIsDebugBuild) {
+        LOG(INFO) << GetMethod()->GetDeclaringClass()->PrettyClass()
+                  << " might not be fully debuggable/deoptimizable due to "
+                  << GetMethod()->PrettyMethod() << " appearing on the stack during zygote fork.";
+      }
+      return true;
+    }
+  };
+  NonDebuggableStacksVisitor visitor(thread);
+  visitor.WalkStack();
+}
+
+static void CollectNonDebuggableClasses() {
+  Runtime* const runtime = Runtime::Current();
+  ScopedSuspendAll suspend("Checking stacks for non-obsoletable methods!", /*long_suspend*/false);
+  MutexLock mu(Thread::Current(), *Locks::thread_list_lock_);
+  runtime->GetThreadList()->ForEach(DoCollectNonDebuggableCallback, nullptr);
+}
+
 static void EnableDebugFeatures(uint32_t debug_flags) {
   // Must match values in com.android.internal.os.Zygote.
   enum {
@@ -131,12 +170,17 @@
     debug_flags &= ~DEBUG_ALWAYS_JIT;
   }
 
+  bool needs_non_debuggable_classes = false;
   if ((debug_flags & DEBUG_JAVA_DEBUGGABLE) != 0) {
     runtime->AddCompilerOption("--debuggable");
     runtime->SetJavaDebuggable(true);
     // Deoptimize the boot image as it may be non-debuggable.
     runtime->DeoptimizeBootImage();
     debug_flags &= ~DEBUG_JAVA_DEBUGGABLE;
+    needs_non_debuggable_classes = true;
+  }
+  if (needs_non_debuggable_classes || kAlwaysCollectNonDebuggableClasses) {
+    CollectNonDebuggableClasses();
   }
 
   if ((debug_flags & DEBUG_NATIVE_DEBUGGABLE) != 0) {
diff --git a/runtime/non_debuggable_classes.cc b/runtime/non_debuggable_classes.cc
new file mode 100644
index 0000000..db121a9
--- /dev/null
+++ b/runtime/non_debuggable_classes.cc
@@ -0,0 +1,42 @@
+/*
+ * Copyright (C) 2017 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "non_debuggable_classes.h"
+
+#include "base/logging.h"
+#include "jni_internal.h"
+#include "mirror/class-inl.h"
+#include "obj_ptr-inl.h"
+#include "ScopedLocalRef.h"
+#include "thread-inl.h"
+
+namespace art {
+
+std::vector<jclass>  NonDebuggableClasses::non_debuggable_classes;
+
+void NonDebuggableClasses::AddNonDebuggableClass(ObjPtr<mirror::Class> klass) {
+  Thread* self = Thread::Current();
+  JNIEnvExt* env = self->GetJniEnv();
+  for (jclass c : non_debuggable_classes) {
+    if (self->DecodeJObject(c)->AsClass() == klass.Ptr()) {
+      return;
+    }
+  }
+  ScopedLocalRef<jclass> lr(env, env->AddLocalReference<jclass>(klass));
+  non_debuggable_classes.push_back(reinterpret_cast<jclass>(env->NewGlobalRef(lr.get())));
+}
+
+}  // namespace art
diff --git a/runtime/non_debuggable_classes.h b/runtime/non_debuggable_classes.h
new file mode 100644
index 0000000..b72afd8
--- /dev/null
+++ b/runtime/non_debuggable_classes.h
@@ -0,0 +1,46 @@
+/*
+ * Copyright (C) 2017 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef ART_RUNTIME_NON_DEBUGGABLE_CLASSES_H_
+#define ART_RUNTIME_NON_DEBUGGABLE_CLASSES_H_
+
+#include <vector>
+
+#include "base/mutex.h"
+#include "jni.h"
+#include "obj_ptr.h"
+
+namespace art {
+
+namespace mirror {
+class Class;
+}  // namespace mirror
+
+struct NonDebuggableClasses {
+ public:
+  static const std::vector<jclass>& GetNonDebuggableClasses() {
+    return non_debuggable_classes;
+  }
+
+  static void AddNonDebuggableClass(ObjPtr<mirror::Class> klass) REQUIRES(Locks::mutator_lock_);
+
+ private:
+  static std::vector<jclass> non_debuggable_classes;
+};
+
+}  // namespace art
+
+#endif  // ART_RUNTIME_NON_DEBUGGABLE_CLASSES_H_
diff --git a/runtime/openjdkjvmti/ti_redefine.cc b/runtime/openjdkjvmti/ti_redefine.cc
index 2d532d1..c0dc16a 100644
--- a/runtime/openjdkjvmti/ti_redefine.cc
+++ b/runtime/openjdkjvmti/ti_redefine.cc
@@ -56,6 +56,7 @@
 #include "mirror/class-inl.h"
 #include "mirror/class_ext.h"
 #include "mirror/object.h"
+#include "non_debuggable_classes.h"
 #include "object_lock.h"
 #include "runtime.h"
 #include "ScopedLocalRef.h"
@@ -245,8 +246,13 @@
     return ERR(UNMODIFIABLE_CLASS);
   }
 
-  // TODO We should check if the class has non-obsoletable methods on the stack
-  LOG(WARNING) << "presence of non-obsoletable methods on stacks is not currently checked";
+  for (jclass c : art::NonDebuggableClasses::GetNonDebuggableClasses()) {
+    if (klass.Get() == art::Thread::Current()->DecodeJObject(c)->AsClass()) {
+      *error_msg = "Class might have stack frames that cannot be made obsolete";
+      return ERR(UNMODIFIABLE_CLASS);
+    }
+  }
+
   return OK;
 }