Fix boot image extension class exclusion.

The code in CompilerDriver was too permissive for arrays and
the code in Transaction was not even checking interfaces.
Move the check to AotClassLinker and fix it.

Test: m test-art-host-gtest
Test: testrunner.py --host --optimizing
Change-Id: I3400b6c0e212e25acf17e3740ba19a8b407e03d3
diff --git a/dex2oat/driver/compiler_driver.cc b/dex2oat/driver/compiler_driver.cc
index c3801f4..99895b1 100644
--- a/dex2oat/driver/compiler_driver.cc
+++ b/dex2oat/driver/compiler_driver.cc
@@ -29,6 +29,7 @@
 #include "android-base/logging.h"
 #include "android-base/strings.h"
 
+#include "aot_class_linker.h"
 #include "art_field-inl.h"
 #include "art_method-inl.h"
 #include "base/arena_allocator.h"
@@ -1283,66 +1284,7 @@
     if (heap->ObjectIsInBootImageSpace(klass)) {
       return false;  // Already included in the boot image we're compiling against.
     }
-
-    // When compiling a boot image extension, we must not include classes
-    // that are defined in dex files belonging to the boot image we're
-    // compiling against but not actually present in that boot image.
-
-    // Treat arrays and primitive types specially because they do not have a DexCache that we
-    // can use to check whether the dex file belongs to the boot image we're compiling against.
-    DCHECK(!klass->IsPrimitive());  // Primitive classes must be in the primary boot image.
-    if (klass->IsArrayClass()) {
-      DCHECK(heap->ObjectIsInBootImageSpace(klass->GetIfTable()));  // IfTable is OK.
-      // Arrays of all dimensions are tied to the dex file of the non-array component type.
-      ObjPtr<mirror::Class> component_type = klass;
-      do {
-        component_type = component_type->GetComponentType();
-      } while (component_type->IsArrayClass());
-      return !component_type->IsPrimitive() &&
-             !heap->ObjectIsInBootImageSpace(component_type->GetDexCache());
-    }
-
-    // Check the class itself.
-    if (heap->ObjectIsInBootImageSpace(klass->GetDexCache())) {
-      return false;
-    }
-
-    // Check superclasses.
-    ObjPtr<mirror::Class> superclass = klass->GetSuperClass();
-    while (!heap->ObjectIsInBootImageSpace(superclass)) {
-      DCHECK(superclass != nullptr);  // Cannot skip Object which is in the primary boot image.
-      if (heap->ObjectIsInBootImageSpace(superclass->GetDexCache())) {
-        return false;
-      }
-      superclass = superclass->GetSuperClass();
-    }
-
-    // Check IfTable. This includes direct and indirect interfaces.
-    ObjPtr<mirror::IfTable> if_table = klass->GetIfTable();
-    for (size_t i = 0, num_interfaces = klass->GetIfTableCount(); i < num_interfaces; ++i) {
-      ObjPtr<mirror::Class> interface = if_table->GetInterface(i);
-      DCHECK(interface != nullptr);
-      if (!heap->ObjectIsInBootImageSpace(interface) &&
-          heap->ObjectIsInBootImageSpace(interface->GetDexCache())) {
-        return false;
-      }
-    }
-
-    if (kIsDebugBuild) {
-      // All virtual methods must come from classes we have already checked above.
-      PointerSize pointer_size = Runtime::Current()->GetClassLinker()->GetImagePointerSize();
-      ObjPtr<mirror::Class> k = klass;
-      while (!heap->ObjectIsInBootImageSpace(k)) {
-        for (auto& m : k->GetVirtualMethods(pointer_size)) {
-          ObjPtr<mirror::Class> declaring_class = m.GetDeclaringClass();
-          CHECK(heap->ObjectIsInBootImageSpace(declaring_class) ||
-                !heap->ObjectIsInBootImageSpace(declaring_class->GetDexCache()));
-        }
-        k = k->GetSuperClass();
-      }
-    }
-
-    return true;
+    return AotClassLinker::CanReferenceInBootImageExtension(klass, heap);
   }
 
   mutable VariableSizedHandleScope hs_;
diff --git a/runtime/aot_class_linker.cc b/runtime/aot_class_linker.cc
index d0ceaf7..77faa2d 100644
--- a/runtime/aot_class_linker.cc
+++ b/runtime/aot_class_linker.cc
@@ -19,6 +19,7 @@
 #include "class_status.h"
 #include "compiler_callbacks.h"
 #include "dex/class_reference.h"
+#include "gc/heap.h"
 #include "handle_scope-inl.h"
 #include "mirror/class-inl.h"
 #include "runtime.h"
@@ -124,4 +125,75 @@
   return ClassLinker::PerformClassVerification(self, klass, log_level, error_msg);
 }
 
+bool AotClassLinker::CanReferenceInBootImageExtension(ObjPtr<mirror::Class> klass, gc::Heap* heap) {
+  // Do not allow referencing a class or instance of a class defined in a dex file
+  // belonging to the boot image we're compiling against but not itself in the boot image;
+  // or a class referencing such classes as component type, superclass or interface.
+  // Allowing this could yield duplicate class objects from multiple extensions.
+
+  if (heap->ObjectIsInBootImageSpace(klass)) {
+    return true;  // Already included in the boot image we're compiling against.
+  }
+
+  // Treat arrays and primitive types specially because they do not have a DexCache that we
+  // can use to check whether the dex file belongs to the boot image we're compiling against.
+  DCHECK(!klass->IsPrimitive());  // Primitive classes must be in the primary boot image.
+  if (klass->IsArrayClass()) {
+    DCHECK(heap->ObjectIsInBootImageSpace(klass->GetIfTable()));  // IfTable is OK.
+    // Arrays of all dimensions are tied to the dex file of the non-array component type.
+    do {
+      klass = klass->GetComponentType();
+    } while (klass->IsArrayClass());
+    if (klass->IsPrimitive()) {
+      return false;
+    }
+    // Do not allow arrays of erroneous classes (the array class is not itself erroneous).
+    if (klass->IsErroneous()) {
+      return false;
+    }
+  }
+
+  // Check the class itself.
+  if (heap->ObjectIsInBootImageSpace(klass->GetDexCache())) {
+    return false;
+  }
+
+  // Check superclasses.
+  ObjPtr<mirror::Class> superclass = klass->GetSuperClass();
+  while (!heap->ObjectIsInBootImageSpace(superclass)) {
+    DCHECK(superclass != nullptr);  // Cannot skip Object which is in the primary boot image.
+    if (heap->ObjectIsInBootImageSpace(superclass->GetDexCache())) {
+      return false;
+    }
+    superclass = superclass->GetSuperClass();
+  }
+
+  // Check IfTable. This includes direct and indirect interfaces.
+  ObjPtr<mirror::IfTable> if_table = klass->GetIfTable();
+  for (size_t i = 0, num_interfaces = klass->GetIfTableCount(); i < num_interfaces; ++i) {
+    ObjPtr<mirror::Class> interface = if_table->GetInterface(i);
+    DCHECK(interface != nullptr);
+    if (!heap->ObjectIsInBootImageSpace(interface) &&
+        heap->ObjectIsInBootImageSpace(interface->GetDexCache())) {
+      return false;
+    }
+  }
+
+  if (kIsDebugBuild) {
+    // All virtual methods must come from classes we have already checked above.
+    PointerSize pointer_size = Runtime::Current()->GetClassLinker()->GetImagePointerSize();
+    ObjPtr<mirror::Class> k = klass;
+    while (!heap->ObjectIsInBootImageSpace(k)) {
+      for (auto& m : k->GetVirtualMethods(pointer_size)) {
+        ObjPtr<mirror::Class> declaring_class = m.GetDeclaringClass();
+        CHECK(heap->ObjectIsInBootImageSpace(declaring_class) ||
+              !heap->ObjectIsInBootImageSpace(declaring_class->GetDexCache()));
+      }
+      k = k->GetSuperClass();
+    }
+  }
+
+  return true;
+}
+
 }  // namespace art
diff --git a/runtime/aot_class_linker.h b/runtime/aot_class_linker.h
index 74da33d..2b50c46 100644
--- a/runtime/aot_class_linker.h
+++ b/runtime/aot_class_linker.h
@@ -20,6 +20,11 @@
 #include "class_linker.h"
 
 namespace art {
+
+namespace gc {
+class Heap;
+}  // namespace gc
+
 // AotClassLinker is only used for AOT compiler, which includes some logic for class initialization
 // which will only be used in pre-compilation.
 class AotClassLinker : public ClassLinker {
@@ -27,6 +32,9 @@
   explicit AotClassLinker(InternTable *intern_table);
   ~AotClassLinker();
 
+  static bool CanReferenceInBootImageExtension(ObjPtr<mirror::Class> klass, gc::Heap* heap)
+      REQUIRES_SHARED(Locks::mutator_lock_);
+
  protected:
   // Overridden version of PerformClassVerification allows skipping verification if the class was
   // previously verified but unloaded.
diff --git a/runtime/transaction.cc b/runtime/transaction.cc
index 47e59a9..46bbea3 100644
--- a/runtime/transaction.cc
+++ b/runtime/transaction.cc
@@ -18,6 +18,7 @@
 
 #include <android-base/logging.h>
 
+#include "aot_class_linker.h"
 #include "base/mutex-inl.h"
 #include "base/stl_util.h"
 #include "gc/accounting/card_table-inl.h"
@@ -141,26 +142,8 @@
     return false;  // No constraints for boot image.
   } else {
     // Boot image extension.
-    // Do not allow storing references to a class or instances of a class defined in a dex file
-    // belonging to the boot image we're compiling against but not itself in the boot image.
-    // Allowing this could yield duplicate class objects from multiple extensions.
-    if (heap->ObjectIsInBootImageSpace(value)) {
-      return false;  // References to boot image objects are OK.
-    }
     ObjPtr<mirror::Class> klass = value->IsClass() ? value->AsClass() : value->GetClass();
-    if (!value->IsClass() && heap->ObjectIsInBootImageSpace(klass)) {
-      // Instances of boot image classes are OK.
-      DCHECK(klass->IsInitialized());
-      return false;
-    }
-    // For arrays we need to determine the dex file based on the element type.
-    while (klass->IsArrayClass()) {
-      klass = klass->GetComponentType();
-    }
-    if (klass->IsPrimitive() || heap->ObjectIsInBootImageSpace(klass->GetDexCache())) {
-      return true;  // Boot image dex file but not boot image `klass`.
-    }
-    return false;
+    return !AotClassLinker::CanReferenceInBootImageExtension(klass, heap);
   }
 }