7191102: nightly failures after JSR 292 lazy method handle update (round 3)

Reviewed-by: twisti, kvn
diff --git a/jdk/src/share/classes/java/lang/invoke/Invokers.java b/jdk/src/share/classes/java/lang/invoke/Invokers.java
index 904a8c7..0e3dcf6 100644
--- a/jdk/src/share/classes/java/lang/invoke/Invokers.java
+++ b/jdk/src/share/classes/java/lang/invoke/Invokers.java
@@ -74,8 +74,18 @@
         MethodHandle invoker = exactInvoker;
         if (invoker != null)  return invoker;
         MethodType mtype = targetType;
-        LambdaForm lform = invokeForm(mtype, MethodTypeForm.LF_EX_INVOKER);
-        invoker = BoundMethodHandle.bindSingle(mtype.invokerType(), lform, mtype);
+        MethodType invokerType = mtype.invokerType();
+        LambdaForm lform;
+        final int MTYPE_ARG_APPENDED = 1;  // argument count for appended mtype value
+        if (mtype.parameterSlotCount() <= MethodType.MAX_MH_INVOKER_ARITY - MTYPE_ARG_APPENDED) {
+            lform = invokeForm(mtype, false, MethodTypeForm.LF_EX_INVOKER);
+            invoker = BoundMethodHandle.bindSingle(invokerType, lform, mtype);
+        } else {
+            // At maximum arity, we cannot afford an extra mtype argument,
+            // so build a fully customized (non-cached) invoker form.
+            lform = invokeForm(mtype, true, MethodTypeForm.LF_EX_INVOKER);
+            invoker = SimpleMethodHandle.make(invokerType, lform);
+        }
         assert(checkInvoker(invoker));
         exactInvoker = invoker;
         return invoker;
@@ -85,9 +95,20 @@
         MethodHandle invoker = generalInvoker;
         if (invoker != null)  return invoker;
         MethodType mtype = targetType;
-        prepareForGenericCall(mtype);
-        LambdaForm lform = invokeForm(mtype, MethodTypeForm.LF_GEN_INVOKER);
-        invoker = BoundMethodHandle.bindSingle(mtype.invokerType(), lform, mtype);
+        MethodType invokerType = mtype.invokerType();
+        LambdaForm lform;
+        final int MTYPE_ARG_APPENDED = 1;  // argument count for appended mtype value
+        assert(GENERIC_INVOKER_SLOP >= MTYPE_ARG_APPENDED);
+        if (mtype.parameterSlotCount() <= MethodType.MAX_MH_INVOKER_ARITY - GENERIC_INVOKER_SLOP) {
+            prepareForGenericCall(mtype);
+            lform = invokeForm(mtype, false, MethodTypeForm.LF_GEN_INVOKER);
+            invoker = BoundMethodHandle.bindSingle(invokerType, lform, mtype);
+        } else {
+            // At maximum arity, we cannot afford an extra mtype argument,
+            // so build a fully customized (non-cached) invoker form.
+            lform = invokeForm(mtype, true, MethodTypeForm.LF_GEN_INVOKER);
+            invoker = SimpleMethodHandle.make(invokerType, lform);
+        }
         assert(checkInvoker(invoker));
         generalInvoker = invoker;
         return invoker;
@@ -102,6 +123,7 @@
     }
 
     static MemberName invokeBasicMethod(MethodType type) {
+        type = type.basicType();
         String name = "invokeBasic";
         try {
             //Lookup.findVirtual(MethodHandle.class, name, type);
@@ -135,9 +157,31 @@
     /*non-public*/ MethodHandle spreadInvoker(int leadingArgCount) {
         MethodHandle vaInvoker = spreadInvokers[leadingArgCount];
         if (vaInvoker != null)  return vaInvoker;
-        MethodHandle gInvoker = generalInvoker();
         int spreadArgCount = targetType.parameterCount() - leadingArgCount;
-        vaInvoker = gInvoker.asSpreader(Object[].class, spreadArgCount);
+        MethodType spreadInvokerType = targetType
+            .replaceParameterTypes(leadingArgCount, targetType.parameterCount(), Object[].class);
+        if (targetType.parameterSlotCount() <= MethodType.MAX_MH_INVOKER_ARITY) {
+            // Factor sinvoker.invoke(mh, a) into ginvoker.asSpreader().invoke(mh, a)
+            // where ginvoker.invoke(mh, a*) => mh.invoke(a*).
+            MethodHandle genInvoker = generalInvoker();
+            vaInvoker = genInvoker.asSpreader(Object[].class, spreadArgCount);
+        } else {
+            // Cannot build a general invoker here of type ginvoker.invoke(mh, a*[254]).
+            // Instead, factor sinvoker.invoke(mh, a) into ainvoker.invoke(filter(mh), a)
+            // where filter(mh) == mh.asSpreader(Object[], spreadArgCount)
+            MethodHandle arrayInvoker = MethodHandles.exactInvoker(spreadInvokerType);
+            MethodHandle makeSpreader;
+            try {
+                makeSpreader = IMPL_LOOKUP
+                    .findVirtual(MethodHandle.class, "asSpreader",
+                        MethodType.methodType(MethodHandle.class, Class.class, int.class));
+            } catch (ReflectiveOperationException ex) {
+                throw new InternalError(ex);
+            }
+            makeSpreader = MethodHandles.insertArguments(makeSpreader, 1, Object[].class, spreadArgCount);
+            vaInvoker = MethodHandles.filterArgument(arrayInvoker, 0, makeSpreader);
+        }
+        assert(vaInvoker.type().equals(spreadInvokerType.invokerType()));
         spreadInvokers[leadingArgCount] = vaInvoker;
         return vaInvoker;
     }
@@ -171,7 +215,7 @@
                     .findStatic(CallSite.class, "uninitializedCallSite",
                                 MethodType.methodType(Empty.class));
             } catch (ReflectiveOperationException ex) {
-                throw new RuntimeException(ex);
+                throw new InternalError(ex);
             }
         }
         invoker = MethodHandles.explicitCastArguments(invoker, MethodType.methodType(targetType.returnType()));
@@ -185,30 +229,39 @@
         return "Invokers"+targetType;
     }
 
-    private static MethodType fixMethodType(Class<?> callerClass, Object type) {
-        if (type instanceof MethodType)
-            return (MethodType) type;
-        else
-            return MethodType.fromMethodDescriptorString((String)type, callerClass.getClassLoader());
-    }
-
-    static MemberName exactInvokerMethod(Class<?> callerClass, Object type, Object[] appendixResult) {
-        MethodType mtype = fixMethodType(callerClass, type);
-        LambdaForm lform = invokeForm(mtype, MethodTypeForm.LF_EX_LINKER);
-        appendixResult[0] = mtype;
+    static MemberName exactInvokerMethod(MethodType mtype, Object[] appendixResult) {
+        LambdaForm lform;
+        final int MTYPE_ARG_APPENDED = 1;  // argument count for appended mtype value
+        if (mtype.parameterSlotCount() <= MethodType.MAX_MH_ARITY - MTYPE_ARG_APPENDED) {
+            lform = invokeForm(mtype, false, MethodTypeForm.LF_EX_LINKER);
+            appendixResult[0] = mtype;
+        } else {
+            lform = invokeForm(mtype, true, MethodTypeForm.LF_EX_LINKER);
+        }
         return lform.vmentry;
     }
 
-    static MemberName genericInvokerMethod(Class<?> callerClass, Object type, Object[] appendixResult) {
-        MethodType mtype = fixMethodType(callerClass, type);
-        LambdaForm lform = invokeForm(mtype, MethodTypeForm.LF_GEN_LINKER);
-        prepareForGenericCall(mtype);
-        appendixResult[0] = mtype;
+    static MemberName genericInvokerMethod(MethodType mtype, Object[] appendixResult) {
+        LambdaForm lform;
+        final int MTYPE_ARG_APPENDED = 1;  // argument count for appended mtype value
+        if (mtype.parameterSlotCount() <= MethodType.MAX_MH_ARITY - (MTYPE_ARG_APPENDED + GENERIC_INVOKER_SLOP)) {
+            lform = invokeForm(mtype, false, MethodTypeForm.LF_GEN_LINKER);
+            appendixResult[0] = mtype;
+            prepareForGenericCall(mtype);
+        } else {
+            lform = invokeForm(mtype, true, MethodTypeForm.LF_GEN_LINKER);
+        }
         return lform.vmentry;
     }
 
-    private static LambdaForm invokeForm(MethodType mtype, int which) {
-        mtype = mtype.basicType();  // normalize Z to I, String to Object, etc.
+    private static LambdaForm invokeForm(MethodType mtype, boolean customized, int which) {
+        boolean isCached;
+        if (!customized) {
+            mtype = mtype.basicType();  // normalize Z to I, String to Object, etc.
+            isCached = true;
+        } else {
+            isCached = false;  // maybe cache if mtype == mtype.basicType()
+        }
         boolean isLinker, isGeneric;
         String debugName;
         switch (which) {
@@ -218,27 +271,32 @@
         case MethodTypeForm.LF_GEN_INVOKER: isLinker = false; isGeneric = true;  debugName = "invoker"; break;
         default: throw new InternalError();
         }
-        LambdaForm lform = mtype.form().cachedLambdaForm(which);
-        if (lform != null)  return lform;
+        LambdaForm lform;
+        if (isCached) {
+            lform = mtype.form().cachedLambdaForm(which);
+            if (lform != null)  return lform;
+        }
         // exactInvokerForm (Object,Object)Object
         //   link with java.lang.invoke.MethodHandle.invokeBasic(MethodHandle,Object,Object)Object/invokeSpecial
         final int THIS_MH      = 0;
         final int CALL_MH      = THIS_MH + (isLinker ? 0 : 1);
         final int ARG_BASE     = CALL_MH + 1;
         final int OUTARG_LIMIT = ARG_BASE + mtype.parameterCount();
-        final int INARG_LIMIT  = OUTARG_LIMIT + (isLinker ? 1 : 0);
+        final int INARG_LIMIT  = OUTARG_LIMIT + (isLinker && !customized ? 1 : 0);
         int nameCursor = OUTARG_LIMIT;
-        final int MTYPE_ARG    = nameCursor++;  // might be last in-argument
+        final int MTYPE_ARG    = customized ? -1 : nameCursor++;  // might be last in-argument
         final int CHECK_TYPE   = nameCursor++;
         final int LINKER_CALL  = nameCursor++;
         MethodType invokerFormType = mtype.invokerType();
         if (isLinker) {
-            invokerFormType = invokerFormType.appendParameterTypes(MemberName.class);
+            if (!customized)
+                invokerFormType = invokerFormType.appendParameterTypes(MemberName.class);
         } else {
             invokerFormType = invokerFormType.invokerType();
         }
         Name[] names = arguments(nameCursor - INARG_LIMIT, invokerFormType);
-        assert(names.length == nameCursor);
+        assert(names.length == nameCursor)
+                : Arrays.asList(mtype, customized, which, nameCursor, names.length);
         if (MTYPE_ARG >= INARG_LIMIT) {
             assert(names[MTYPE_ARG] == null);
             names[MTYPE_ARG] = BoundMethodHandle.getSpeciesData("L").getterName(names[THIS_MH], 0);
@@ -248,31 +306,42 @@
         // Make the final call.  If isGeneric, then prepend the result of type checking.
         MethodType outCallType;
         Object[] outArgs;
+        Object mtypeArg = (customized ? mtype : names[MTYPE_ARG]);
         if (!isGeneric) {
-            names[CHECK_TYPE] = new Name(NF_checkExactType, names[CALL_MH], names[MTYPE_ARG]);
+            names[CHECK_TYPE] = new Name(NF_checkExactType, names[CALL_MH], mtypeArg);
             // mh.invokeExact(a*):R => checkExactType(mh, TYPEOF(a*:R)); mh.invokeBasic(a*)
             outArgs = Arrays.copyOfRange(names, CALL_MH, OUTARG_LIMIT, Object[].class);
             outCallType = mtype;
+        } else if (customized) {
+            names[CHECK_TYPE] = new Name(NF_asType, names[CALL_MH], mtypeArg);
+            // mh.invokeGeneric(a*):R =>
+            //  let mt=TYPEOF(a*:R), tmh=asType(mh, mt);
+            //    tmh.invokeBasic(a*)
+            outArgs = Arrays.copyOfRange(names, CALL_MH, OUTARG_LIMIT, Object[].class);
+            outCallType = mtype;
         } else {
-            names[CHECK_TYPE] = new Name(NF_checkGenericType, names[CALL_MH], names[MTYPE_ARG]);
+            names[CHECK_TYPE] = new Name(NF_checkGenericType, names[CALL_MH], mtypeArg);
             // mh.invokeGeneric(a*):R =>
             //  let mt=TYPEOF(a*:R), gamh=checkGenericType(mh, mt);
             //    gamh.invokeBasic(mt, mh, a*)
             final int PREPEND_GAMH = 0, PREPEND_MT = 1, PREPEND_COUNT = 2;
+            assert(GENERIC_INVOKER_SLOP == PREPEND_COUNT);
             outArgs = Arrays.copyOfRange(names, CALL_MH, OUTARG_LIMIT + PREPEND_COUNT, Object[].class);
             // prepend arguments:
             System.arraycopy(outArgs, 0, outArgs, PREPEND_COUNT, outArgs.length - PREPEND_COUNT);
             outArgs[PREPEND_GAMH] = names[CHECK_TYPE];
-            outArgs[PREPEND_MT] = names[MTYPE_ARG];
+            outArgs[PREPEND_MT] = mtypeArg;
             outCallType = mtype.insertParameterTypes(0, MethodType.class, MethodHandle.class);
         }
         names[LINKER_CALL] = new Name(invokeBasicMethod(outCallType), outArgs);
         lform = new LambdaForm(debugName, INARG_LIMIT, names);
         if (isLinker)
             lform.compileToBytecode();  // JVM needs a real methodOop
-        lform = mtype.form().setCachedLambdaForm(which, lform);
+        if (isCached)
+            lform = mtype.form().setCachedLambdaForm(which, lform);
         return lform;
     }
+    private static final int GENERIC_INVOKER_SLOP = 2;  // used elsewhere to avoid arity problems
 
     /*non-public*/ static
     WrongMethodTypeException newWrongMethodTypeException(MethodType actual, MethodType expected) {
@@ -370,6 +439,7 @@
     // Local constant functions:
     private static final NamedFunction NF_checkExactType;
     private static final NamedFunction NF_checkGenericType;
+    private static final NamedFunction NF_asType;
     private static final NamedFunction NF_getCallSiteTarget;
     static {
         try {
@@ -377,6 +447,8 @@
                     .getDeclaredMethod("checkExactType", Object.class, Object.class));
             NF_checkGenericType = new NamedFunction(Invokers.class
                     .getDeclaredMethod("checkGenericType", Object.class, Object.class));
+            NF_asType = new NamedFunction(MethodHandle.class
+                    .getDeclaredMethod("asType", MethodType.class));
             NF_getCallSiteTarget = new NamedFunction(Invokers.class
                     .getDeclaredMethod("getCallSiteTarget", Object.class));
             NF_checkExactType.resolve();