8006439: Improve MethodHandles coverage

Reviewed-by: jrose, twisti

src/share/classes/java/lang/invoke/MethodHandleImpl.java

@@ -801,12 +801,11 @@ import static java.lang.invoke.MethodHandles.Lookup.IMPL_LOOKUP;
         static
         MethodHandle bindCaller(MethodHandle mh, Class<?> hostClass) {
             // Do not use this function to inject calls into system classes.
-            if (hostClass == null) {
-                hostClass = C_Trampoline;
-            } else if (hostClass.isArray() ||
+            if (hostClass == null
+                ||    (hostClass.isArray() ||
                        hostClass.isPrimitive() ||
                        hostClass.getName().startsWith("java.") ||
-                       hostClass.getName().startsWith("sun.")) {
+                       hostClass.getName().startsWith("sun."))) {
                 throw new InternalError();  // does not happen, and should not anyway
             }
             // For simplicity, convert mh to a varargs-like method.
@@ -816,23 +815,6 @@ import static java.lang.invoke.MethodHandles.Lookup.IMPL_LOOKUP;
             return restoreToType(bccInvoker.bindTo(vamh), mh.type());
         }
 
-        // This class ("Trampoline") is known to be inside a dead-end class loader.
-        // Inject all doubtful calls into this class.
-        private static Class<?> C_Trampoline;
-        static {
-            Class<?> tramp = null;
-            try {
-                final int FRAME_COUNT_ARG = 1;  // [0] Reflection [1] Trampoline
-                java.lang.reflect.Method gcc = sun.reflect.Reflection.class.getMethod("getCallerClass", int.class);
-                tramp = (Class<?>) sun.reflect.misc.MethodUtil.invoke(gcc, null, new Object[]{ FRAME_COUNT_ARG });
-                if (tramp.getClassLoader() == BindCaller.class.getClassLoader())
-                    throw new RuntimeException(tramp.getName()+" class loader");
-            } catch (Throwable ex) {
-                throw new InternalError(ex);
-            }
-            C_Trampoline = tramp;
-        }
-
         private static MethodHandle makeInjectedInvoker(Class<?> hostClass) {
             Class<?> bcc = UNSAFE.defineAnonymousClass(hostClass, T_BYTES, null);
             if (hostClass.getClassLoader() != bcc.getClassLoader())

src/share/classes/java/lang/invoke/MethodHandleNatives.java

@@ -393,11 +393,14 @@ class MethodHandleNatives {
      */
     // FIXME: Replace this pattern match by an annotation @sun.reflect.CallerSensitive.
     static boolean isCallerSensitive(MemberName mem) {
-        assert(mem.isInvocable());
+        if (!mem.isInvocable())  return false;  // fields are not caller sensitive
         Class<?> defc = mem.getDeclaringClass();
         switch (mem.getName()) {
         case "doPrivileged":
+        case "doPrivilegedWithCombiner":
             return defc == java.security.AccessController.class;
+        case "checkMemberAccess":
+            return canBeCalledVirtual(mem, java.lang.SecurityManager.class);
         case "getUnsafe":
             return defc == sun.misc.Unsafe.class;
         case "lookup":
@@ -455,7 +458,7 @@ class MethodHandleNatives {
             if (defc == java.util.concurrent.atomic.AtomicReferenceFieldUpdater.class)  return true;
             break;
         case "getContextClassLoader":
-            return defc == java.lang.Thread.class;
+            return canBeCalledVirtual(mem, java.lang.Thread.class);
         case "getPackage":
         case "getPackages":
             return defc == java.lang.Package.class;
@@ -473,6 +476,8 @@ class MethodHandleNatives {
             break;
         case "getCallerClassLoader":
             return defc == java.lang.ClassLoader.class;
+        case "registerAsParallelCapable":
+            return canBeCalledVirtual(mem, java.lang.ClassLoader.class);
         case "getProxyClass":
         case "newProxyInstance":
             return defc == java.lang.reflect.Proxy.class;
@@ -494,4 +499,11 @@ class MethodHandleNatives {
             throw new InternalError(e);
         }
     }
+    static boolean canBeCalledVirtual(MemberName symbolicRef, Class<?> definingClass) {
+        Class<?> symbolicRefClass = symbolicRef.getDeclaringClass();
+        if (symbolicRefClass == definingClass)  return true;
+        if (symbolicRef.isStatic() || symbolicRef.isPrivate())  return false;
+        return (definingClass.isAssignableFrom(symbolicRefClass) ||  // Msym overrides Mdef
+                symbolicRefClass.isInterface());                     // Mdef implements Msym
+    }
 }

src/share/classes/java/lang/invoke/MethodHandles.java

@@ -598,7 +598,8 @@ public class MethodHandles {
         MethodHandle findStatic(Class<?> refc, String name, MethodType type) throws NoSuchMethodException, IllegalAccessException {
             MemberName method = resolveOrFail(REF_invokeStatic, refc, name, type);
             checkSecurityManager(refc, method);  // stack walk magic: do not refactor
-            return getDirectMethod(REF_invokeStatic, refc, method);
+            Class<?> callerClass = findBoundCallerClass(method);  // stack walk magic: do not refactor
+            return getDirectMethod(REF_invokeStatic, refc, method, callerClass);
         }
 
         /**
@@ -652,7 +653,8 @@ public class MethodHandles {
             byte refKind = (refc.isInterface() ? REF_invokeInterface : REF_invokeVirtual);
             MemberName method = resolveOrFail(refKind, refc, name, type);
             checkSecurityManager(refc, method);  // stack walk magic: do not refactor
-            return getDirectMethod(refKind, refc, method);
+            Class<?> callerClass = findBoundCallerClass(method);
+            return getDirectMethod(refKind, refc, method, callerClass);
         }
         private MethodHandle findVirtualForMH(String name, MethodType type) {
             // these names require special lookups because of the implicit MethodType argument
@@ -736,7 +738,8 @@ public class MethodHandles {
             Lookup specialLookup = this.in(specialCaller);
             MemberName method = specialLookup.resolveOrFail(REF_invokeSpecial, refc, name, type);
             checkSecurityManager(refc, method);  // stack walk magic: do not refactor
-            return specialLookup.getDirectMethod(REF_invokeSpecial, refc, method);
+            Class<?> callerClass = findBoundCallerClass(method);
+            return specialLookup.getDirectMethod(REF_invokeSpecial, refc, method, callerClass);
         }
 
         /**
@@ -879,7 +882,8 @@ return mh1;
             Class<? extends Object> refc = receiver.getClass(); // may get NPE
             MemberName method = resolveOrFail(REF_invokeSpecial, refc, name, type);
             checkSecurityManager(refc, method);  // stack walk magic: do not refactor
-            MethodHandle mh = getDirectMethodNoRestrict(REF_invokeSpecial, refc, method);
+            Class<?> callerClass = findBoundCallerClass(method);  // stack walk magic: do not refactor
+            MethodHandle mh = getDirectMethodNoRestrict(REF_invokeSpecial, refc, method, callerClass);
             return mh.bindReceiver(receiver).setVarargs(method);
         }
 
@@ -910,8 +914,9 @@ return mh1;
             if (refKind == REF_invokeSpecial)
                 refKind = REF_invokeVirtual;
             assert(method.isMethod());
+            Class<?> callerClass = findBoundCallerClass(method);  // stack walk magic: do not refactor
             Lookup lookup = m.isAccessible() ? IMPL_LOOKUP : this;
-            return lookup.getDirectMethod(refKind, method.getDeclaringClass(), method);
+            return lookup.getDirectMethod(refKind, method.getDeclaringClass(), method, callerClass);
         }
 
         /**
@@ -940,8 +945,9 @@ return mh1;
             Lookup specialLookup = this.in(specialCaller);
             MemberName method = new MemberName(m, true);
             assert(method.isMethod());
+            Class<?> callerClass = findBoundCallerClass(method);  // stack walk magic: do not refactor
             // ignore m.isAccessible:  this is a new kind of access
-            return specialLookup.getDirectMethod(REF_invokeSpecial, method.getDeclaringClass(), method);
+            return specialLookup.getDirectMethod(REF_invokeSpecial, method.getDeclaringClass(), method, callerClass);
         }
 
         /**
@@ -1039,8 +1045,30 @@ return mh1;
                 throw new MemberName(refc).makeAccessException("symbolic reference class is not public", this);
         }
 
+        /**
+         * Find my trustable caller class if m is a caller sensitive method.
+         * If this lookup object has private access, then the caller class is the lookupClass.
+         * Otherwise, it is the caller of the currently executing public API method (e.g., findVirtual).
+         * This is the same caller class as is used by checkSecurityManager.
+         * This function performs stack walk magic: do not refactor it.
+         */
+        Class<?> findBoundCallerClass(MemberName m) {
+            Class<?> callerClass = null;
+            if (MethodHandleNatives.isCallerSensitive(m)) {
+                // Do not refactor this to a more "logical" place, since it is stack walk magic.
+                // Note that this is the same expression as in Step 2 below in checkSecurityManager.
+                callerClass = ((allowedModes & PRIVATE) != 0
+                               ? lookupClass  // for strong access modes, no extra check
+                               // next line does stack walk magic; do not refactor:
+                               : getCallerClassAtEntryPoint(true));
+            }
+            return callerClass;
+        }
         /**
          * Perform necessary <a href="MethodHandles.Lookup.html#secmgr">access checks</a>.
+         * Determines a trustable caller class to compare with refc, the symbolic reference class.
+         * If this lookup object has private access, then the caller class is the lookupClass.
+         * Otherwise, it is the caller of the currently executing public API method (e.g., findVirtual).
          * This function performs stack walk magic: do not refactor it.
          */
         void checkSecurityManager(Class<?> refc, MemberName m) {
@@ -1195,22 +1223,22 @@ return mh1;
             return mh.viewAsType(narrowType);
         }
 
-        private MethodHandle getDirectMethod(byte refKind, Class<?> refc, MemberName method) throws IllegalAccessException {
+        private MethodHandle getDirectMethod(byte refKind, Class<?> refc, MemberName method, Class<?> callerClass) throws IllegalAccessException {
             return getDirectMethodCommon(refKind, refc, method,
                     (refKind == REF_invokeSpecial ||
                         (MethodHandleNatives.refKindHasReceiver(refKind) &&
-                            restrictProtectedReceiver(method))));
+                            restrictProtectedReceiver(method))), callerClass);
         }
-        private MethodHandle getDirectMethodNoRestrict(byte refKind, Class<?> refc, MemberName method) throws IllegalAccessException {
-            return getDirectMethodCommon(refKind, refc, method, false);
+        private MethodHandle getDirectMethodNoRestrict(byte refKind, Class<?> refc, MemberName method, Class<?> callerClass) throws IllegalAccessException {
+            return getDirectMethodCommon(refKind, refc, method, false, callerClass);
         }
         private MethodHandle getDirectMethodCommon(byte refKind, Class<?> refc, MemberName method,
-                                                   boolean doRestrict) throws IllegalAccessException {
+                                                   boolean doRestrict, Class<?> callerClass) throws IllegalAccessException {
             checkMethod(refKind, refc, method);
             if (method.isMethodHandleInvoke())
                 return fakeMethodHandleInvoke(method);
             MethodHandle mh = DirectMethodHandle.make(refc, method);
-            mh = maybeBindCaller(method, mh);
+            mh = maybeBindCaller(method, mh, callerClass);
             mh = mh.setVarargs(method);
             if (doRestrict)
                 mh = restrictReceiver(method, mh, lookupClass());
@@ -1219,12 +1247,14 @@ return mh1;
         private MethodHandle fakeMethodHandleInvoke(MemberName method) {
             return throwException(method.getReturnType(), UnsupportedOperationException.class);
         }
-        private MethodHandle maybeBindCaller(MemberName method, MethodHandle mh) throws IllegalAccessException {
+        private MethodHandle maybeBindCaller(MemberName method, MethodHandle mh,
+                                             Class<?> callerClass)
+                                             throws IllegalAccessException {
             if (allowedModes == TRUSTED || !MethodHandleNatives.isCallerSensitive(method))
                 return mh;
             Class<?> hostClass = lookupClass;
             if ((allowedModes & PRIVATE) == 0)  // caller must use full-power lookup
-                hostClass = null;
+                hostClass = callerClass;  // callerClass came from a security manager style stack walk
             MethodHandle cbmh = MethodHandleImpl.bindCaller(mh, hostClass);
             // Note: caller will apply varargs after this step happens.
             return cbmh;
@@ -1262,7 +1292,7 @@ return mh1;
             } else if (MethodHandleNatives.refKindIsMethod(refKind)) {
                 MemberName method = (resolved != null) ? resolved
                         : resolveOrFail(refKind, defc, name, (MethodType) type);
-                return getDirectMethod(refKind, defc, method);
+                return getDirectMethod(refKind, defc, method, lookupClass);
             } else if (refKind == REF_newInvokeSpecial) {
                 assert(name == null || name.equals("<init>"));
                 MemberName ctor = (resolved != null) ? resolved