8227601: Better collection of references

Reviewed-by: smarks, ahgross, skoivu, rhalade

src/share/classes/sun/rmi/registry/RegistryImpl_Skel.java

@@ -27,13 +27,8 @@
 package sun.rmi.registry;
 
 import java.io.IOException;
-import java.io.InputStream;
-import java.rmi.AccessException;
-import java.rmi.server.RemoteCall;
 
-import sun.rmi.transport.Connection;
 import sun.rmi.transport.StreamRemoteCall;
-import sun.rmi.transport.tcp.TCPConnection;
 
 /**
  * Skeleton to dispatch RegistryImpl methods.
@@ -56,7 +51,7 @@ public final class RegistryImpl_Skel
         return operations.clone();
     }
 
-    public void dispatch(java.rmi.Remote obj, java.rmi.server.RemoteCall call, int opnum, long hash)
+    public void dispatch(java.rmi.Remote obj, java.rmi.server.RemoteCall remoteCall, int opnum, long hash)
             throws java.lang.Exception {
         if (opnum < 0) {
             if (hash == 7583982177005850366L) {
@@ -78,6 +73,7 @@ public final class RegistryImpl_Skel
         }
 
         sun.rmi.registry.RegistryImpl server = (sun.rmi.registry.RegistryImpl) obj;
+        StreamRemoteCall call = (StreamRemoteCall) remoteCall;
         switch (opnum) {
             case 0: // bind(String, Remote)
             {
@@ -90,7 +86,8 @@ public final class RegistryImpl_Skel
                     java.io.ObjectInput in = call.getInputStream();
                     $param_String_1 = (java.lang.String) in.readObject();
                     $param_Remote_2 = (java.rmi.Remote) in.readObject();
-                } catch (java.io.IOException | java.lang.ClassNotFoundException e) {
+                } catch (ClassCastException | IOException | ClassNotFoundException e) {
+                    call.discardPendingRefs();
                     throw new java.rmi.UnmarshalException("error unmarshalling arguments", e);
                 } finally {
                     call.releaseInputStream();
@@ -123,7 +120,8 @@ public final class RegistryImpl_Skel
                 try {
                     java.io.ObjectInput in = call.getInputStream();
                     $param_String_1 = (java.lang.String) in.readObject();
-                } catch (java.io.IOException | java.lang.ClassNotFoundException e) {
+                } catch (ClassCastException | IOException | ClassNotFoundException e) {
+                    call.discardPendingRefs();
                     throw new java.rmi.UnmarshalException("error unmarshalling arguments", e);
                 } finally {
                     call.releaseInputStream();
@@ -149,7 +147,8 @@ public final class RegistryImpl_Skel
                     java.io.ObjectInput in = call.getInputStream();
                     $param_String_1 = (java.lang.String) in.readObject();
                     $param_Remote_2 = (java.rmi.Remote) in.readObject();
-                } catch (java.io.IOException | java.lang.ClassNotFoundException e) {
+                } catch (ClassCastException | IOException | java.lang.ClassNotFoundException e) {
+                    call.discardPendingRefs();
                     throw new java.rmi.UnmarshalException("error unmarshalling arguments", e);
                 } finally {
                     call.releaseInputStream();
@@ -172,7 +171,8 @@ public final class RegistryImpl_Skel
                 try {
                     java.io.ObjectInput in = call.getInputStream();
                     $param_String_1 = (java.lang.String) in.readObject();
-                } catch (java.io.IOException | java.lang.ClassNotFoundException e) {
+                } catch (ClassCastException | IOException | ClassNotFoundException e) {
+                    call.discardPendingRefs();
                     throw new java.rmi.UnmarshalException("error unmarshalling arguments", e);
                 } finally {
                     call.releaseInputStream();

src/share/classes/sun/rmi/registry/RegistryImpl_Stub.java

 /*
- * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2019, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -24,6 +24,11 @@
  */
 
 package sun.rmi.registry;
+
+import java.io.IOException;
+
+import sun.rmi.transport.StreamRemoteCall;
+
 /**
  * Stubs to invoke RegistryImpl remote methods.
  * Originally generated from RMIC but frozen to match RegistryImpl_Skel.
@@ -57,7 +62,7 @@ public final class RegistryImpl_Stub
     public void bind(java.lang.String $param_String_1, java.rmi.Remote $param_Remote_2)
             throws java.rmi.AccessException, java.rmi.AlreadyBoundException, java.rmi.RemoteException {
         try {
-            java.rmi.server.RemoteCall call = ref.newCall((java.rmi.server.RemoteObject) this, operations, 0, interfaceHash);
+            StreamRemoteCall call = (StreamRemoteCall)ref.newCall(this, operations, 0, interfaceHash);
             try {
                 java.io.ObjectOutput out = call.getOutputStream();
                 out.writeObject($param_String_1);
@@ -82,15 +87,14 @@ public final class RegistryImpl_Stub
     public java.lang.String[] list()
             throws java.rmi.AccessException, java.rmi.RemoteException {
         try {
-            java.rmi.server.RemoteCall call = ref.newCall((java.rmi.server.RemoteObject) this, operations, 1, interfaceHash);
+            StreamRemoteCall call = (StreamRemoteCall)ref.newCall(this, operations, 1, interfaceHash);
             ref.invoke(call);
             java.lang.String[] $result;
             try {
                 java.io.ObjectInput in = call.getInputStream();
                 $result = (java.lang.String[]) in.readObject();
-            } catch (java.io.IOException e) {
-                throw new java.rmi.UnmarshalException("error unmarshalling return", e);
-            } catch (java.lang.ClassNotFoundException e) {
+            } catch (ClassCastException | IOException | ClassNotFoundException e) {
+                call.discardPendingRefs();
                 throw new java.rmi.UnmarshalException("error unmarshalling return", e);
             } finally {
                 ref.done(call);
@@ -109,7 +113,7 @@ public final class RegistryImpl_Stub
     public java.rmi.Remote lookup(java.lang.String $param_String_1)
             throws java.rmi.AccessException, java.rmi.NotBoundException, java.rmi.RemoteException {
         try {
-            java.rmi.server.RemoteCall call = ref.newCall((java.rmi.server.RemoteObject) this, operations, 2, interfaceHash);
+            StreamRemoteCall call = (StreamRemoteCall)ref.newCall(this, operations, 2, interfaceHash);
             try {
                 java.io.ObjectOutput out = call.getOutputStream();
                 out.writeObject($param_String_1);
@@ -121,9 +125,8 @@ public final class RegistryImpl_Stub
             try {
                 java.io.ObjectInput in = call.getInputStream();
                 $result = (java.rmi.Remote) in.readObject();
-            } catch (java.io.IOException e) {
-                throw new java.rmi.UnmarshalException("error unmarshalling return", e);
-            } catch (java.lang.ClassNotFoundException e) {
+            } catch (ClassCastException | IOException | ClassNotFoundException e) {
+                call.discardPendingRefs();
                 throw new java.rmi.UnmarshalException("error unmarshalling return", e);
             } finally {
                 ref.done(call);
@@ -144,7 +147,7 @@ public final class RegistryImpl_Stub
     public void rebind(java.lang.String $param_String_1, java.rmi.Remote $param_Remote_2)
             throws java.rmi.AccessException, java.rmi.RemoteException {
         try {
-            java.rmi.server.RemoteCall call = ref.newCall((java.rmi.server.RemoteObject) this, operations, 3, interfaceHash);
+            StreamRemoteCall call = (StreamRemoteCall)ref.newCall(this, operations, 3, interfaceHash);
             try {
                 java.io.ObjectOutput out = call.getOutputStream();
                 out.writeObject($param_String_1);
@@ -167,7 +170,7 @@ public final class RegistryImpl_Stub
     public void unbind(java.lang.String $param_String_1)
             throws java.rmi.AccessException, java.rmi.NotBoundException, java.rmi.RemoteException {
         try {
-            java.rmi.server.RemoteCall call = ref.newCall((java.rmi.server.RemoteObject) this, operations, 4, interfaceHash);
+            StreamRemoteCall call = (StreamRemoteCall)ref.newCall(this, operations, 4, interfaceHash);
             try {
                 java.io.ObjectOutput out = call.getOutputStream();
                 out.writeObject($param_String_1);

src/share/classes/sun/rmi/transport/DGCImpl_Skel.java

 /*
- * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2019, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -25,6 +25,8 @@
 
 package sun.rmi.transport;
 
+import java.io.IOException;
+
 /**
  * Skeleton to dispatch DGC methods.
  * Originally generated by RMIC but frozen to match the stubs.
@@ -43,12 +45,13 @@ public final class DGCImpl_Skel
         return operations.clone();
     }
 
-    public void dispatch(java.rmi.Remote obj, java.rmi.server.RemoteCall call, int opnum, long hash)
+    public void dispatch(java.rmi.Remote obj, java.rmi.server.RemoteCall remoteCall, int opnum, long hash)
             throws java.lang.Exception {
         if (hash != interfaceHash)
             throw new java.rmi.server.SkeletonMismatchException("interface hash mismatch");
 
         sun.rmi.transport.DGCImpl server = (sun.rmi.transport.DGCImpl) obj;
+        StreamRemoteCall call = (StreamRemoteCall) remoteCall;
         switch (opnum) {
             case 0: // clean(ObjID[], long, VMID, boolean)
             {
@@ -62,9 +65,8 @@ public final class DGCImpl_Skel
                     $param_long_2 = in.readLong();
                     $param_VMID_3 = (java.rmi.dgc.VMID) in.readObject();
                     $param_boolean_4 = in.readBoolean();
-                } catch (java.io.IOException e) {
-                    throw new java.rmi.UnmarshalException("error unmarshalling arguments", e);
-                } catch (java.lang.ClassNotFoundException e) {
+                } catch (ClassCastException | IOException | ClassNotFoundException e) {
+                    call.discardPendingRefs();
                     throw new java.rmi.UnmarshalException("error unmarshalling arguments", e);
                 } finally {
                     call.releaseInputStream();
@@ -88,9 +90,8 @@ public final class DGCImpl_Skel
                     $param_arrayOf_ObjID_1 = (java.rmi.server.ObjID[]) in.readObject();
                     $param_long_2 = in.readLong();
                     $param_Lease_3 = (java.rmi.dgc.Lease) in.readObject();
-                } catch (java.io.IOException e) {
-                    throw new java.rmi.UnmarshalException("error unmarshalling arguments", e);
-                } catch (java.lang.ClassNotFoundException e) {
+                } catch (ClassCastException | IOException | ClassNotFoundException e) {
+                    call.discardPendingRefs();
                     throw new java.rmi.UnmarshalException("error unmarshalling arguments", e);
                 } finally {
                     call.releaseInputStream();

src/share/classes/sun/rmi/transport/DGCImpl_Stub.java

 /*
- * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2019, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -25,16 +25,15 @@
 
 package sun.rmi.transport;
 
-import java.io.ObjectInputStream;
+import sun.rmi.transport.tcp.TCPConnection;
+
+import java.io.IOException;
 import java.rmi.dgc.Lease;
 import java.rmi.dgc.VMID;
 import java.rmi.server.UID;
-import java.security.AccessController;
-import java.security.PrivilegedAction;
+import java.util.ArrayList;
 
 import sun.misc.ObjectInputFilter;
-import sun.rmi.server.UnicastRef;
-import sun.rmi.transport.tcp.TCPConnection;
 
 /**
  * Stubs to invoke DGC remote methods.
@@ -72,7 +71,9 @@ public final class DGCImpl_Stub
     public void clean(java.rmi.server.ObjID[] $param_arrayOf_ObjID_1, long $param_long_2, java.rmi.dgc.VMID $param_VMID_3, boolean $param_boolean_4)
             throws java.rmi.RemoteException {
         try {
-            java.rmi.server.RemoteCall call = ref.newCall((java.rmi.server.RemoteObject) this, operations, 0, interfaceHash);
+            StreamRemoteCall call = (StreamRemoteCall)ref.newCall((java.rmi.server.RemoteObject) this,
+                    operations, 0, interfaceHash);
+            call.setObjectInputFilter(DGCImpl_Stub::leaseFilter);
             try {
                 java.io.ObjectOutput out = call.getOutputStream();
                 out.writeObject($param_arrayOf_ObjID_1);
@@ -97,7 +98,10 @@ public final class DGCImpl_Stub
     public java.rmi.dgc.Lease dirty(java.rmi.server.ObjID[] $param_arrayOf_ObjID_1, long $param_long_2, java.rmi.dgc.Lease $param_Lease_3)
             throws java.rmi.RemoteException {
         try {
-            java.rmi.server.RemoteCall call = ref.newCall((java.rmi.server.RemoteObject) this, operations, 1, interfaceHash);
+            StreamRemoteCall call =
+                    (StreamRemoteCall)ref.newCall((java.rmi.server.RemoteObject) this,
+                            operations, 1, interfaceHash);
+            call.setObjectInputFilter(DGCImpl_Stub::leaseFilter);
             try {
                 java.io.ObjectOutput out = call.getOutputStream();
                 out.writeObject($param_arrayOf_ObjID_1);
@@ -108,26 +112,17 @@ public final class DGCImpl_Stub
             }
             ref.invoke(call);
             java.rmi.dgc.Lease $result;
-            Connection connection = ((StreamRemoteCall) call).getConnection();
+            Connection connection = call.getConnection();
             try {
                 java.io.ObjectInput in = call.getInputStream();
 
-                if (in instanceof ObjectInputStream) {
-                    /**
-                     * Set a filter on the stream for the return value.
-                     */
-                    ObjectInputStream ois = (ObjectInputStream) in;
-                    AccessController.doPrivileged((PrivilegedAction<Void>)() -> {
-                        ObjectInputFilter.Config.setObjectInputFilter(ois, DGCImpl_Stub::leaseFilter);
-                        return null;
-                    });
-                }
                 $result = (java.rmi.dgc.Lease) in.readObject();
-            } catch (java.io.IOException | java.lang.ClassNotFoundException e) {
+            } catch (ClassCastException | IOException | ClassNotFoundException e) {
                 if (connection instanceof TCPConnection) {
                     // Modified to prevent re-use of the connection after an exception
                     ((TCPConnection) connection).getChannel().free(connection, false);
                 }
+                call.discardPendingRefs();
                 throw new java.rmi.UnmarshalException("error unmarshalling return", e);
             } finally {
                 ref.done(call);
@@ -146,6 +141,10 @@ public final class DGCImpl_Stub
      * ObjectInputFilter to filter DGCClient return value (a Lease).
      * The list of acceptable classes is very short and explicit.
      * The depth and array sizes are limited.
+     * <p>
+     * The filter must accept normal and exception returns.
+     * A DGC server may throw exceptions that may have a cause
+     * and suppressed exceptions.
      *
      * @param filterInfo access to class, arrayLength, etc.
      * @return  {@link ObjectInputFilter.Status#ALLOWED} if allowed,
@@ -172,7 +171,16 @@ public final class DGCImpl_Stub
             }
             return (clazz == UID.class ||
                     clazz == VMID.class ||
-                    clazz == Lease.class)
+                    clazz == Lease.class ||
+                    (Throwable.class.isAssignableFrom(clazz) &&
+                            clazz.getClassLoader() ==
+                            Object.class.getClassLoader()) ||
+                    clazz == StackTraceElement.class ||
+                    clazz == ArrayList.class ||     // for suppressed exceptions, if any
+                    clazz == Object.class ||
+                    clazz.getName().equals("java.util.Collections$UnmodifiableList") ||
+                    clazz.getName().equals("java.util.Collections$UnmodifiableCollection") ||
+                    clazz.getName().equals("java.util.Collections$UnmodifiableRandomAccessList"))
                     ? ObjectInputFilter.Status.ALLOWED
                     : ObjectInputFilter.Status.REJECTED;
         }

src/share/classes/sun/rmi/transport/StreamRemoteCall.java

 /*
- * Copyright (c) 1996, 2017, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1996, 2019, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -36,6 +36,10 @@ import java.rmi.MarshalException;
 import java.rmi.UnmarshalException;
 import java.rmi.server.ObjID;
 import java.rmi.server.RemoteCall;
+import java.security.AccessController;
+import java.security.PrivilegedAction;
+
+import sun.misc.ObjectInputFilter;
 import sun.rmi.runtime.Log;
 import sun.rmi.server.UnicastRef;
 import sun.rmi.transport.tcp.TCPEndpoint;
@@ -50,6 +54,7 @@ public class StreamRemoteCall implements RemoteCall {
     private ConnectionInputStream in = null;
     private ConnectionOutputStream out = null;
     private Connection conn;
+    private ObjectInputFilter filter = null;
     private boolean resultStarted = false;
     private Exception serverException = null;
 
@@ -123,6 +128,13 @@ public class StreamRemoteCall implements RemoteCall {
         }
     }
 
+    public void setObjectInputFilter(ObjectInputFilter filter) {
+        if (in != null) {
+            throw new IllegalStateException("set filter must occur before calling getInputStream");
+        }
+        this.filter = filter;
+    }
+
     /**
      * Get the InputStream the stub/skeleton should get results/arguments
      * from.
@@ -132,6 +144,12 @@ public class StreamRemoteCall implements RemoteCall {
             Transport.transportLog.log(Log.VERBOSE, "getting input stream");
 
             in = new ConnectionInputStream(conn.getInputStream());
+            if (filter != null) {
+                AccessController.doPrivileged((PrivilegedAction<Void>) () -> {
+                    ObjectInputFilter.Config.setObjectInputFilter(in, filter);
+                    return null;
+                });
+            }
         }
         return in;
     }
@@ -251,6 +269,7 @@ public class StreamRemoteCall implements RemoteCall {
             try {
                 ex = in.readObject();
             } catch (Exception e) {
+                discardPendingRefs();
                 throw new UnmarshalException("Error unmarshaling return", e);
             }
 
@@ -259,6 +278,7 @@ public class StreamRemoteCall implements RemoteCall {
             if (ex instanceof Exception) {
                 exceptionReceivedFromServer((Exception) ex);
             } else {
+                discardPendingRefs();
                 throw new UnmarshalException("Return type not Exception");
             }
             // Exception is thrown before fallthrough can occur

test/java/rmi/testlibrary/TestSocketFactory.java

 /*
- * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2019, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -92,7 +92,8 @@ public class TestSocketFactory extends RMISocketFactory
 
     static final byte[] EMPTY_BYTE_ARRAY = new byte[0];
 
-    public static final boolean DEBUG = false;
+    // True to enable logging of matches and replacements.
+    private static volatile boolean debugLogging = false;
 
     /**
      * Debugging output can be synchronized with logging of RMI actions.
@@ -100,8 +101,8 @@ public class TestSocketFactory extends RMISocketFactory
      * @param format a printf format
      * @param args   any args
      */
-    private static void DEBUG(String format, Object... args) {
-        if (DEBUG) {
+    public static void DEBUG(String format, Object... args) {
+        if (debugLogging) {
             System.err.printf(format, args);
         }
     }
@@ -116,6 +117,17 @@ public class TestSocketFactory extends RMISocketFactory
         this.replaceBytes = EMPTY_BYTE_ARRAY;
     }
 
+    /**
+     * Set debug to true to generate logging output of matches and substitutions.
+     * @param debug {@code true} to generate logging output
+     * @return the previous value
+     */
+    public static boolean setDebug(boolean debug) {
+        boolean oldDebug = debugLogging;
+        debugLogging = debug;
+        return oldDebug;
+    }
+
     /**
      * Set the match and replacement bytes, with an empty trigger.
      * The match and replacements are propagated to all existing sockets.