8156804: Better constraint checking

Summary: Apply serialization filtering to RMI Registry and DGC
Reviewed-by: dfuchs, ahgross

src/share/classes/java/rmi/MarshalledObject.java

 /*
- * Copyright (c) 1997, 2013, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2016, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -34,9 +34,14 @@ import java.io.ObjectOutputStream;
 import java.io.ObjectStreamConstants;
 import java.io.OutputStream;
 import java.io.Serializable;
+import java.security.AccessController;
+import java.security.PrivilegedAction;
+
 import sun.rmi.server.MarshalInputStream;
 import sun.rmi.server.MarshalOutputStream;
 
+import sun.misc.ObjectInputFilter;
+
 /**
  * A <code>MarshalledObject</code> contains a byte stream with the serialized
  * representation of an object given to its constructor.  The <code>get</code>
@@ -90,6 +95,9 @@ public final class MarshalledObject<T> implements Serializable {
      */
     private int hash;
 
+    /** Filter used when creating the instance from a stream; may be null. */
+    private transient ObjectInputFilter objectInputFilter = null;
+
     /** Indicate compatibility with 1.2 version of class. */
     private static final long serialVersionUID = 8988374069173025854L;
 
@@ -132,6 +140,20 @@ public final class MarshalledObject<T> implements Serializable {
         hash = h;
     }
 
+    /**
+     * Reads in the state of the object and saves the stream's
+     * serialization filter to be used when the object is deserialized.
+     *
+     * @param stream the stream
+     * @throws IOException if an I/O error occurs
+     * @throws ClassNotFoundException if a class cannot be found
+     */
+    private void readObject(ObjectInputStream stream)
+        throws IOException, ClassNotFoundException {
+        stream.defaultReadObject();     // read in all fields
+        objectInputFilter = ObjectInputFilter.Config.getObjectInputFilter(stream);
+    }
+
     /**
      * Returns a new copy of the contained marshalledobject.  The internal
      * representation is deserialized with the semantics used for
@@ -155,7 +177,7 @@ public final class MarshalledObject<T> implements Serializable {
         ByteArrayInputStream lin =
             (locBytes == null ? null : new ByteArrayInputStream(locBytes));
         MarshalledObjectInputStream in =
-            new MarshalledObjectInputStream(bin, lin);
+            new MarshalledObjectInputStream(bin, lin, objectInputFilter);
         @SuppressWarnings("unchecked")
         T obj = (T) in.readObject();
         in.close();
@@ -295,11 +317,24 @@ public final class MarshalledObject<T> implements Serializable {
          * <code>null</code>, then all annotations will be
          * <code>null</code>.
          */
-        MarshalledObjectInputStream(InputStream objIn, InputStream locIn)
+        MarshalledObjectInputStream(InputStream objIn, InputStream locIn,
+                    ObjectInputFilter filter)
             throws IOException
         {
             super(objIn);
             this.locIn = (locIn == null ? null : new ObjectInputStream(locIn));
+            if (filter != null) {
+                AccessController.doPrivileged(new PrivilegedAction<Void>() {
+                    @Override
+                    public Void run() {
+                        ObjectInputFilter.Config.setObjectInputFilter(MarshalledObjectInputStream.this, filter);
+                        if (MarshalledObjectInputStream.this.locIn != null) {
+                            ObjectInputFilter.Config.setObjectInputFilter(MarshalledObjectInputStream.this.locIn, filter);
+                        }
+                        return null;
+                    }
+                });
+            }
         }
 
         /**

src/share/classes/sun/rmi/registry/RegistryImpl.java

 /*
- * Copyright (c) 1996, 2012, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1996, 2016, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -30,11 +30,9 @@ import java.util.Hashtable;
 import java.util.MissingResourceException;
 import java.util.ResourceBundle;
 import java.io.FilePermission;
-import java.io.IOException;
 import java.net.*;
 import java.rmi.*;
 import java.rmi.server.ObjID;
-import java.rmi.server.RemoteServer;
 import java.rmi.server.ServerNotActiveException;
 import java.rmi.registry.Registry;
 import java.rmi.server.RMIClientSocketFactory;
@@ -47,14 +45,18 @@ import java.security.PrivilegedActionException;
 import java.security.PrivilegedExceptionAction;
 import java.security.PermissionCollection;
 import java.security.Permissions;
+import java.security.PrivilegedAction;
 import java.security.ProtectionDomain;
+import java.security.Security;
 import java.text.MessageFormat;
-import sun.rmi.server.LoaderHandler;
+
+import sun.misc.ObjectInputFilter;
+
+import sun.rmi.runtime.Log;
+import sun.rmi.server.UnicastRef;
 import sun.rmi.server.UnicastServerRef;
 import sun.rmi.server.UnicastServerRef2;
 import sun.rmi.transport.LiveRef;
-import sun.rmi.transport.ObjectTable;
-import sun.rmi.transport.Target;
 
 /**
  * A "registry" exists on every node that allows RMI connections to
@@ -85,6 +87,47 @@ public class RegistryImpl extends java.rmi.server.RemoteServer
 
     private static ResourceBundle resources = null;
 
+    /**
+     * Property name of the RMI Registry serial filter to augment
+     * the built-in list of allowed types.
+     * Setting the property in the {@code lib/security/java.security} file
+     * will enable the augmented filter.
+     */
+    private static final String REGISTRY_FILTER_PROPNAME = "sun.rmi.registry.registryFilter";
+
+    /** Registry max depth of remote invocations. **/
+    private static int REGISTRY_MAX_DEPTH = 5;
+
+    /** Registry maximum array size in remote invocations. **/
+    private static int REGISTRY_MAX_ARRAY_SIZE = 10000;
+
+    /**
+     * The registryFilter created from the value of the {@code "sun.rmi.registry.registryFilter"}
+     * property.
+     */
+    private static final ObjectInputFilter registryFilter =
+            AccessController.doPrivileged((PrivilegedAction<ObjectInputFilter>)RegistryImpl::initRegistryFilter);
+
+    /**
+     * Initialize the registryFilter from the security properties or system property; if any
+     * @return an ObjectInputFilter, or null
+     */
+    private static ObjectInputFilter initRegistryFilter() {
+        ObjectInputFilter filter = null;
+        String props = System.getProperty(REGISTRY_FILTER_PROPNAME);
+        if (props == null) {
+            props = Security.getProperty(REGISTRY_FILTER_PROPNAME);
+        }
+        if (props != null) {
+            filter = ObjectInputFilter.Config.createFilter(props);
+            Log regLog = Log.getLog("sun.rmi.registry", "registry", -1);
+            if (regLog.isLoggable(Log.BRIEF)) {
+                regLog.log(Log.BRIEF, "registryFilter = " + filter);
+            }
+        }
+        return filter;
+    }
+
     /**
      * Construct a new RegistryImpl on the specified port with the
      * given custom socket factory pair.
@@ -100,7 +143,7 @@ public class RegistryImpl extends java.rmi.server.RemoteServer
                 AccessController.doPrivileged(new PrivilegedExceptionAction<Void>() {
                     public Void run() throws RemoteException {
                         LiveRef lref = new LiveRef(id, port, csf, ssf);
-                        setup(new UnicastServerRef2(lref));
+                        setup(new UnicastServerRef2(lref, RegistryImpl::registryFilter));
                         return null;
                     }
                 }, null, new SocketPermission("localhost:"+port, "listen,accept"));
@@ -109,7 +152,7 @@ public class RegistryImpl extends java.rmi.server.RemoteServer
             }
         } else {
             LiveRef lref = new LiveRef(id, port, csf, ssf);
-            setup(new UnicastServerRef2(lref));
+            setup(new UnicastServerRef2(lref, RegistryImpl::registryFilter));
         }
     }
 
@@ -125,7 +168,7 @@ public class RegistryImpl extends java.rmi.server.RemoteServer
                 AccessController.doPrivileged(new PrivilegedExceptionAction<Void>() {
                     public Void run() throws RemoteException {
                         LiveRef lref = new LiveRef(id, port);
-                        setup(new UnicastServerRef(lref));
+                        setup(new UnicastServerRef(lref, RegistryImpl::registryFilter));
                         return null;
                     }
                 }, null, new SocketPermission("localhost:"+port, "listen,accept"));
@@ -134,7 +177,7 @@ public class RegistryImpl extends java.rmi.server.RemoteServer
             }
         } else {
             LiveRef lref = new LiveRef(id, port);
-            setup(new UnicastServerRef(lref));
+            setup(new UnicastServerRef(lref, RegistryImpl::registryFilter));
         }
     }
 
@@ -155,7 +198,7 @@ public class RegistryImpl extends java.rmi.server.RemoteServer
     /**
      * Returns the remote object for specified name in the registry.
      * @exception RemoteException If remote operation failed.
-     * @exception NotBound If name is not currently bound.
+     * @exception NotBoundException If name is not currently bound.
      */
     public Remote lookup(String name)
         throws RemoteException, NotBoundException
@@ -188,7 +231,7 @@ public class RegistryImpl extends java.rmi.server.RemoteServer
     /**
      * Unbind the name.
      * @exception RemoteException If remote operation failed.
-     * @exception NotBound If name is not currently bound.
+     * @exception NotBoundException If name is not currently bound.
      */
     public void unbind(String name)
         throws RemoteException, NotBoundException, AccessException
@@ -332,6 +375,60 @@ public class RegistryImpl extends java.rmi.server.RemoteServer
         }
     }
 
+    /**
+     * ObjectInputFilter to filter Registry input objects.
+     * The list of acceptable classes is limited to classes normally
+     * stored in a registry.
+     *
+     * @param filterInfo access to the class, array length, etc.
+     * @return  {@link ObjectInputFilter.Status#ALLOWED} if allowed,
+     *          {@link ObjectInputFilter.Status#REJECTED} if rejected,
+     *          otherwise {@link ObjectInputFilter.Status#UNDECIDED}
+     */
+    private static ObjectInputFilter.Status registryFilter(ObjectInputFilter.FilterInfo filterInfo) {
+        if (registryFilter != null) {
+            ObjectInputFilter.Status status = registryFilter.checkInput(filterInfo);
+            if (status != ObjectInputFilter.Status.UNDECIDED) {
+                // The Registry filter can override the built-in white-list
+                return status;
+            }
+        }
+
+        if (filterInfo.depth() > REGISTRY_MAX_DEPTH) {
+            return ObjectInputFilter.Status.REJECTED;
+        }
+        Class<?> clazz = filterInfo.serialClass();
+        if (clazz != null) {
+            if (clazz.isArray()) {
+                if (filterInfo.arrayLength() >= 0 && filterInfo.arrayLength() > REGISTRY_MAX_ARRAY_SIZE) {
+                    return ObjectInputFilter.Status.REJECTED;
+                }
+                do {
+                    // Arrays are allowed depending on the component type
+                    clazz = clazz.getComponentType();
+                } while (clazz.isArray());
+            }
+            if (clazz.isPrimitive()) {
+                // Arrays of primitives are allowed
+                return ObjectInputFilter.Status.ALLOWED;
+            }
+            if (String.class == clazz
+                    || java.lang.Number.class.isAssignableFrom(clazz)
+                    || Remote.class.isAssignableFrom(clazz)
+                    || java.lang.reflect.Proxy.class.isAssignableFrom(clazz)
+                    || UnicastRef.class.isAssignableFrom(clazz)
+                    || RMIClientSocketFactory.class.isAssignableFrom(clazz)
+                    || RMIServerSocketFactory.class.isAssignableFrom(clazz)
+                    || java.rmi.activation.ActivationID.class.isAssignableFrom(clazz)
+                    || java.rmi.server.UID.class.isAssignableFrom(clazz)) {
+                return ObjectInputFilter.Status.ALLOWED;
+            } else {
+                return ObjectInputFilter.Status.REJECTED;
+            }
+        }
+        return ObjectInputFilter.Status.UNDECIDED;
+    }
+
     /**
      * Main program to start a registry. <br>
      * The port number can be specified on the command line.

src/share/classes/sun/rmi/server/UnicastServerRef.java

@@ -27,6 +27,7 @@ package sun.rmi.server;
 
 import java.io.IOException;
 import java.io.ObjectInput;
+import java.io.ObjectInputStream;
 import java.io.ObjectOutput;
 import java.io.ObjectStreamClass;
 import java.lang.reflect.InvocationTargetException;
@@ -53,8 +54,8 @@ import java.util.HashMap;
 import java.util.Map;
 import java.util.WeakHashMap;
 import java.util.concurrent.atomic.AtomicInteger;
+import sun.misc.ObjectInputFilter;
 import sun.rmi.runtime.Log;
-import static sun.rmi.server.UnicastRef.marshalValue;
 import sun.rmi.transport.LiveRef;
 import sun.rmi.transport.Target;
 import sun.rmi.transport.tcp.TCPTransport;
@@ -64,6 +65,10 @@ import sun.security.action.GetBooleanAction;
  * UnicastServerRef implements the remote reference layer server-side
  * behavior for remote objects exported with the "UnicastRef" reference
  * type.
+ * If an {@link ObjectInputFilter ObjectInputFilter} is supplied it is
+ * invoked during deserialization to filter the arguments,
+ * otherwise the default filter of {@link ObjectInputStream ObjectInputStream}
+ * applies.
  *
  * @author  Ann Wollrath
  * @author  Roger Riggs
@@ -106,6 +111,9 @@ public class UnicastServerRef extends UnicastRef
      */
     private transient Skeleton skel;
 
+    // The ObjectInputFilter for checking the invocation arguments
+    private final transient ObjectInputFilter filter;
+
     /** maps method hash to Method object for each remote method */
     private transient Map<Long,Method> hashToMethod_Map = null;
 
@@ -124,16 +132,29 @@ public class UnicastServerRef extends UnicastRef
 
     /**
      * Create a new (empty) Unicast server remote reference.
+     * The filter is null to defer to the  default ObjectInputStream filter, if any.
      */
     public UnicastServerRef() {
+        this.filter = null;
     }
 
     /**
      * Construct a Unicast server remote reference for a specified
      * liveRef.
+     * The filter is null to defer to the  default ObjectInputStream filter, if any.
      */
     public UnicastServerRef(LiveRef ref) {
         super(ref);
+        this.filter = null;
+    }
+
+    /**
+     * Construct a Unicast server remote reference for a specified
+     * liveRef and filter.
+     */
+    public UnicastServerRef(LiveRef ref, ObjectInputFilter filter) {
+        super(ref);
+        this.filter = filter;
     }
 
     /**
@@ -142,6 +163,7 @@ public class UnicastServerRef extends UnicastRef
      */
     public UnicastServerRef(int port) {
         super(new LiveRef(port));
+        this.filter = null;
     }
 
     /**
@@ -366,9 +388,26 @@ public class UnicastServerRef extends UnicastRef
         }
     }
 
+    /**
+     * Sets a filter for invocation arguments, if a filter has been set.
+     * Called by dispatch before the arguments are read.
+     */
     protected void unmarshalCustomCallData(ObjectInput in)
-        throws IOException, ClassNotFoundException
-    {}
+            throws IOException, ClassNotFoundException {
+        if (filter != null &&
+                in instanceof ObjectInputStream) {
+            // Set the filter on the stream
+            ObjectInputStream ois = (ObjectInputStream) in;
+
+            AccessController.doPrivileged(new PrivilegedAction<Void>() {
+                @Override
+                public Void run() {
+                    ObjectInputFilter.Config.setObjectInputFilter(ois, filter);
+                    return null;
+                }
+            });
+        }
+    }
 
     /**
      * Handle server-side dispatch using the RMI 1.1 stub/skeleton

src/share/classes/sun/rmi/server/UnicastServerRef2.java

@@ -25,12 +25,15 @@
 
 package sun.rmi.server;
 
-import java.io.IOException;
+
 import java.io.ObjectOutput;
-import java.rmi.*;
-import java.rmi.server.*;
-import sun.rmi.transport.*;
-import sun.rmi.transport.tcp.*;
+import java.rmi.server.RMIClientSocketFactory;
+import java.rmi.server.RMIServerSocketFactory;
+import java.rmi.server.RemoteRef;
+
+import sun.misc.ObjectInputFilter;
+
+import sun.rmi.transport.LiveRef;
 
 /**
  * Server-side ref for a remote impl that uses a custom socket factory.
@@ -58,6 +61,16 @@ public class UnicastServerRef2 extends UnicastServerRef
         super(ref);
     }
 
+    /**
+     * Construct a Unicast server remote reference for a specified
+     * liveRef and filter.
+     */
+    public UnicastServerRef2(LiveRef ref,
+                             ObjectInputFilter filter)
+    {
+        super(ref, filter);
+    }
+
     /**
      * Construct a Unicast server remote reference to be exported
      * on the specified port.
@@ -69,6 +82,18 @@ public class UnicastServerRef2 extends UnicastServerRef
         super(new LiveRef(port, csf, ssf));
     }
 
+    /**
+     * Construct a Unicast server remote reference to be exported
+     * on the specified port.
+     */
+    public UnicastServerRef2(int port,
+                             RMIClientSocketFactory csf,
+                             RMIServerSocketFactory ssf,
+                             ObjectInputFilter filter)
+    {
+        super(new LiveRef(port, csf, ssf), filter);
+    }
+
     /**
      * Returns the class of the ref type to be serialized
      */

src/share/classes/sun/rmi/transport/DGCImpl.java

 /*
- * Copyright (c) 1996, 2015, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1996, 2016, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -34,11 +34,13 @@ import java.rmi.server.LogStream;
 import java.rmi.server.ObjID;
 import java.rmi.server.RemoteServer;
 import java.rmi.server.ServerNotActiveException;
+import java.rmi.server.UID;
 import java.security.AccessControlContext;
 import java.security.AccessController;
 import java.security.Permissions;
 import java.security.PrivilegedAction;
 import java.security.ProtectionDomain;
+import java.security.Security;
 import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.HashMap;
@@ -49,6 +51,9 @@ import java.util.Set;
 import java.util.concurrent.Future;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
+
+import sun.misc.ObjectInputFilter;
+
 import sun.rmi.runtime.Log;
 import sun.rmi.runtime.RuntimeUtil;
 import sun.rmi.server.UnicastRef;
@@ -101,6 +106,45 @@ final class DGCImpl implements DGC {
         return dgc;
     }
 
+    /**
+     * Property name of the DGC serial filter to augment
+     * the built-in list of allowed types.
+     * Setting the property in the {@code lib/security/java.security} file
+     * will enable the augmented filter.
+     */
+    private static final String DGC_FILTER_PROPNAME = "sun.rmi.transport.dgcFilter";
+
+    /** Registry max depth of remote invocations. **/
+    private static int DGC_MAX_DEPTH = 5;
+
+    /** Registry maximum array size in remote invocations. **/
+    private static int DGC_MAX_ARRAY_SIZE = 10000;
+    /**
+     * The dgcFilter created from the value of the {@code  "sun.rmi.transport.dgcFilter"}
+     * property.
+     */
+    private static final ObjectInputFilter dgcFilter =
+            AccessController.doPrivileged((PrivilegedAction<ObjectInputFilter>)DGCImpl::initDgcFilter);
+
+    /**
+     * Initialize the dgcFilter from the security properties or system property; if any
+     * @return an ObjectInputFilter, or null
+     */
+    private static ObjectInputFilter initDgcFilter() {
+        ObjectInputFilter filter = null;
+        String props = System.getProperty(DGC_FILTER_PROPNAME);
+        if (props == null) {
+            props = Security.getProperty(DGC_FILTER_PROPNAME);
+        }
+        if (props != null) {
+            filter = ObjectInputFilter.Config.createFilter(props);
+            if (dgcLog.isLoggable(Log.BRIEF)) {
+                dgcLog.log(Log.BRIEF, "dgcFilter = " + filter);
+            }
+        }
+        return filter;
+    }
+
     /**
      * Construct a new server-side remote object collector at
      * a particular port. Disallow construction from outside.
@@ -295,7 +339,8 @@ final class DGCImpl implements DGC {
                         dgc = new DGCImpl();
                         ObjID dgcID = new ObjID(ObjID.DGC_ID);
                         LiveRef ref = new LiveRef(dgcID, 0);
-                        UnicastServerRef disp = new UnicastServerRef(ref);
+                        UnicastServerRef disp = new UnicastServerRef(ref,
+                                DGCImpl::checkInput);
                         Remote stub =
                             Util.createProxy(DGCImpl.class,
                                              new UnicastRef(ref), true);
@@ -326,6 +371,53 @@ final class DGCImpl implements DGC {
         });
     }
 
+    /**
+     * ObjectInputFilter to filter DGC input objects.
+     * The list of acceptable classes is very short and explicit.
+     * The depth and array sizes are limited.
+     *
+     * @param filterInfo access to class, arrayLength, etc.
+     * @return  {@link ObjectInputFilter.Status#ALLOWED} if allowed,
+     *          {@link ObjectInputFilter.Status#REJECTED} if rejected,
+     *          otherwise {@link ObjectInputFilter.Status#UNDECIDED}
+     */
+    private static ObjectInputFilter.Status checkInput(ObjectInputFilter.FilterInfo filterInfo) {
+        if (dgcFilter != null) {
+            ObjectInputFilter.Status status = dgcFilter.checkInput(filterInfo);
+            if (status != ObjectInputFilter.Status.UNDECIDED) {
+                // The DGC filter can override the built-in white-list
+                return status;
+            }
+        }
+
+        if (filterInfo.depth() > DGC_MAX_DEPTH) {
+            return ObjectInputFilter.Status.REJECTED;
+        }
+        Class<?> clazz = filterInfo.serialClass();
+        if (clazz != null) {
+            while (clazz.isArray()) {
+                if (filterInfo.arrayLength() >= 0 && filterInfo.arrayLength() > DGC_MAX_ARRAY_SIZE) {
+                    return ObjectInputFilter.Status.REJECTED;
+                }
+                // Arrays are allowed depending on the component type
+                clazz = clazz.getComponentType();
+            }
+            if (clazz.isPrimitive()) {
+                // Arrays of primitives are allowed
+                return ObjectInputFilter.Status.ALLOWED;
+            }
+            return (clazz == ObjID.class ||
+                    clazz == UID.class ||
+                    clazz == VMID.class ||
+                    clazz == Lease.class)
+                    ? ObjectInputFilter.Status.ALLOWED
+                    : ObjectInputFilter.Status.REJECTED;
+        }
+        // Not a class, not size limited
+        return ObjectInputFilter.Status.UNDECIDED;
+    }
+
+
     private static class LeaseInfo {
         VMID vmid;
         long expiration;

src/share/lib/security/java.security-aix

@@ -737,3 +737,28 @@ jdk.jar.disabledAlgorithms=MD2, RSA keySize < 1024
 # Otherwise, the status is UNDECIDED.
 #
 #jdk.serialFilter=pattern;pattern
+
+#
+# RMI Registry Serial Filter
+#
+# The filter pattern uses the same format as jdk.serialFilter.
+# This filter can override the builtin filter if additional types need to be
+# allowed or rejected from the RMI Registry.
+#
+#sun.rmi.registry.registryFilter=pattern;pattern
+
+#
+# RMI Distributed Garbage Collector (DGC) Serial Filter
+#
+# The filter pattern uses the same format as jdk.serialFilter.
+# This filter can override the builtin filter if additional types need to be
+# allowed or rejected from the RMI DGC.
+#
+# The builtin DGC filter can approximately be represented as the filter pattern:
+#
+#sun.rmi.transport.dgcFilter=\
+#    java.rmi.server.ObjID;\
+#    java.rmi.server.UID;\
+#    java.rmi.dgc.VMID;\
+#    java.rmi.dgc.Lease;\
+#    maxdepth=5;maxarray=10000

src/share/lib/security/java.security-linux

@@ -737,3 +737,28 @@ jdk.jar.disabledAlgorithms=MD2, RSA keySize < 1024
 # Otherwise, the status is UNDECIDED.
 #
 #jdk.serialFilter=pattern;pattern
+
+#
+# RMI Registry Serial Filter
+#
+# The filter pattern uses the same format as jdk.serialFilter.
+# This filter can override the builtin filter if additional types need to be
+# allowed or rejected from the RMI Registry.
+#
+#sun.rmi.registry.registryFilter=pattern;pattern
+
+#
+# RMI Distributed Garbage Collector (DGC) Serial Filter
+#
+# The filter pattern uses the same format as jdk.serialFilter.
+# This filter can override the builtin filter if additional types need to be
+# allowed or rejected from the RMI DGC.
+#
+# The builtin DGC filter can approximately be represented as the filter pattern:
+#
+#sun.rmi.transport.dgcFilter=\
+#    java.rmi.server.ObjID;\
+#    java.rmi.server.UID;\
+#    java.rmi.dgc.VMID;\
+#    java.rmi.dgc.Lease;\
+#    maxdepth=5;maxarray=10000

src/share/lib/security/java.security-macosx

@@ -740,3 +740,28 @@ jdk.jar.disabledAlgorithms=MD2, RSA keySize < 1024
 # Otherwise, the status is UNDECIDED.
 #
 #jdk.serialFilter=pattern;pattern
+
+#
+# RMI Registry Serial Filter
+#
+# The filter pattern uses the same format as jdk.serialFilter.
+# This filter can override the builtin filter if additional types need to be
+# allowed or rejected from the RMI Registry.
+#
+#sun.rmi.registry.registryFilter=pattern;pattern
+
+#
+# RMI Distributed Garbage Collector (DGC) Serial Filter
+#
+# The filter pattern uses the same format as jdk.serialFilter.
+# This filter can override the builtin filter if additional types need to be
+# allowed or rejected from the RMI DGC.
+#
+# The builtin DGC filter can approximately be represented as the filter pattern:
+#
+#sun.rmi.transport.dgcFilter=\
+#    java.rmi.server.ObjID;\
+#    java.rmi.server.UID;\
+#    java.rmi.dgc.VMID;\
+#    java.rmi.dgc.Lease;\
+#    maxdepth=5;maxarray=10000

src/share/lib/security/java.security-solaris

@@ -739,3 +739,28 @@ jdk.jar.disabledAlgorithms=MD2, RSA keySize < 1024
 # Otherwise, the status is UNDECIDED.
 #
 #jdk.serialFilter=pattern;pattern
+
+#
+# RMI Registry Serial Filter
+#
+# The filter pattern uses the same format as jdk.serialFilter.
+# This filter can override the builtin filter if additional types need to be
+# allowed or rejected from the RMI Registry.
+#
+#sun.rmi.registry.registryFilter=pattern;pattern
+
+#
+# RMI Distributed Garbage Collector (DGC) Serial Filter
+#
+# The filter pattern uses the same format as jdk.serialFilter.
+# This filter can override the builtin filter if additional types need to be
+# allowed or rejected from the RMI DGC.
+#
+# The builtin DGC filter can approximately be represented as the filter pattern:
+#
+#sun.rmi.transport.dgcFilter=\
+#    java.rmi.server.ObjID;\
+#    java.rmi.server.UID;\
+#    java.rmi.dgc.VMID;\
+#    java.rmi.dgc.Lease;\
+#    maxdepth=5;maxarray=10000

src/share/lib/security/java.security-windows

@@ -740,3 +740,28 @@ jdk.jar.disabledAlgorithms=MD2, RSA keySize < 1024
 # Otherwise, the status is UNDECIDED.
 #
 #jdk.serialFilter=pattern;pattern
+
+#
+# RMI Registry Serial Filter
+#
+# The filter pattern uses the same format as jdk.serialFilter.
+# This filter can override the builtin filter if additional types need to be
+# allowed or rejected from the RMI Registry.
+#
+#sun.rmi.registry.registryFilter=pattern;pattern
+
+#
+# RMI Distributed Garbage Collector (DGC) Serial Filter
+#
+# The filter pattern uses the same format as jdk.serialFilter.
+# This filter can override the builtin filter if additional types need to be
+# allowed or rejected from the RMI DGC.
+#
+# The builtin DGC filter can approximately be represented as the filter pattern:
+#
+#sun.rmi.transport.dgcFilter=\
+#    java.rmi.server.ObjID;\
+#    java.rmi.server.UID;\
+#    java.rmi.dgc.VMID;\
+#    java.rmi.dgc.Lease;\
+#    maxdepth=5;maxarray=10000

test/java/rmi/MarshalledObject/MOFilterTest.java

+/*
+ * Copyright (c) 2016, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InvalidClassException;
+import java.io.ObjectInput;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+import java.io.Serializable;
+import java.rmi.MarshalledObject;
+import java.util.Objects;
+
+import sun.misc.ObjectInputFilter;
+
+import org.testng.Assert;
+import org.testng.annotations.Test;
+import org.testng.annotations.DataProvider;
+
+/* @test
+ * @run testng/othervm  MOFilterTest
+ *
+ * @summary Test MarshalledObject applies ObjectInputFilter
+ */
+@Test
+public class MOFilterTest {
+
+    /**
+     * Two cases are tested.
+     * The filter = null and a filter set to verify the calls to the filter.
+     * @return array objects with test parameters for each test case
+     */
+    @DataProvider(name = "FilterCases")
+    public static Object[][] filterCases() {
+        return new Object[][] {
+                {true},     // run the test with the filter
+                {false},    // run the test without the filter
+
+        };
+    }
+
+    /**
+     * Test that MarshalledObject inherits the ObjectInputFilter from
+     * the stream it was deserialized from.
+     */
+    @Test(dataProvider="FilterCases")
+    static void delegatesToMO(boolean withFilter) {
+        try {
+            Serializable testobj = Integer.valueOf(5);
+            MarshalledObject<Serializable> mo = new MarshalledObject<>(testobj);
+            Assert.assertEquals(mo.get(), testobj, "MarshalledObject.get returned a non-equals test object");
+
+            byte[] bytes = writeObjects(mo);
+
+            try (ByteArrayInputStream bais = new ByteArrayInputStream(bytes);
+                 ObjectInputStream ois = new ObjectInputStream(bais)) {
+
+                CountingFilter filter1 = new CountingFilter();
+                ObjectInputFilter.Config.setObjectInputFilter(ois, withFilter ? filter1 : null);
+                MarshalledObject<?> actualMO = (MarshalledObject<?>)ois.readObject();
+                int count = filter1.getCount();
+
+                actualMO.get();
+                int expectedCount = withFilter ? count + 2 : count;
+                int actualCount = filter1.getCount();
+                Assert.assertEquals(actualCount, expectedCount, "filter called wrong number of times during get()");
+            }
+        } catch (IOException ioe) {
+            Assert.fail("Unexpected IOException", ioe);
+        } catch (ClassNotFoundException cnf) {
+            Assert.fail("Deserializing", cnf);
+        }
+    }
+
+    /**
+     * Write objects and return a byte array with the bytes.
+     *
+     * @param objects zero or more objects to serialize
+     * @return the byte array of the serialized objects
+     * @throws IOException if an exception occurs
+     */
+    static byte[] writeObjects(Object... objects)  throws IOException {
+        byte[] bytes;
+        try (ByteArrayOutputStream baos = new ByteArrayOutputStream();
+             ObjectOutputStream oos = new ObjectOutputStream(baos)) {
+            for (Object o : objects) {
+                oos.writeObject(o);
+            }
+            bytes = baos.toByteArray();
+        }
+        return bytes;
+    }
+
+
+    static class CountingFilter implements ObjectInputFilter {
+
+        private int count;      // count of calls to the filter
+
+        CountingFilter() {
+            count = 0;
+        }
+
+        int getCount() {
+            return count;
+        }
+
+        /**
+         * Filter that rejects class Integer and allows others
+         *
+         * @param filterInfo access to the class, arrayLength, etc.
+         * @return {@code STATUS.REJECTED}
+         */
+        public ObjectInputFilter.Status checkInput(FilterInfo filterInfo) {
+            count++;
+            return ObjectInputFilter.Status.ALLOWED;
+        }
+    }
+
+}

test/java/rmi/registry/serialFilter/RegistryFilterTest.java

+/*
+ * Copyright (c) 2016, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.ObjectOutputStream;
+import java.io.Serializable;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.rmi.MarshalledObject;
+import java.rmi.NotBoundException;
+import java.rmi.Remote;
+import java.rmi.RemoteException;
+import java.rmi.AlreadyBoundException;
+import java.rmi.registry.LocateRegistry;
+import java.rmi.registry.Registry;
+import java.util.Objects;
+import java.security.Security;
+
+import org.testng.Assert;
+import org.testng.TestNG;
+import org.testng.annotations.BeforeSuite;
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Test;
+
+/*
+ * @test
+ * @library /java/rmi/testlibrary
+ * @modules java.rmi/sun.rmi.registry
+ *          java.rmi/sun.rmi.server
+ *          java.rmi/sun.rmi.transport
+ *          java.rmi/sun.rmi.transport.tcp
+ * @build TestLibrary
+ * @summary Test filters for the RMI Registry
+ * @run testng/othervm RegistryFilterTest
+ * @run testng/othervm
+ *        -Dsun.rmi.registry.registryFilter=!java.lang.Long;!RegistryFilterTest$RejectableClass
+ *        RegistryFilterTest
+ * @run testng/othervm/policy=security.policy
+ *        -Djava.security.properties=${test.src}/java.security-extra1
+ *        RegistryFilterTest
+ */
+public class RegistryFilterTest {
+    private static Registry impl;
+    private static int port;
+    private static Registry registry;
+
+    static final int REGISTRY_MAX_ARRAY = 10000;
+
+    static final String registryFilter =
+            System.getProperty("sun.rmi.registry.registryFilter",
+                    Security.getProperty("sun.rmi.registry.registryFilter"));
+
+    @DataProvider(name = "bindAllowed")
+    static Object[][] bindAllowedObjects() {
+        Object[][] objects = {
+        };
+        return objects;
+    }
+
+    /**
+     * Data RMI Regiry bind test.
+     * - name
+     * - Object
+     * - true/false if object is blacklisted by a filter (implicit or explicit)
+     * @return array of test data
+     */
+    @DataProvider(name = "bindData")
+    static Object[][] bindObjects() {
+        Object[][] data = {
+                { "byte[max]", new XX(new byte[REGISTRY_MAX_ARRAY]), false },
+                { "String", new XX("now is the time"), false},
+                { "String[]", new XX(new String[3]), false},
+                { "Long[4]", new XX(new Long[4]), registryFilter != null },
+                { "rej-byte[toobig]", new XX(new byte[REGISTRY_MAX_ARRAY + 1]), true },
+                { "rej-MarshalledObject", createMarshalledObject(), true },
+                { "rej-RejectableClass", new RejectableClass(), registryFilter != null},
+        };
+        return data;
+    }
+
+    static XX createMarshalledObject() {
+        try {
+            return new XX(new MarshalledObject<>(null));
+        } catch (IOException ioe) {
+            return new XX(ioe);
+        }
+    }
+
+    @BeforeSuite
+    static void setupRegistry() {
+        try {
+            impl = TestLibrary.createRegistryOnEphemeralPort();
+            port = TestLibrary.getRegistryPort(impl);
+            registry = LocateRegistry.getRegistry("localhost", port);
+        } catch (RemoteException ex) {
+            Assert.fail("initialization of registry", ex);
+        }
+
+        System.out.printf("RMI Registry filter: %s%n", registryFilter);
+    }
+
+
+    /*
+     * Test registry rejects an object with the max array size  + 1.
+     */
+    @Test(dataProvider="bindData")
+    public void simpleBind(String name, Remote obj, boolean blacklisted) throws RemoteException, AlreadyBoundException, NotBoundException {
+        try {
+            registry.bind(name, obj);
+            Assert.assertFalse(blacklisted, "Registry filter did not reject (but should have) ");
+            registry.unbind(name);
+        } catch (Exception rex) {
+            Assert.assertTrue(blacklisted, "Registry filter should not have rejected");
+        }
+    }
+
+    /*
+    * Test registry rejects an object with a well known class
+    * if blacklisted in the security properties.
+    */
+    @Test
+    public void simpleRejectableClass() throws RemoteException, AlreadyBoundException, NotBoundException {
+        RejectableClass r1 = null;
+        try {
+            String name = "reject1";
+            r1 = new RejectableClass();
+            registry.bind(name, r1);
+            registry.unbind(name);
+            Assert.assertNull(registryFilter, "Registry filter should not have rejected");
+        } catch (Exception rex) {
+            Assert.assertNotNull(registryFilter, "Registry filter should have rejected");
+        }
+    }
+
+    /**
+     * A simple Serializable Remote object that is passed by value.
+     * It and its contents are checked by the Registry serial filter.
+     */
+    static class XX implements Serializable, Remote {
+        private static final long serialVersionUID = 362498820763181265L;
+
+        final Object obj;
+
+        XX(Object obj) {
+            this.obj = obj;
+        }
+
+        public String toString() {
+            return super.toString() + "//" + Objects.toString(obj);
+        }
+    }
+    /**
+     * A simple Serializable Remote object that is passed by value.
+     * It and its contents are checked by the Registry serial filter.
+     */
+    static class RejectableClass implements Serializable, Remote {
+        private static final long serialVersionUID = 362498820763181264L;
+
+        RejectableClass() {}
+    }
+
+}

test/java/rmi/registry/serialFilter/java.security-extra1

+# RMI Registry Input Serial Filter
+#
+# The filter pattern uses the same format as java.io.ObjectInputStream.serialFilter.
+# This filter can override the builtin filter if additional types need to be
+# allowed or rejected from the RMI Registry.
+#
+sun.rmi.registry.registryFilter=!java.lang.Long;!RegistryFilterTest$RejectableClass
+

test/java/rmi/registry/serialFilter/security.policy

+grant {
+        permission java.security.AllPermission;
+};
+

test/sun/rmi/server/UnicastServerRef/FilterUSRTest.java

+/*
+ * Copyright (c) 2016, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+import java.io.InvalidClassException;
+import java.io.Serializable;
+
+import java.rmi.Remote;
+import java.rmi.RemoteException;
+import java.rmi.UnmarshalException;
+
+import java.util.Objects;
+
+import sun.misc.ObjectInputFilter;
+import sun.rmi.server.UnicastServerRef;
+import sun.rmi.server.UnicastServerRef2;
+import sun.rmi.transport.LiveRef;
+
+import org.testng.Assert;
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Test;
+
+/*
+ * @test
+ * @modules java.rmi/sun.rmi.registry
+ *          java.rmi/sun.rmi.server
+ *          java.rmi/sun.rmi.transport
+ *          java.rmi/sun.rmi.transport.tcp
+ * @run testng/othervm FilterUSRTest
+ * @summary Check objects exported with ObjectInputFilters via internal UnicastServerRef(2)
+ */
+public class FilterUSRTest {
+
+    /**
+     * Data to test serialFilter call counts.
+     * - name
+     * - Object
+     * - expected count of calls to checkInput.
+     *
+     * @return array of test data
+     */
+    @DataProvider(name = "bindData")
+    static Object[][] bindObjects() {
+        Object[][] data = {
+                {"SimpleString", "SimpleString", 0},
+                {"String", new XX("now is the time"), 1},
+                {"String[]", new XX(new String[3]), 3},
+                {"Long[4]", new XX(new Long[4]), 3},
+                {"RejectME", new XX(new RejectME()), -1},
+        };
+        return data;
+    }
+
+    /*
+     * Test exporting an object with a serialFilter using UnicastServerRef.exportObject().
+     * Send some objects and check the number of calls to the serialFilter.
+     */
+    @Test(dataProvider = "bindData")
+    public void UnicastServerRef(String name, Object obj, int expectedFilterCount) throws RemoteException {
+        try {
+            RemoteImpl impl = RemoteImpl.create();
+            UnicastServerRef ref = new UnicastServerRef(new LiveRef(0), impl.checker);
+
+            Echo client = (Echo) ref.exportObject(impl, null, false);
+
+            int count = client.filterCount(obj);
+            System.out.printf("count: %d, obj: %s%n", count, obj);
+            Assert.assertEquals(count, expectedFilterCount, "wrong number of filter calls");
+        } catch (RemoteException rex) {
+            if (expectedFilterCount == -1 &&
+                    UnmarshalException.class.equals(rex.getCause().getClass()) &&
+                    InvalidClassException.class.equals(rex.getCause().getCause().getClass())) {
+                return; // normal expected exception
+            }
+            rex.printStackTrace();
+            Assert.fail("unexpected remote exception", rex);
+        } catch (Exception ex) {
+            Assert.fail("unexpected exception", ex);
+        }
+    }
+
+    /*
+     * Test exporting an object with a serialFilter using UnicastServerRef2.exportObject()
+     * with explicit (but null) SocketFactories.
+     * Send some objects and check the number of calls to the serialFilter.
+     */
+    @Test(dataProvider = "bindData")
+    public void UnicastServerRef2(String name, Object obj, int expectedFilterCount) throws RemoteException {
+        try {
+            RemoteImpl impl = RemoteImpl.create();
+            UnicastServerRef2 ref = new UnicastServerRef2(0, null, null, impl.checker);
+
+            Echo client = (Echo) ref.exportObject(impl, null, false);
+
+            int count = client.filterCount(obj);
+            System.out.printf("count: %d, obj: %s%n", count, obj);
+            Assert.assertEquals(count, expectedFilterCount, "wrong number of filter calls");
+        } catch (RemoteException rex) {
+            if (expectedFilterCount == -1 &&
+                    UnmarshalException.class.equals(rex.getCause().getClass()) &&
+                    InvalidClassException.class.equals(rex.getCause().getCause().getClass())) {
+                return; // normal expected exception
+            }
+            rex.printStackTrace();
+            Assert.fail("unexpected remote exception", rex);
+        } catch (Exception rex) {
+            Assert.fail("unexpected exception", rex);
+        }
+    }
+
+    /**
+     * A simple Serializable holding an object that is passed by value.
+     * It and its contents are checked by the filter.
+     */
+    static class XX implements Serializable {
+        private static final long serialVersionUID = 362498820763181265L;
+
+        final Object obj;
+
+        XX(Object obj) {
+            this.obj = obj;
+        }
+
+        public String toString() {
+            return super.toString() + "//" + Objects.toString(obj);
+        }
+    }
+
+    interface Echo extends Remote {
+        int filterCount(Object obj) throws RemoteException;
+    }
+
+    /**
+     * This remote object just counts the calls to the serialFilter
+     * and returns it.  The caller can check the number against
+     * what was expected for the object passed as an argument.
+     * A new RemoteImpl is used for each test so the count starts at zero again.
+     */
+    static class RemoteImpl implements Echo {
+
+        private static final long serialVersionUID = 1L;
+
+        transient Checker checker;
+
+        static RemoteImpl create() throws RemoteException {
+            RemoteImpl impl = new RemoteImpl(new Checker());
+            return impl;
+        }
+
+        private RemoteImpl(Checker checker) throws RemoteException {
+            this.checker = checker;
+        }
+
+        public int filterCount(Object obj) throws RemoteException {
+            return checker.count();
+        }
+
+    }
+
+    /**
+     * A ObjectInputFilter that just counts when it is called.
+     */
+    static class Checker implements ObjectInputFilter {
+        int count;
+
+        @Override
+        public Status checkInput(ObjectInputFilter.FilterInfo info) {
+            if (info.serialClass() == RejectME.class) {
+                return Status.REJECTED;
+            }
+            count++;
+            return Status.UNDECIDED;
+        }
+
+        public int count() {
+            return count;
+        }
+    }
+
+    /**
+     * A class to be rejected by the filter.
+     */
+    static class RejectME implements Serializable {
+        private static final long serialVersionUID = 2L;
+    }
+
+}