8185346: Relax RMI Registry Serial Filter to allow arrays of any type

Summary: Registry filter should allow arrays of any type
Reviewed-by: rriggs, coffeys

src/share/classes/sun/misc/ObjectInputFilter.java

@@ -379,7 +379,20 @@ public interface ObjectInputFilter {
          */
         public static ObjectInputFilter createFilter(String pattern) {
             Objects.requireNonNull(pattern, "pattern");
-            return Global.createFilter(pattern);
+            return Global.createFilter(pattern, true);
+        }
+
+        /**
+         * Returns an ObjectInputFilter from a string of patterns that
+         * checks only the length for arrays, not the component type.
+         *
+         * @param pattern the pattern string to parse; not null
+         * @return a filter to check a class being deserialized;
+         *          {@code null} if no patterns
+         */
+        public static ObjectInputFilter createFilter2(String pattern) {
+            Objects.requireNonNull(pattern, "pattern");
+            return Global.createFilter(pattern, false);
         }
 
         /**
@@ -413,18 +426,24 @@ public interface ObjectInputFilter {
              * Maximum length of any array.
              */
             private long maxArrayLength;
+            /**
+             * True to check the component type for arrays.
+             */
+            private final boolean checkComponentType;
 
             /**
              * Returns an ObjectInputFilter from a string of patterns.
              *
              * @param pattern the pattern string to parse
+             * @param checkComponentType true if the filter should check
+             *                           the component type of arrays
              * @return a filter to check a class being deserialized; not null
              * @throws IllegalArgumentException if the parameter is malformed
              *                if the pattern is missing the name, the long value
              *                is not a number or is negative.
              */
-            static ObjectInputFilter createFilter(String pattern) {
-                Global filter = new Global(pattern);
+            static ObjectInputFilter createFilter(String pattern, boolean checkComponentType) {
+                Global filter = new Global(pattern, checkComponentType);
                 return filter.isEmpty() ? null : filter;
             }
 
@@ -432,10 +451,13 @@ public interface ObjectInputFilter {
              * Construct a new filter from the pattern String.
              *
              * @param pattern a pattern string of filters
+             * @param checkComponentType true if the filter should check
+             *                           the component type of arrays
              * @throws IllegalArgumentException if the pattern is malformed
              */
-            private Global(String pattern) {
+            private Global(String pattern, boolean checkComponentType) {
                 this.pattern = pattern;
+                this.checkComponentType = checkComponentType;
 
                 maxArrayLength = Long.MAX_VALUE; // Default values are unlimited
                 maxDepth = Long.MAX_VALUE;
@@ -594,6 +616,10 @@ public interface ObjectInputFilter {
                             // array length is too big
                             return Status.REJECTED;
                         }
+                        if (!checkComponentType) {
+                            // As revised; do not check the component type for arrays
+                            return Status.UNDECIDED;
+                        }
                         do {
                             // Arrays are decided based on the component type
                             clazz = clazz.getComponentType();

src/share/classes/sun/rmi/registry/RegistryImpl.java

@@ -103,7 +103,7 @@ public class RegistryImpl extends java.rmi.server.RemoteServer
     private static final int REGISTRY_MAX_DEPTH = 20;
 
     /** Registry maximum array size in remote invocations. **/
-    private static final int REGISTRY_MAX_ARRAY_SIZE = 10000;
+    private static final int REGISTRY_MAX_ARRAY_SIZE = 1_000_000;
 
     /**
      * The registryFilter created from the value of the {@code "sun.rmi.registry.registryFilter"}
@@ -123,7 +123,7 @@ public class RegistryImpl extends java.rmi.server.RemoteServer
             props = Security.getProperty(REGISTRY_FILTER_PROPNAME);
         }
         if (props != null) {
-            filter = ObjectInputFilter.Config.createFilter(props);
+            filter = ObjectInputFilter.Config.createFilter2(props);
             Log regLog = Log.getLog("sun.rmi.registry", "registry", -1);
             if (regLog.isLoggable(Log.BRIEF)) {
                 regLog.log(Log.BRIEF, "registryFilter = " + filter);
@@ -419,17 +419,10 @@ public class RegistryImpl extends java.rmi.server.RemoteServer
         Class<?> clazz = filterInfo.serialClass();
         if (clazz != null) {
             if (clazz.isArray()) {
-                if (filterInfo.arrayLength() >= 0 && filterInfo.arrayLength() > REGISTRY_MAX_ARRAY_SIZE) {
-                    return ObjectInputFilter.Status.REJECTED;
-                }
-                do {
-                    // Arrays are allowed depending on the component type
-                    clazz = clazz.getComponentType();
-                } while (clazz.isArray());
-            }
-            if (clazz.isPrimitive()) {
-                // Arrays of primitives are allowed
-                return ObjectInputFilter.Status.ALLOWED;
+                // Arrays are REJECTED only if they exceed the limit
+                return (filterInfo.arrayLength() >= 0 && filterInfo.arrayLength() > REGISTRY_MAX_ARRAY_SIZE)
+                    ? ObjectInputFilter.Status.REJECTED
+                    : ObjectInputFilter.Status.UNDECIDED;
             }
             if (String.class == clazz
                     || java.lang.Number.class.isAssignableFrom(clazz)

src/share/lib/security/java.security-aix

@@ -889,10 +889,33 @@ jdk.xml.dsig.secureValidationPolicy=\
 #
 # The filter pattern uses the same format as jdk.serialFilter.
 # This filter can override the builtin filter if additional types need to be
-# allowed or rejected from the RMI Registry.
-#
-#sun.rmi.registry.registryFilter=pattern;pattern
-
+# allowed or rejected from the RMI Registry or to decrease limits but not
+# to increase limits.
+# If the limits (maxdepth, maxrefs, or maxbytes) are exceeded, the object is rejected.
+#
+# Each non-array type is allowed or rejected if it matches one of the patterns,
+# evaluated from left to right, and is otherwise allowed. Arrays of any
+# component type, including subarrays and arrays of primitives, are allowed.
+#
+# Array construction of any component type, including subarrays and arrays of
+# primitives, are allowed unless the length is greater than the maxarray limit.
+# The filter is applied to each array element.
+#
+# The built-in filter allows subclasses of allowed classes and
+# can approximately be represented as the pattern:
+#
+#sun.rmi.registry.registryFilter=\
+#    maxarray=1000000;\
+#    maxdepth=20;\
+#    java.lang.String;\
+#    java.lang.Number;\
+#    java.lang.reflect.Proxy;\
+#    java.rmi.Remote;\
+#    sun.rmi.server.UnicastRef;\
+#    sun.rmi.server.RMIClientSocketFactory;\
+#    sun.rmi.server.RMIServerSocketFactory;\
+#    java.rmi.activation.ActivationID;\
+#    java.rmi.server.UID
 #
 # RMI Distributed Garbage Collector (DGC) Serial Filter
 #

src/share/lib/security/java.security-linux

@@ -889,10 +889,33 @@ jdk.xml.dsig.secureValidationPolicy=\
 #
 # The filter pattern uses the same format as jdk.serialFilter.
 # This filter can override the builtin filter if additional types need to be
-# allowed or rejected from the RMI Registry.
-#
-#sun.rmi.registry.registryFilter=pattern;pattern
-
+# allowed or rejected from the RMI Registry or to decrease limits but not
+# to increase limits.
+# If the limits (maxdepth, maxrefs, or maxbytes) are exceeded, the object is rejected.
+#
+# Each non-array type is allowed or rejected if it matches one of the patterns,
+# evaluated from left to right, and is otherwise allowed. Arrays of any
+# component type, including subarrays and arrays of primitives, are allowed.
+#
+# Array construction of any component type, including subarrays and arrays of
+# primitives, are allowed unless the length is greater than the maxarray limit.
+# The filter is applied to each array element.
+#
+# The built-in filter allows subclasses of allowed classes and
+# can approximately be represented as the pattern:
+#
+#sun.rmi.registry.registryFilter=\
+#    maxarray=1000000;\
+#    maxdepth=20;\
+#    java.lang.String;\
+#    java.lang.Number;\
+#    java.lang.reflect.Proxy;\
+#    java.rmi.Remote;\
+#    sun.rmi.server.UnicastRef;\
+#    sun.rmi.server.RMIClientSocketFactory;\
+#    sun.rmi.server.RMIServerSocketFactory;\
+#    java.rmi.activation.ActivationID;\
+#    java.rmi.server.UID
 #
 # RMI Distributed Garbage Collector (DGC) Serial Filter
 #

src/share/lib/security/java.security-macosx

@@ -892,10 +892,33 @@ jdk.xml.dsig.secureValidationPolicy=\
 #
 # The filter pattern uses the same format as jdk.serialFilter.
 # This filter can override the builtin filter if additional types need to be
-# allowed or rejected from the RMI Registry.
-#
-#sun.rmi.registry.registryFilter=pattern;pattern
-
+# allowed or rejected from the RMI Registry or to decrease limits but not
+# to increase limits.
+# If the limits (maxdepth, maxrefs, or maxbytes) are exceeded, the object is rejected.
+#
+# Each non-array type is allowed or rejected if it matches one of the patterns,
+# evaluated from left to right, and is otherwise allowed. Arrays of any
+# component type, including subarrays and arrays of primitives, are allowed.
+#
+# Array construction of any component type, including subarrays and arrays of
+# primitives, are allowed unless the length is greater than the maxarray limit.
+# The filter is applied to each array element.
+#
+# The built-in filter allows subclasses of allowed classes and
+# can approximately be represented as the pattern:
+#
+#sun.rmi.registry.registryFilter=\
+#    maxarray=1000000;\
+#    maxdepth=20;\
+#    java.lang.String;\
+#    java.lang.Number;\
+#    java.lang.reflect.Proxy;\
+#    java.rmi.Remote;\
+#    sun.rmi.server.UnicastRef;\
+#    sun.rmi.server.RMIClientSocketFactory;\
+#    sun.rmi.server.RMIServerSocketFactory;\
+#    java.rmi.activation.ActivationID;\
+#    java.rmi.server.UID
 #
 # RMI Distributed Garbage Collector (DGC) Serial Filter
 #

src/share/lib/security/java.security-solaris

@@ -891,10 +891,33 @@ jdk.xml.dsig.secureValidationPolicy=\
 #
 # The filter pattern uses the same format as jdk.serialFilter.
 # This filter can override the builtin filter if additional types need to be
-# allowed or rejected from the RMI Registry.
-#
-#sun.rmi.registry.registryFilter=pattern;pattern
-
+# allowed or rejected from the RMI Registry or to decrease limits but not
+# to increase limits.
+# If the limits (maxdepth, maxrefs, or maxbytes) are exceeded, the object is rejected.
+#
+# Each non-array type is allowed or rejected if it matches one of the patterns,
+# evaluated from left to right, and is otherwise allowed. Arrays of any
+# component type, including subarrays and arrays of primitives, are allowed.
+#
+# Array construction of any component type, including subarrays and arrays of
+# primitives, are allowed unless the length is greater than the maxarray limit.
+# The filter is applied to each array element.
+#
+# The built-in filter allows subclasses of allowed classes and
+# can approximately be represented as the pattern:
+#
+#sun.rmi.registry.registryFilter=\
+#    maxarray=1000000;\
+#    maxdepth=20;\
+#    java.lang.String;\
+#    java.lang.Number;\
+#    java.lang.reflect.Proxy;\
+#    java.rmi.Remote;\
+#    sun.rmi.server.UnicastRef;\
+#    sun.rmi.server.RMIClientSocketFactory;\
+#    sun.rmi.server.RMIServerSocketFactory;\
+#    java.rmi.activation.ActivationID;\
+#    java.rmi.server.UID
 #
 # RMI Distributed Garbage Collector (DGC) Serial Filter
 #

src/share/lib/security/java.security-windows

@@ -892,10 +892,33 @@ jdk.xml.dsig.secureValidationPolicy=\
 #
 # The filter pattern uses the same format as jdk.serialFilter.
 # This filter can override the builtin filter if additional types need to be
-# allowed or rejected from the RMI Registry.
-#
-#sun.rmi.registry.registryFilter=pattern;pattern
-
+# allowed or rejected from the RMI Registry or to decrease limits but not
+# to increase limits.
+# If the limits (maxdepth, maxrefs, or maxbytes) are exceeded, the object is rejected.
+#
+# Each non-array type is allowed or rejected if it matches one of the patterns,
+# evaluated from left to right, and is otherwise allowed. Arrays of any
+# component type, including subarrays and arrays of primitives, are allowed.
+#
+# Array construction of any component type, including subarrays and arrays of
+# primitives, are allowed unless the length is greater than the maxarray limit.
+# The filter is applied to each array element.
+#
+# The built-in filter allows subclasses of allowed classes and
+# can approximately be represented as the pattern:
+#
+#sun.rmi.registry.registryFilter=\
+#    maxarray=1000000;\
+#    maxdepth=20;\
+#    java.lang.String;\
+#    java.lang.Number;\
+#    java.lang.reflect.Proxy;\
+#    java.rmi.Remote;\
+#    sun.rmi.server.UnicastRef;\
+#    sun.rmi.server.RMIClientSocketFactory;\
+#    sun.rmi.server.RMIServerSocketFactory;\
+#    java.rmi.activation.ActivationID;\
+#    java.rmi.server.UID
 #
 # RMI Distributed Garbage Collector (DGC) Serial Filter
 #

test/java/rmi/registry/serialFilter/RegistryFilterTest.java

@@ -35,7 +35,6 @@ import java.security.Security;
 import java.util.Objects;
 
 import org.testng.Assert;
-import org.testng.TestNG;
 import org.testng.annotations.BeforeSuite;
 import org.testng.annotations.DataProvider;
 import org.testng.annotations.Test;
@@ -43,10 +42,6 @@ import org.testng.annotations.Test;
 /*
  * @test
  * @library /java/rmi/testlibrary
- * @modules java.rmi/sun.rmi.registry
- *          java.rmi/sun.rmi.server
- *          java.rmi/sun.rmi.transport
- *          java.rmi/sun.rmi.transport.tcp
  * @build TestLibrary
  * @summary Test filters for the RMI Registry
  * @run testng/othervm RegistryFilterTest
@@ -65,21 +60,14 @@ public class RegistryFilterTest {
 
     static final int REGISTRY_MAX_DEPTH = 20;
 
-    static final int REGISTRY_MAX_ARRAY = 10000;
+    static final int REGISTRY_MAX_ARRAY = 1_000_000;
 
     static final String registryFilter =
             System.getProperty("sun.rmi.registry.registryFilter",
                     Security.getProperty("sun.rmi.registry.registryFilter"));
 
-    @DataProvider(name = "bindAllowed")
-    static Object[][] bindAllowedObjects() {
-        Object[][] objects = {
-        };
-        return objects;
-    }
-
     /**
-     * Data RMI Regiry bind test.
+     * Data RMI Registry bind test.
      * - name
      * - Object
      * - true/false if object is blacklisted by a filter (implicit or explicit)
@@ -90,9 +78,11 @@ public class RegistryFilterTest {
         Object[][] data = {
                 { "byte[max]", new XX(new byte[REGISTRY_MAX_ARRAY]), false },
                 { "String", new XX("now is the time"), false},
-                { "String[]", new XX(new String[3]), false},
-                { "Long[4]", new XX(new Long[4]), registryFilter != null },
+                { "String[3]", new XX(new String[3]), false},
+                { "Long[4]", new XX(new Long[4]), false },
+                { "Object[REGISTRY_MAX_ARRAY]", new XX(new Object[REGISTRY_MAX_ARRAY]), false },
                 { "rej-byte[toobig]", new XX(new byte[REGISTRY_MAX_ARRAY + 1]), true },
+                { "rej-Object[toobig]", new XX(new Object[REGISTRY_MAX_ARRAY + 1]), true },
                 { "rej-MarshalledObject", createMarshalledObject(), true },
                 { "rej-RejectableClass", new RejectableClass(), registryFilter != null},
         };