8234836: Improve serialization handling

Reviewed-by: skoivu, rhalade, chegar

src/share/classes/java/io/ObjectInputStream.java

@@ -50,6 +50,7 @@ import sun.misc.SharedSecrets;
 import sun.reflect.misc.ReflectUtil;
 import sun.misc.JavaOISAccess;
 import sun.util.logging.PlatformLogger;
+import sun.security.action.GetBooleanAction;
 
 /**
  * An ObjectInputStream deserializes primitive data and objects previously
@@ -243,6 +244,23 @@ public class ObjectInputStream
         /** queue for WeakReferences to audited subclasses */
         static final ReferenceQueue<Class<?>> subclassAuditsQueue =
             new ReferenceQueue<>();
+
+        /**
+         * Property to permit setting a filter after objects
+         * have been read.
+         * See {@link #setObjectInputFilter(ObjectInputFilter)}
+         */
+        static final boolean SET_FILTER_AFTER_READ =
+                privilegedGetProperty("jdk.serialSetFilterAfterRead");
+
+        private static boolean privilegedGetProperty(String theProp) {
+            if (System.getSecurityManager() == null) {
+                return Boolean.getBoolean(theProp);
+            } else {
+                return AccessController.doPrivileged(
+                        new GetBooleanAction(theProp));
+            }
+        }
     }
 
     static {
@@ -1250,6 +1268,10 @@ public class ObjectInputStream
                 serialFilter != ObjectInputFilter.Config.getSerialFilter()) {
             throw new IllegalStateException("filter can not be set more than once");
         }
+        if (totalObjectRefs > 0 && !Caches.SET_FILTER_AFTER_READ) {
+            throw new IllegalStateException(
+                    "filter can not be set after an object has been read");
+        }
         this.serialFilter = filter;
     }
 

test/java/io/Serializable/serialFilter/SerialFilterTest.java

@@ -51,8 +51,10 @@ import org.testng.annotations.Test;
 import org.testng.annotations.DataProvider;
 
 /* @test
+ * @bug 8234836
  * @build SerialFilterTest
  * @run testng/othervm  SerialFilterTest
+ * @run testng/othervm  -Djdk.serialSetFilterAfterRead=true SerialFilterTest
  *
  * @summary Test ObjectInputFilters
  */
@@ -76,6 +78,10 @@ public class SerialFilterTest implements Serializable {
      */
     private static final Object otherObject = Integer.valueOf(0);
 
+    // Cache value of jdk.serialSetFilterAfterRead property.
+    static final boolean SET_FILTER_AFTER_READ =
+            Boolean.getBoolean("jdk.serialSetFilterAfterRead");
+
     /**
      * DataProvider for the individual patterns to test.
      * Expand the patterns into cases for each of the Std and Compatibility APIs.
@@ -297,6 +303,46 @@ public class SerialFilterTest implements Serializable {
         }
     }
 
+    /**
+     * After reading some objects from the stream, setting a filter is disallowed.
+     * If the filter was allowed to be set, it would have unpredictable behavior.
+     * Objects already read would not be checked again, including class descriptors.
+     *
+     * Note: To mitigate possible incompatibility a system property can be set
+     * to revert to the old behavior but it re-enables the incorrect use.
+     */
+    @Test
+    static void testNonSettableAfterReadObject() throws IOException, ClassNotFoundException {
+        String expected1 = "text1";
+        String expected2 = "text2";
+        byte[] bytes = writeObjects(expected1, expected2);
+
+        for (boolean toggle: new boolean[] {true, false}) {
+            try (ByteArrayInputStream bais = new ByteArrayInputStream(bytes);
+                 ObjectInputStream ois = new ObjectInputStream(bais)) {
+                Object actual1 = toggle ? ois.readObject() : ois.readUnshared();
+                Assert.assertEquals(actual1, expected1, "unexpected string");
+                // Attempt to set filter
+                ObjectInputFilter filter = new ObjectInputFilter() {
+                    @Override
+                    public Status checkInput(FilterInfo filterInfo) {
+                        return null;
+                    }
+                };
+                ObjectInputFilter.Config.setObjectInputFilter(ois, filter);
+                if (!SET_FILTER_AFTER_READ)
+                    Assert.fail("Should not be able to set filter after readObject has been called");
+            } catch (IllegalStateException ise) {
+                // success, the exception was expected
+                if (SET_FILTER_AFTER_READ)
+                    Assert.fail("With jdk.serialSetFilterAfterRead property set = true; " +
+                            "should be able to set the filter after a read");
+            } catch (EOFException eof) {
+                Assert.fail("Should not reach end-of-file", eof);
+            }
+        }
+    }
+
     /**
      * Test that if an Objects readReadResolve method returns an array
      * that the callback to the filter includes the proper array length.