8149338: JVM Crash caused by Marlin renderer not handling NaN coordinates

Summary: use first / last Y crossings to compute edge min/max Y and ensure consistency with edgeBuckets / edgeBucketCounts arrays
Reviewed-by: flar, prr

src/share/classes/sun/java2d/marlin/Renderer.java

@@ -148,8 +148,8 @@ final class Renderer implements PathConsumer2D, MarlinConst {
 //////////////////////////////////////////////////////////////////////////////
 //  EDGE LIST
 //////////////////////////////////////////////////////////////////////////////
-    private float edgeMinY = Float.POSITIVE_INFINITY;
-    private float edgeMaxY = Float.NEGATIVE_INFINITY;
+    private int edgeMinY = Integer.MAX_VALUE;
+    private int edgeMaxY = Integer.MIN_VALUE;
     private float edgeMinX = Float.POSITIVE_INFINITY;
     private float edgeMaxX = Float.NEGATIVE_INFINITY;
 
@@ -357,18 +357,21 @@ final class Renderer implements PathConsumer2D, MarlinConst {
             }
             return;
         }
-        // edge min/max X/Y are in subpixel space (inclusive)
-        if (y1 < edgeMinY) {
-            edgeMinY = y1;
+
+        // edge min/max X/Y are in subpixel space (inclusive) within bounds:
+        // note: Use integer crossings to ensure consistent range within
+        // edgeBuckets / edgeBucketCounts arrays in case of NaN values (int = 0)
+        if (firstCrossing < edgeMinY) {
+            edgeMinY = firstCrossing;
         }
-        if (y2 > edgeMaxY) {
-            edgeMaxY = y2;
+        if (lastCrossing > edgeMaxY) {
+            edgeMaxY = lastCrossing;
         }
 
         // Use double-precision for improved accuracy:
         final double x1d   = x1;
         final double y1d   = y1;
-        final double slope = (x2 - x1d) / (y2 - y1d);
+        final double slope = (x1d - x2) / (y1d - y2);
 
         if (slope >= 0.0) { // <==> x1 < x2
             if (x1 < edgeMinX) {
@@ -504,7 +507,7 @@ final class Renderer implements PathConsumer2D, MarlinConst {
     private float x0, y0;
 
     // Position of most recent 'moveTo' command
-    private float pix_sx0, pix_sy0;
+    private float sx0, sy0;
 
     // per-thread renderer context
     final RendererContext rdrCtx;
@@ -570,8 +573,8 @@ final class Renderer implements PathConsumer2D, MarlinConst {
             edgeBucketCounts = rdrCtx.getIntArray(edgeBucketsLength);
         }
 
-        edgeMinY = Float.POSITIVE_INFINITY;
-        edgeMaxY = Float.NEGATIVE_INFINITY;
+        edgeMinY = Integer.MAX_VALUE;
+        edgeMaxY = Integer.MIN_VALUE;
         edgeMinX = Float.POSITIVE_INFINITY;
         edgeMaxX = Float.NEGATIVE_INFINITY;
 
@@ -628,7 +631,7 @@ final class Renderer implements PathConsumer2D, MarlinConst {
             blkFlags = blkFlags_initial;
         }
 
-        if (edgeMinY != Float.POSITIVE_INFINITY) {
+        if (edgeMinY != Integer.MAX_VALUE) {
             // if context is maked as DIRTY:
             if (rdrCtx.dirty) {
                 // may happen if an exception if thrown in the pipeline processing:
@@ -688,16 +691,18 @@ final class Renderer implements PathConsumer2D, MarlinConst {
     @Override
     public void moveTo(float pix_x0, float pix_y0) {
         closePath();
-        this.pix_sx0 = pix_x0;
-        this.pix_sy0 = pix_y0;
-        this.y0 = tosubpixy(pix_y0);
-        this.x0 = tosubpixx(pix_x0);
+        final float sx = tosubpixx(pix_x0);
+        final float sy = tosubpixy(pix_y0);
+        this.sx0 = sx;
+        this.sy0 = sy;
+        this.x0 = sx;
+        this.y0 = sy;
     }
 
     @Override
     public void lineTo(float pix_x1, float pix_y1) {
-        float x1 = tosubpixx(pix_x1);
-        float y1 = tosubpixy(pix_y1);
+        final float x1 = tosubpixx(pix_x1);
+        final float y1 = tosubpixy(pix_y1);
         addLine(x0, y0, x1, y1);
         x0 = x1;
         y0 = y1;
@@ -729,8 +734,9 @@ final class Renderer implements PathConsumer2D, MarlinConst {
 
     @Override
     public void closePath() {
-        // lineTo expects its input in pixel coordinates.
-        lineTo(pix_sx0, pix_sy0);
+        addLine(x0, y0, sx0, sy0);
+        x0 = sx0;
+        y0 = sy0;
     }
 
     @Override
@@ -1396,7 +1402,7 @@ final class Renderer implements PathConsumer2D, MarlinConst {
         if (doMonitors) {
             RendererContext.stats.mon_rdr_endRendering.start();
         }
-        if (edgeMinY == Float.POSITIVE_INFINITY) {
+        if (edgeMinY == Integer.MAX_VALUE) {
             return false; // undefined edges bounds
         }
 
@@ -1407,11 +1413,10 @@ final class Renderer implements PathConsumer2D, MarlinConst {
         final int spminX = FloatMath.max(FloatMath.ceil_int(edgeMinX - 0.5f), boundsMinX);
         final int spmaxX = FloatMath.min(FloatMath.ceil_int(edgeMaxX - 0.5f), boundsMaxX - 1);
 
-        // y1 (and y2) are already biased by -0.5 in tosubpixy():
-        final int spminY = FloatMath.max(FloatMath.ceil_int(edgeMinY), _boundsMinY);
-        int maxY = FloatMath.ceil_int(edgeMaxY);
-
+        // edge Min/Max Y are already rounded to subpixels within bounds:
+        final int spminY = edgeMinY;
         final int spmaxY;
+        int maxY = edgeMaxY;
 
         if (maxY <= _boundsMaxY - 1) {
             spmaxY = maxY;

test/sun/java2d/marlin/CrashNaNTest.java

+/*
+ * Copyright (c) 2016, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+import java.awt.Color;
+import java.awt.Graphics2D;
+import java.awt.RenderingHints;
+import java.awt.geom.Path2D;
+import java.awt.image.BufferedImage;
+import java.io.File;
+import java.io.IOException;
+import static java.lang.Double.NaN;
+import java.util.Locale;
+import java.util.logging.Handler;
+import java.util.logging.LogRecord;
+import java.util.logging.Logger;
+import javax.imageio.ImageIO;
+
+/**
+ * @test
+ * @bug 8149338
+ * @summary Verifies that Marlin supports NaN coordinates and no JVM crash happens !
+ * @run main CrashNaNTest
+ */
+public class CrashNaNTest {
+
+    static final boolean SAVE_IMAGE = false;
+
+    public static void main(String argv[]) {
+        Locale.setDefault(Locale.US);
+
+        // initialize j.u.l Looger:
+        final Logger log = Logger.getLogger("sun.java2d.marlin");
+        log.addHandler(new Handler() {
+            @Override
+            public void publish(LogRecord record) {
+                Throwable th = record.getThrown();
+                // detect any Throwable:
+                if (th != null) {
+                    System.out.println("Test failed:\n" + record.getMessage());
+                    th.printStackTrace(System.out);
+
+                    throw new RuntimeException("Test failed: ", th);
+                }
+            }
+
+            @Override
+            public void flush() {
+            }
+
+            @Override
+            public void close() throws SecurityException {
+            }
+        });
+
+        // enable Marlin logging & internal checks:
+        System.setProperty("sun.java2d.renderer.log", "true");
+        System.setProperty("sun.java2d.renderer.useLogger", "true");
+        System.setProperty("sun.java2d.renderer.doChecks", "true");
+
+        final int width = 400;
+        final int height = 400;
+
+        final BufferedImage image = new BufferedImage(width, height,
+                BufferedImage.TYPE_INT_ARGB);
+
+        final Graphics2D g2d = (Graphics2D) image.getGraphics();
+        try {
+            g2d.setRenderingHint(RenderingHints.KEY_ANTIALIASING,
+                    RenderingHints.VALUE_ANTIALIAS_ON);
+
+            g2d.setBackground(Color.WHITE);
+            g2d.clearRect(0, 0, width, height);
+
+            final Path2D.Double path = new Path2D.Double();
+            path.moveTo(30, 30);
+            path.lineTo(100, 100);
+
+            for (int i = 0; i < 20000; i++) {
+                path.lineTo(110 + 0.01 * i, 110);
+                path.lineTo(111 + 0.01 * i, 100);
+            }
+
+            path.lineTo(NaN, 200);
+            path.lineTo(200, 200);
+            path.lineTo(200, NaN);
+            path.lineTo(300, 300);
+            path.lineTo(NaN, NaN);
+            path.lineTo(100, 100);
+            path.closePath();
+
+            final Path2D.Double path2 = new Path2D.Double();
+            path2.moveTo(0,0);
+            path2.lineTo(width,height);
+            path2.lineTo(10, 10);
+            path2.closePath();
+
+            for (int i = 0; i < 1; i++) {
+                final long start = System.nanoTime();
+                g2d.setColor(Color.BLUE);
+                g2d.fill(path);
+
+                g2d.fill(path2);
+
+                final long time = System.nanoTime() - start;
+                System.out.println("paint: duration= " + (1e-6 * time) + " ms.");
+            }
+
+            if (SAVE_IMAGE) {
+                try {
+                    final File file = new File("CrashNaNTest.png");
+                    System.out.println("Writing file: "
+                            + file.getAbsolutePath());
+                    ImageIO.write(image, "PNG", file);
+                } catch (IOException ex) {
+                    System.out.println("Writing file failure:");
+                    ex.printStackTrace();
+                }
+            }
+        } finally {
+            g2d.dispose();
+        }
+    }
+}

test/sun/java2d/marlin/TextClipErrorTest.java

@@ -69,26 +69,14 @@ public class TextClipErrorTest {
             @Override
             public void publish(LogRecord record) {
                 Throwable th = record.getThrown();
-                // detect potential Throwable thrown by XxxArrayCache.check():
-                if (th != null && th.getClass() == Throwable.class) {
-                    StackTraceElement[] stackElements = th.getStackTrace();
-
-                    for (int i = 0; i < stackElements.length; i++) {
-                        StackTraceElement e = stackElements[i];
-
-                        if (e.getClassName().startsWith("sun.java2d.marlin")
-                            && e.getClassName().contains("ArrayCache")
-                            && "check".equals(e.getMethodName()))
-                        {
-                            System.out.println("Test failed:\n"
-                                + record.getMessage());
+                // detect any Throwable:
+                if (th != null) {
+                    System.out.println("Test failed:\n" + record.getMessage());
                     th.printStackTrace(System.out);
 
                     throw new RuntimeException("Test failed: ", th);
                 }
             }
-                }
-            }
 
             @Override
             public void flush() {