8225279: Better XRender interpolation

Reviewed-by: serb, prr, rhalade, ahgross

src/solaris/native/sun/java2d/x11/X11SurfaceData.c

@@ -78,6 +78,7 @@ static void X11SD_SwapBytes(X11SDOps *xsdo, XImage *img, int depth, int bpp);
 static XImage * X11SD_GetImage(JNIEnv *env, X11SDOps *xsdo,
                                SurfaceDataBounds *bounds,
                                jint lockFlags);
+static int X11SD_GetBitmapPad(int pixelStride);
 
 extern jfieldID validID;
 
@@ -438,11 +439,33 @@ jboolean XShared_initSurface(JNIEnv *env, X11SDOps *xsdo, jint depth, jint width
         xsdo->drawable = drawable;
         xsdo->isPixmap = JNI_FALSE;
     } else {
+        jboolean sizeIsInvalid = JNI_FALSE;
+        jlong scan = 0;
+
         /*
          * width , height must be nonzero otherwise XCreatePixmap
          * generates BadValue in error_handler
          */
         if (width <= 0 || height <= 0 || width > 32767 || height > 32767) {
+            sizeIsInvalid = JNI_TRUE;
+        } else {
+            XImage* tmpImg = NULL;
+
+            AWT_LOCK();
+            tmpImg = XCreateImage(awt_display,
+                xsdo->configData->awt_visInfo.visual,
+                depth, ZPixmap, 0, NULL, width, height,
+                X11SD_GetBitmapPad(xsdo->configData->pixelStride), 0);
+            if (tmpImg) {
+                scan = (jlong) tmpImg->bytes_per_line;
+                XDestroyImage(tmpImg);
+                tmpImg = NULL;
+            }
+            AWT_UNLOCK();
+            JNU_CHECK_EXCEPTION_RETURN(env, JNI_FALSE);
+        }
+
+        if (sizeIsInvalid || (scan * height > 0x7FFFFFFFL)) {
             JNU_ThrowOutOfMemoryError(env,
                                   "Can't create offscreen surface");
             return JNI_FALSE;
@@ -457,7 +480,7 @@ jboolean XShared_initSurface(JNIEnv *env, X11SDOps *xsdo, jint depth, jint width
         xsdo->pmHeight = height;
 
 #ifdef MITSHM
-        xsdo->shmPMData.pmSize = width * height * depth;
+        xsdo->shmPMData.pmSize = (jlong) width * height * depth;
         xsdo->shmPMData.pixelsReadThreshold = width * height / 8;
         if (forceSharedPixmaps) {
             AWT_LOCK();
@@ -560,7 +583,7 @@ XImage* X11SD_CreateSharedImage(X11SDOps *xsdo,
         return NULL;
     }
     shminfo->shmid =
-        shmget(IPC_PRIVATE, height * img->bytes_per_line,
+        shmget(IPC_PRIVATE, (size_t) height * img->bytes_per_line,
                IPC_CREAT|mitShmPermissionMask);
     if (shminfo->shmid < 0) {
         J2dRlsTraceLn1(J2D_TRACE_ERROR,
@@ -622,7 +645,7 @@ XImage* X11SD_GetSharedImage(X11SDOps *xsdo, jint width, jint height,
         XSync(awt_display, False);
         retImage = cachedXImage;
         cachedXImage = (XImage *)NULL;
-    } else if (width * height * xsdo->depth > 0x10000) {
+    } else if ((jlong) width * height * xsdo->depth > 0x10000) {
         retImage = X11SD_CreateSharedImage(xsdo, width, height);
     }
     return retImage;
@@ -978,7 +1001,7 @@ static void X11SD_GetRasInfo(JNIEnv *env,
             int scan = xpriv->img->bytes_per_line;
             xpriv->x = x;
             xpriv->y = y;
-            pRasInfo->rasBase = xpriv->img->data - x * mult - y * scan;
+            pRasInfo->rasBase = xpriv->img->data - x * mult - (intptr_t) y * scan;
             pRasInfo->pixelStride = mult;
             pRasInfo->pixelBitOffset = 0;
             pRasInfo->scanStride = scan;
@@ -1140,8 +1163,8 @@ X11SD_FindClip(SurfaceDataBounds *b, SurfaceDataBounds *bounds, X11SDOps *xsdo)
 
 static void
 X11SD_SwapBytes(X11SDOps *xsdo, XImage * img, int depth, int bpp) {
-    int lengthInBytes = img->height * img->bytes_per_line;
-    int i;
+    jlong lengthInBytes = (jlong) img->height * img->bytes_per_line;
+    jlong i;
 
     switch (depth) {
     case 12:
@@ -1214,7 +1237,7 @@ static XImage * X11SD_GetImage(JNIEnv *env, X11SDOps *xsdo,
     Drawable drawable;
     int depth = xsdo->depth;
     int mult = xsdo->configData->pixelStride;
-    int pad = (mult == 3) ? 32 : mult * 8; // pad must be 8, 16, or 32
+    int pad = X11SD_GetBitmapPad(mult);
     jboolean readBits = lockFlags & SD_LOCK_NEED_PIXELS;
 
     x = bounds->x1;
@@ -1280,7 +1303,7 @@ static XImage * X11SD_GetImage(JNIEnv *env, X11SDOps *xsdo,
             }
 
             scan = img->bytes_per_line;
-            img->data = malloc(h * scan);
+            img->data = malloc((size_t) h * scan);
             if (img->data == NULL) {
                 XFree(img);
                 return NULL;
@@ -1315,7 +1338,7 @@ static XImage * X11SD_GetImage(JNIEnv *env, X11SDOps *xsdo,
                     int i;
 
                     img_addr = img->data +
-                        (temp.y1 - y) * scan + (temp.x1 - x) * mult;
+                        (intptr_t) (temp.y1 - y) * scan + (temp.x1 - x) * mult;
                     temp_scan = temp_image->bytes_per_line;
                     temp_addr = temp_image->data;
                     bytes_to_copy = (temp.x2 - temp.x1) * mult;
@@ -1349,7 +1372,7 @@ static XImage * X11SD_GetImage(JNIEnv *env, X11SDOps *xsdo,
                 return NULL;
             }
 
-            img->data = malloc(h * img->bytes_per_line);
+            img->data = malloc((size_t) h * img->bytes_per_line);
             if (img->data == NULL) {
                 XFree(img);
                 return NULL;
@@ -1532,6 +1555,11 @@ X11SD_ReleasePixmapWithBg(JNIEnv *env, X11SDOps *xsdo)
 #endif /* MITSHM */
 }
 
+static int X11SD_GetBitmapPad(int pixelStride) {
+    // pad must be 8, 16, or 32
+    return (pixelStride == 3) ? 32 : pixelStride * 8;
+}
+
 #endif /* !HEADLESS */
 
 /*

src/solaris/native/sun/java2d/x11/X11SurfaceData.h

@@ -81,7 +81,7 @@ typedef struct {
     XShmSegmentInfo     *shmSegInfo;    /* Shared Memory Segment Info */
     jint                bytesPerLine;   /* needed for ShMem lock */
     jboolean            xRequestSent;   /* true if x request is sent w/o XSync */
-    jint                pmSize;
+    jlong               pmSize;
 
     jboolean            usingShmPixmap;
     Drawable            pixmap;