diff --git a/src/solaris/native/sun/java2d/x11/X11SurfaceData.c b/src/solaris/native/sun/java2d/x11/X11SurfaceData.c index b4518af9d9246315fa12ba1ababc79d4d4a3c30e..7d1c058f31bf5432717601547e9a3c320ee088f6 100644 --- a/src/solaris/native/sun/java2d/x11/X11SurfaceData.c +++ b/src/solaris/native/sun/java2d/x11/X11SurfaceData.c @@ -78,6 +78,7 @@ static void X11SD_SwapBytes(X11SDOps *xsdo, XImage *img, int depth, int bpp); static XImage * X11SD_GetImage(JNIEnv *env, X11SDOps *xsdo, SurfaceDataBounds *bounds, jint lockFlags); +static int X11SD_GetBitmapPad(int pixelStride); extern jfieldID validID; @@ -438,11 +439,33 @@ jboolean XShared_initSurface(JNIEnv *env, X11SDOps *xsdo, jint depth, jint width xsdo->drawable = drawable; xsdo->isPixmap = JNI_FALSE; } else { + jboolean sizeIsInvalid = JNI_FALSE; + jlong scan = 0; + /* * width , height must be nonzero otherwise XCreatePixmap * generates BadValue in error_handler */ if (width <= 0 || height <= 0 || width > 32767 || height > 32767) { + sizeIsInvalid = JNI_TRUE; + } else { + XImage* tmpImg = NULL; + + AWT_LOCK(); + tmpImg = XCreateImage(awt_display, + xsdo->configData->awt_visInfo.visual, + depth, ZPixmap, 0, NULL, width, height, + X11SD_GetBitmapPad(xsdo->configData->pixelStride), 0); + if (tmpImg) { + scan = (jlong) tmpImg->bytes_per_line; + XDestroyImage(tmpImg); + tmpImg = NULL; + } + AWT_UNLOCK(); + JNU_CHECK_EXCEPTION_RETURN(env, JNI_FALSE); + } + + if (sizeIsInvalid || (scan * height > 0x7FFFFFFFL)) { JNU_ThrowOutOfMemoryError(env, "Can't create offscreen surface"); return JNI_FALSE; @@ -457,7 +480,7 @@ jboolean XShared_initSurface(JNIEnv *env, X11SDOps *xsdo, jint depth, jint width xsdo->pmHeight = height; #ifdef MITSHM - xsdo->shmPMData.pmSize = width * height * depth; + xsdo->shmPMData.pmSize = (jlong) width * height * depth; xsdo->shmPMData.pixelsReadThreshold = width * height / 8; if (forceSharedPixmaps) { AWT_LOCK(); @@ -560,7 +583,7 @@ XImage* X11SD_CreateSharedImage(X11SDOps *xsdo, return NULL; } shminfo->shmid = - shmget(IPC_PRIVATE, height * img->bytes_per_line, + shmget(IPC_PRIVATE, (size_t) height * img->bytes_per_line, IPC_CREAT|mitShmPermissionMask); if (shminfo->shmid < 0) { J2dRlsTraceLn1(J2D_TRACE_ERROR, @@ -622,7 +645,7 @@ XImage* X11SD_GetSharedImage(X11SDOps *xsdo, jint width, jint height, XSync(awt_display, False); retImage = cachedXImage; cachedXImage = (XImage *)NULL; - } else if (width * height * xsdo->depth > 0x10000) { + } else if ((jlong) width * height * xsdo->depth > 0x10000) { retImage = X11SD_CreateSharedImage(xsdo, width, height); } return retImage; @@ -978,7 +1001,7 @@ static void X11SD_GetRasInfo(JNIEnv *env, int scan = xpriv->img->bytes_per_line; xpriv->x = x; xpriv->y = y; - pRasInfo->rasBase = xpriv->img->data - x * mult - y * scan; + pRasInfo->rasBase = xpriv->img->data - x * mult - (intptr_t) y * scan; pRasInfo->pixelStride = mult; pRasInfo->pixelBitOffset = 0; pRasInfo->scanStride = scan; @@ -1140,8 +1163,8 @@ X11SD_FindClip(SurfaceDataBounds *b, SurfaceDataBounds *bounds, X11SDOps *xsdo) static void X11SD_SwapBytes(X11SDOps *xsdo, XImage * img, int depth, int bpp) { - int lengthInBytes = img->height * img->bytes_per_line; - int i; + jlong lengthInBytes = (jlong) img->height * img->bytes_per_line; + jlong i; switch (depth) { case 12: @@ -1214,7 +1237,7 @@ static XImage * X11SD_GetImage(JNIEnv *env, X11SDOps *xsdo, Drawable drawable; int depth = xsdo->depth; int mult = xsdo->configData->pixelStride; - int pad = (mult == 3) ? 32 : mult * 8; // pad must be 8, 16, or 32 + int pad = X11SD_GetBitmapPad(mult); jboolean readBits = lockFlags & SD_LOCK_NEED_PIXELS; x = bounds->x1; @@ -1280,7 +1303,7 @@ static XImage * X11SD_GetImage(JNIEnv *env, X11SDOps *xsdo, } scan = img->bytes_per_line; - img->data = malloc(h * scan); + img->data = malloc((size_t) h * scan); if (img->data == NULL) { XFree(img); return NULL; @@ -1315,7 +1338,7 @@ static XImage * X11SD_GetImage(JNIEnv *env, X11SDOps *xsdo, int i; img_addr = img->data + - (temp.y1 - y) * scan + (temp.x1 - x) * mult; + (intptr_t) (temp.y1 - y) * scan + (temp.x1 - x) * mult; temp_scan = temp_image->bytes_per_line; temp_addr = temp_image->data; bytes_to_copy = (temp.x2 - temp.x1) * mult; @@ -1349,7 +1372,7 @@ static XImage * X11SD_GetImage(JNIEnv *env, X11SDOps *xsdo, return NULL; } - img->data = malloc(h * img->bytes_per_line); + img->data = malloc((size_t) h * img->bytes_per_line); if (img->data == NULL) { XFree(img); return NULL; @@ -1532,6 +1555,11 @@ X11SD_ReleasePixmapWithBg(JNIEnv *env, X11SDOps *xsdo) #endif /* MITSHM */ } +static int X11SD_GetBitmapPad(int pixelStride) { + // pad must be 8, 16, or 32 + return (pixelStride == 3) ? 32 : pixelStride * 8; +} + #endif /* !HEADLESS */ /* diff --git a/src/solaris/native/sun/java2d/x11/X11SurfaceData.h b/src/solaris/native/sun/java2d/x11/X11SurfaceData.h index 91ad4626a4c51202d8e2bce625f63ed9ed5b4027..cc8ae6a9936cd0e7ec66c4f716f8e035a90230cc 100644 --- a/src/solaris/native/sun/java2d/x11/X11SurfaceData.h +++ b/src/solaris/native/sun/java2d/x11/X11SurfaceData.h @@ -81,7 +81,7 @@ typedef struct { XShmSegmentInfo *shmSegInfo; /* Shared Memory Segment Info */ jint bytesPerLine; /* needed for ShMem lock */ jboolean xRequestSent; /* true if x request is sent w/o XSync */ - jint pmSize; + jlong pmSize; jboolean usingShmPixmap; Drawable pixmap;