8001034: Memory management improvements

Reviewed-by: mschoene, prr, jgodinez

src/solaris/native/sun/awt/awt_GraphicsEnv.c

@@ -906,6 +906,8 @@ static jint canUseShmExt = UNSET_MITSHM;
 static jint canUseShmExtPixmaps = UNSET_MITSHM;
 static jboolean xshmAttachFailed = JNI_FALSE;
 
+extern int mitShmPermissionMask;
+
 int J2DXErrHandler(Display *display, XErrorEvent *xerr) {
     int ret = 0;
     if (xerr->minor_code == X_ShmAttach) {
@@ -944,7 +946,8 @@ void TryInitMITShm(JNIEnv *env, jint *shmExt, jint *shmPixmaps) {
      * we need to test that we can actually do XShmAttach.
      */
     if (XShmQueryExtension(awt_display)) {
-        shminfo.shmid = shmget(IPC_PRIVATE, 0x10000, IPC_CREAT|0777);
+        shminfo.shmid = shmget(IPC_PRIVATE, 0x10000,
+                               IPC_CREAT|mitShmPermissionMask);
         if (shminfo.shmid < 0) {
             AWT_UNLOCK();
             J2dRlsTraceLn1(J2D_TRACE_ERROR,

src/solaris/native/sun/awt/awt_GraphicsEnv.h

@@ -47,6 +47,9 @@
 #include <X11/extensions/shmproto.h>
 #endif
 
+#define MITSHM_PERM_COMMON (0666)
+#define MITSHM_PERM_OWNER  (0600)
+
 extern int XShmQueryExtension();
 
 void TryInitMITShm(JNIEnv *env, jint *shmExt, jint *shmPixmaps);

src/solaris/native/sun/java2d/x11/X11SurfaceData.c

@@ -89,6 +89,7 @@ static jclass xorCompClass;
 jint useMitShmExt = CANT_USE_MITSHM;
 jint useMitShmPixmaps = CANT_USE_MITSHM;
 jint forceSharedPixmaps = JNI_FALSE;
+int mitShmPermissionMask = MITSHM_PERM_OWNER;
 
 /* Cached shared image, one for all surface datas. */
 static XImage * cachedXImage;
@@ -119,6 +120,13 @@ jboolean XShared_initIDs(JNIEnv *env, jboolean allowShmPixmaps)
     if (getenv("NO_AWT_MITSHM") == NULL &&
         getenv("NO_J2D_MITSHM") == NULL) {
         char * force;
+        char * permission = getenv("J2D_MITSHM_PERMISSION");
+        if (permission != NULL) {
+            if (strcmp(permission, "common") == 0) {
+                mitShmPermissionMask = MITSHM_PERM_COMMON;
+            }
+        }
+
         TryInitMITShm(env, &useMitShmExt, &useMitShmPixmaps);
 
         if(allowShmPixmaps) {
@@ -536,7 +544,8 @@ XImage* X11SD_CreateSharedImage(X11SDOps *xsdo,
         return NULL;
     }
     shminfo->shmid =
-        shmget(IPC_PRIVATE, height * img->bytes_per_line, IPC_CREAT|0777);
+        shmget(IPC_PRIVATE, height * img->bytes_per_line,
+               IPC_CREAT|mitShmPermissionMask);
     if (shminfo->shmid < 0) {
         J2dRlsTraceLn1(J2D_TRACE_ERROR,
                        "X11SD_SetupSharedSegment shmget has failed: %s",