8025775: JNI warnings in TryXShmAttach

Reviewed-by: art, anthony

src/solaris/classes/sun/awt/X11/XConstants.java

@@ -674,9 +674,4 @@ final public class XConstants {
     public static final long XkbModifierMapMask = (1L<<2);
     public static final long XkbVirtualModsMask = (1L<<6); //server map
 
-    /*****************************************************************
-     * X SHARED MEMORY EXTENSION FUNCTIONS
-     *****************************************************************/
-
-    public static final int X_ShmAttach = 1;
 }

src/solaris/classes/sun/awt/X11/XErrorHandler.java

@@ -42,29 +42,6 @@ public abstract class XErrorHandler {
         }
     }
 
-    /**
-     * This is a base synthetic error handler containing a boolean flag which allows
-     * to show that an error is handled or not.
-     */
-    public static class XErrorHandlerWithFlag extends XBaseErrorHandler {
-        private volatile boolean errorOccurred = false;
-
-        public boolean getErrorOccurredFlag() {
-            return errorOccurred;
-        }
-
-        /**
-         * Sets an internal boolean flag to a particular value. Should be always called with
-         * <code>false</code> value of the parameter <code>errorOccurred</code> before this
-         * error handler is set as current.
-         * @param errorOccurred <code>true</code> to indicate that an error was handled,
-         *     <code>false</code> to reset the internal boolean flag
-         */
-        public void setErrorOccurredFlag(boolean errorOccurred) {
-            this.errorOccurred = errorOccurred;
-        }
-    }
-
     /*
      * Instead of validating window id, we simply call XGetWindowProperty,
      * but temporary install this function as the error handler to ignore
@@ -99,51 +76,4 @@ public abstract class XErrorHandler {
             return theInstance;
         }
     }
-
-    /**
-     * This is a synthetic error handler for errors generated by the native function
-     * <code>XShmAttach</code>. If an error is handled, an internal boolean flag of the
-     * handler is set to <code>true</code>.
-     */
-    public static final class XShmAttachHandler extends XErrorHandlerWithFlag {
-        private XShmAttachHandler() {}
-
-        @Override
-        public int handleError(long display, XErrorEvent err) {
-            if (err.get_minor_code() == XConstants.X_ShmAttach) {
-                setErrorOccurredFlag(true);
-                return 0;
-            }
-            return super.handleError(display, err);
-        }
-
-        // Shared instance
-        private static XShmAttachHandler theInstance = new XShmAttachHandler();
-        public static XShmAttachHandler getInstance() {
-            return theInstance;
-        }
-    }
-
-    /**
-     * This is a synthetic error handler for <code>BadAlloc</code> errors generated by the
-     * native <code>glX*</code> functions. Its internal boolean flag is set to <code>true</code>,
-     * if an error is handled.
-     */
-    public static final class GLXBadAllocHandler extends XErrorHandlerWithFlag {
-        private GLXBadAllocHandler() {}
-
-        @Override
-        public int handleError(long display, XErrorEvent err) {
-            if (err.get_error_code() == XConstants.BadAlloc) {
-                setErrorOccurredFlag(true);
-                return 0;
-            }
-            return super.handleError(display, err);
-        }
-
-        private static GLXBadAllocHandler theInstance = new GLXBadAllocHandler();
-        public static GLXBadAllocHandler getInstance() {
-            return theInstance;
-        }
-    }
 }

src/solaris/classes/sun/awt/X11/XErrorHandlerUtil.java

@@ -42,7 +42,7 @@ public final class XErrorHandlerUtil {
     private static long display;
 
     /**
-     * Error handler at the moment of <code>XErrorHandlerUtil</code> initialization.
+     * Error handler at the moment of {@code XErrorHandlerUtil} initialization.
      */
     private static long saved_error_handler;
 
@@ -63,7 +63,7 @@ public final class XErrorHandlerUtil {
         new GetBooleanAction("sun.awt.noisyerrorhandler"));
 
     /**
-     * The flag indicating that <code>init</code> was called already.
+     * The flag indicating that {@code init} was called already.
      */
     private static boolean initPassed;
 
@@ -73,9 +73,9 @@ public final class XErrorHandlerUtil {
     private XErrorHandlerUtil() {}
 
     /**
-     * Sets the toolkit global error handler, stores the connection to X11 server, which
-     * will be used during an error handling process. This method is called once from
-     * <code>awt_init_Display</code> function defined in <code>awt_GraphicsEnv.c</code>
+     * Sets the toolkit global error handler, stores the connection to X11 server,
+     * which will be used during an error handling process. This method is called
+     * once from {@code awt_init_Display} function defined in {@code awt_GraphicsEnv.c}
      * file immediately after the connection to X11 window server is opened.
      * @param display the connection to X11 server which should be stored
      */
@@ -109,9 +109,9 @@ public final class XErrorHandlerUtil {
     }
 
     private static void RESTORE_XERROR_HANDLER(boolean doXSync) {
+        if (doXSync) {
             // Wait until all requests are processed by the X server
             // and only then uninstall the error handler.
-        if (doXSync) {
             XSync();
         }
         current_error_handler = null;

src/solaris/native/sun/awt/awt_GraphicsEnv.c

@@ -906,6 +906,20 @@ Java_sun_awt_X11GraphicsDevice_getDisplay(JNIEnv *env, jobject this)
 
 static jint canUseShmExt = UNSET_MITSHM;
 static jint canUseShmExtPixmaps = UNSET_MITSHM;
+static jboolean xshmAttachFailed = JNI_FALSE;
+
+int XShmAttachXErrHandler(Display *display, XErrorEvent *xerr) {
+    if (xerr->minor_code == X_ShmAttach) {
+        xshmAttachFailed = JNI_TRUE;
+    }
+    return 0;
+}
+jboolean isXShmAttachFailed() {
+    return xshmAttachFailed;
+}
+void resetXShmAttachFailed() {
+    xshmAttachFailed = JNI_FALSE;
+}
 
 extern int mitShmPermissionMask;
 
@@ -913,7 +927,6 @@ void TryInitMITShm(JNIEnv *env, jint *shmExt, jint *shmPixmaps) {
     XShmSegmentInfo shminfo;
     int XShmMajor, XShmMinor;
     int a, b, c;
-    jboolean xShmAttachResult;
 
     AWT_LOCK();
     if (canUseShmExt != UNSET_MITSHM) {
@@ -957,14 +970,21 @@ void TryInitMITShm(JNIEnv *env, jint *shmExt, jint *shmPixmaps) {
         }
         shminfo.readOnly = True;
 
-        xShmAttachResult = TryXShmAttach(env, awt_display, &shminfo);
+        resetXShmAttachFailed();
+        /**
+         * The J2DXErrHandler handler will set xshmAttachFailed
+         * to JNI_TRUE if any Shm error has occured.
+         */
+        EXEC_WITH_XERROR_HANDLER(XShmAttachXErrHandler,
+                                 XShmAttach(awt_display, &shminfo));
+
         /**
          * Get rid of the id now to reduce chances of leaking
          * system resources.
          */
         shmctl(shminfo.shmid, IPC_RMID, 0);
 
-        if (xShmAttachResult == JNI_TRUE) {
+        if (isXShmAttachFailed() == JNI_FALSE) {
             canUseShmExt = CAN_USE_MITSHM;
             /* check if we can use shared pixmaps */
             XShmQueryVersion(awt_display, &XShmMajor, &XShmMinor,
@@ -979,23 +999,6 @@ void TryInitMITShm(JNIEnv *env, jint *shmExt, jint *shmPixmaps) {
     }
     AWT_UNLOCK();
 }
-
-/*
- * Must be called with the acquired AWT lock.
- */
-jboolean TryXShmAttach(JNIEnv *env, Display *display, XShmSegmentInfo *shminfo) {
-    jboolean errorOccurredFlag = JNI_FALSE;
-    jobject errorHandlerRef;
-
-    /*
-     * XShmAttachHandler will set its internal flag to JNI_TRUE, if any Shm error occurs.
-     */
-    EXEC_WITH_XERROR_HANDLER(env, "sun/awt/X11/XErrorHandler$XShmAttachHandler",
-        "()Lsun/awt/X11/XErrorHandler$XShmAttachHandler;", JNI_TRUE,
-        errorHandlerRef, errorOccurredFlag,
-        XShmAttach(display, shminfo));
-    return errorOccurredFlag == JNI_FALSE ? JNI_TRUE : JNI_FALSE;
-}
 #endif /* MITSHM */
 
 /*

src/solaris/native/sun/awt/awt_GraphicsEnv.h

@@ -53,7 +53,8 @@
 extern int XShmQueryExtension();
 
 void TryInitMITShm(JNIEnv *env, jint *shmExt, jint *shmPixmaps);
-jboolean TryXShmAttach(JNIEnv *env, Display *display, XShmSegmentInfo *shminfo);
+void resetXShmAttachFailed();
+jboolean isXShmAttachFailed();
 
 #endif /* MITSHM */
 

src/solaris/native/sun/awt/awt_util.c

@@ -41,6 +41,11 @@
 
 #include "java_awt_event_MouseWheelEvent.h"
 
+/*
+ * Called by "ToolkitErrorHandler" function in "XlibWrapper.c" file.
+ */
+XErrorHandler current_native_xerror_handler = NULL;
+
 extern jint getModifiers(uint32_t state, jint button, jint keyCode);
 extern jint getButton(uint32_t button);
 

src/solaris/native/sun/awt/awt_util.h

@@ -29,57 +29,29 @@
 #ifndef HEADLESS
 #include "gdefs.h"
 
-/*
- * Expected types of arguments of the macro.
- * (JNIEnv*, const char*, const char*, jboolean, jobject)
- */
-#define WITH_XERROR_HANDLER(env, handlerClassName, getInstanceSignature,                          \
-                            handlerHasFlag, handlerRef) do {                                      \
-    handlerRef = JNU_CallStaticMethodByName(env, NULL, handlerClassName, "getInstance",           \
-        getInstanceSignature).l;                                                                  \
-    if (handlerHasFlag == JNI_TRUE) {                                                             \
-        JNU_CallMethodByName(env, NULL, handlerRef, "setErrorOccurredFlag", "(Z)V", JNI_FALSE);   \
-    }                                                                                             \
-    JNU_CallStaticMethodByName(env, NULL, "sun/awt/X11/XErrorHandlerUtil", "WITH_XERROR_HANDLER", \
-        "(Lsun/awt/X11/XErrorHandler;)V", handlerRef);                                            \
+#define WITH_XERROR_HANDLER(f) do {             \
+    XSync(awt_display, False);                  \
+    current_native_xerror_handler = (f);        \
 } while (0)
 
-/*
- * Expected types of arguments of the macro.
- * (JNIEnv*, jboolean)
- */
-#define RESTORE_XERROR_HANDLER(env, doXSync) do {                                                 \
-    JNU_CallStaticMethodByName(env, NULL, "sun/awt/X11/XErrorHandlerUtil",                        \
-        "RESTORE_XERROR_HANDLER", "(Z)V", doXSync);                                               \
+#define RESTORE_XERROR_HANDLER do {             \
+    XSync(awt_display, False);                  \
+    current_native_xerror_handler = NULL;       \
 } while (0)
 
-/*
- * Expected types of arguments of the macro.
- * (JNIEnv*, const char*, const char*, jboolean, jobject, jboolean, No type - C expression)
- */
-#define EXEC_WITH_XERROR_HANDLER(env, handlerClassName, getInstanceSignature, handlerHasFlag,     \
-                                 handlerRef, errorOccurredFlag, code) do {                        \
-    handlerRef = NULL;                                                                            \
-    WITH_XERROR_HANDLER(env, handlerClassName, getInstanceSignature, handlerHasFlag, handlerRef); \
+#define EXEC_WITH_XERROR_HANDLER(f, code) do {  \
+    WITH_XERROR_HANDLER(f);                     \
     do {                                        \
         code;                                   \
     } while (0);                                \
-    RESTORE_XERROR_HANDLER(env, JNI_TRUE);                                                        \
-    if (handlerHasFlag == JNI_TRUE) {                                                             \
-        GET_HANDLER_ERROR_OCCURRED_FLAG(env, handlerRef, errorOccurredFlag);                      \
-    }                                                                                             \
+    RESTORE_XERROR_HANDLER;                     \
 } while (0)
 
 /*
- * Expected types of arguments of the macro.
- * (JNIEnv*, jobject, jboolean)
+ * Called by "ToolkitErrorHandler" function in "XlibWrapper.c" file.
  */
-#define GET_HANDLER_ERROR_OCCURRED_FLAG(env, handlerRef, errorOccurredFlag) do {                  \
-    if (handlerRef != NULL) {                                                                     \
-        errorOccurredFlag = JNU_CallMethodByName(env, NULL, handlerRef, "getErrorOccurredFlag",   \
-            "()Z").z;                                                                             \
-    }                                                                                             \
-} while (0)
+extern XErrorHandler current_native_xerror_handler;
+
 #endif /* !HEADLESS */
 
 #ifndef INTERSECTS

src/solaris/native/sun/java2d/opengl/GLXSurfaceData.c

@@ -48,6 +48,8 @@ extern DisposeFunc    OGLSD_Dispose;
 extern void
     OGLSD_SetNativeDimensions(JNIEnv *env, OGLSDOps *oglsdo, jint w, jint h);
 
+jboolean surfaceCreationFailed = JNI_FALSE;
+
 #endif /* !HEADLESS */
 
 JNIEXPORT void JNICALL
@@ -347,6 +349,15 @@ OGLSD_InitOGLWindow(JNIEnv *env, OGLSDOps *oglsdo)
     return JNI_TRUE;
 }
 
+static int
+GLXSD_BadAllocXErrHandler(Display *display, XErrorEvent *xerr)
+{
+    if (xerr->error_code == BadAlloc) {
+        surfaceCreationFailed = JNI_TRUE;
+    }
+    return 0;
+}
+
 JNIEXPORT jboolean JNICALL
 Java_sun_java2d_opengl_GLXSurfaceData_initPbuffer
     (JNIEnv *env, jobject glxsd,
@@ -362,8 +373,6 @@ Java_sun_java2d_opengl_GLXSurfaceData_initPbuffer
     int attrlist[] = {GLX_PBUFFER_WIDTH, 0,
                       GLX_PBUFFER_HEIGHT, 0,
                       GLX_PRESERVED_CONTENTS, GL_FALSE, 0};
-    jboolean errorOccurredFlag;
-    jobject errorHandlerRef;
 
     J2dTraceLn3(J2D_TRACE_INFO,
                 "GLXSurfaceData_initPbuffer: w=%d h=%d opq=%d",
@@ -391,15 +400,12 @@ Java_sun_java2d_opengl_GLXSurfaceData_initPbuffer
     attrlist[1] = width;
     attrlist[3] = height;
 
-    errorOccurredFlag = JNI_FALSE;
-    WITH_XERROR_HANDLER(env, "sun/awt/X11/XErrorHandler$GLXBadAllocHandler",
-        "()Lsun/awt/X11/XErrorHandler$GLXBadAllocHandler;", JNI_TRUE, errorHandlerRef);
-    pbuffer = j2d_glXCreatePbuffer(awt_display, glxinfo->fbconfig, attrlist);
-    XSync(awt_display, False);
-    RESTORE_XERROR_HANDLER(env, JNI_FALSE);
-    GET_HANDLER_ERROR_OCCURRED_FLAG(env, errorHandlerRef, errorOccurredFlag);
-
-    if ((pbuffer == 0) || errorOccurredFlag) {
+    surfaceCreationFailed = JNI_FALSE;
+    EXEC_WITH_XERROR_HANDLER(
+        GLXSD_BadAllocXErrHandler,
+        pbuffer = j2d_glXCreatePbuffer(awt_display,
+                                       glxinfo->fbconfig, attrlist));
+    if ((pbuffer == 0) || surfaceCreationFailed) {
         J2dRlsTraceLn(J2D_TRACE_ERROR,
             "GLXSurfaceData_initPbuffer: could not create glx pbuffer");
         return JNI_FALSE;

src/solaris/native/sun/java2d/x11/X11SurfaceData.c

@@ -65,6 +65,7 @@ static UnlockFunc X11SD_Unlock;
 static DisposeFunc X11SD_Dispose;
 static GetPixmapBgFunc X11SD_GetPixmapWithBg;
 static ReleasePixmapBgFunc X11SD_ReleasePixmapWithBg;
+extern int XShmAttachXErrHandler(Display *display, XErrorEvent *xerr);
 extern AwtGraphicsConfigDataPtr
     getGraphicsConfigFromComponentPeer(JNIEnv *env, jobject this);
 extern struct X11GraphicsConfigIDs x11GraphicsConfigIDs;
@@ -532,8 +533,6 @@ XImage* X11SD_CreateSharedImage(X11SDOps *xsdo,
 {
     XImage *img = NULL;
     XShmSegmentInfo *shminfo;
-    JNIEnv* env;
-    jboolean xShmAttachResult;
 
     shminfo = malloc(sizeof(XShmSegmentInfo));
     if (shminfo == NULL) {
@@ -573,8 +572,9 @@ XImage* X11SD_CreateSharedImage(X11SDOps *xsdo,
 
     shminfo->readOnly = False;
 
-    env = (JNIEnv*)JNU_GetEnv(jvm, JNI_VERSION_1_2);
-    xShmAttachResult = TryXShmAttach(env, awt_display, shminfo);
+    resetXShmAttachFailed();
+    EXEC_WITH_XERROR_HANDLER(XShmAttachXErrHandler,
+                             XShmAttach(awt_display, shminfo));
 
     /*
      * Once the XSync round trip has finished then we
@@ -583,7 +583,7 @@ XImage* X11SD_CreateSharedImage(X11SDOps *xsdo,
      */
     shmctl(shminfo->shmid, IPC_RMID, 0);
 
-    if (xShmAttachResult == JNI_FALSE) {
+    if (isXShmAttachFailed() == JNI_TRUE) {
         J2dRlsTraceLn1(J2D_TRACE_ERROR,
                        "X11SD_SetupSharedSegment XShmAttach has failed: %s",
                        strerror(errno));

src/solaris/native/sun/xawt/XlibWrapper.c

@@ -41,6 +41,7 @@
 #include <sizecalc.h>
 
 #include <awt.h>
+#include <awt_util.h>
 #include <jvm.h>
 
 #include <Region.h>
@@ -1266,6 +1267,10 @@ JNIEXPORT jboolean JNICALL Java_sun_awt_X11_XlibWrapper_IsKanaKeyboard
 JavaVM* jvm = NULL;
 static int ToolkitErrorHandler(Display * dpy, XErrorEvent * event) {
     JNIEnv * env;
+    // First call the native synthetic error handler declared in "awt_util.h" file.
+    if (current_native_xerror_handler != NULL) {
+        current_native_xerror_handler(dpy, event);
+    }
     if (jvm != NULL) {
         env = (JNIEnv *)JNU_GetEnv(jvm, JNI_VERSION_1_2);
         if (env) {