8015730: PIT: On Linux, OGL=true and fbobject=false leads to deadlock or infinite loop

Reviewed-by: art, anthony

src/solaris/classes/sun/awt/X11/XErrorHandlerUtil.java

@@ -105,9 +105,15 @@ public final class XErrorHandlerUtil {
      * Unsets a current synthetic error handler. Must be called with the acquired AWT lock.
      */
     public static void RESTORE_XERROR_HANDLER() {
+        RESTORE_XERROR_HANDLER(true);
+    }
+
+    private static void RESTORE_XERROR_HANDLER(boolean doXSync) {
         // Wait until all requests are processed by the X server
         // and only then uninstall the error handler.
-        XSync();
+        if (doXSync) {
+            XSync();
+        }
         current_error_handler = null;
     }
 

src/solaris/native/sun/awt/awt_util.h

@@ -46,11 +46,11 @@
 
 /*
  * Expected types of arguments of the macro.
- * (JNIEnv*)
+ * (JNIEnv*, jboolean)
  */
-#define RESTORE_XERROR_HANDLER(env) do {                                                          \
+#define RESTORE_XERROR_HANDLER(env, doXSync) do {                                                 \
     JNU_CallStaticMethodByName(env, NULL, "sun/awt/X11/XErrorHandlerUtil",                        \
-        "RESTORE_XERROR_HANDLER", "()V");                                                         \
+        "RESTORE_XERROR_HANDLER", "(Z)V", doXSync);                                               \
 } while (0)
 
 /*
@@ -64,8 +64,18 @@
     do {                                                                                          \
         code;                                                                                     \
     } while (0);                                                                                  \
-    RESTORE_XERROR_HANDLER(env);                                                                  \
+    RESTORE_XERROR_HANDLER(env, JNI_TRUE);                                                        \
     if (handlerHasFlag == JNI_TRUE) {                                                             \
+        GET_HANDLER_ERROR_OCCURRED_FLAG(env, handlerRef, errorOccurredFlag);                      \
+    }                                                                                             \
+} while (0)
+
+/*
+ * Expected types of arguments of the macro.
+ * (JNIEnv*, jobject, jboolean)
+ */
+#define GET_HANDLER_ERROR_OCCURRED_FLAG(env, handlerRef, errorOccurredFlag) do {                  \
+    if (handlerRef != NULL) {                                                                     \
         errorOccurredFlag = JNU_CallMethodByName(env, NULL, handlerRef, "getErrorOccurredFlag",   \
             "()Z").z;                                                                             \
     }                                                                                             \

src/solaris/native/sun/java2d/opengl/GLXSurfaceData.c

@@ -392,10 +392,12 @@ Java_sun_java2d_opengl_GLXSurfaceData_initPbuffer
     attrlist[3] = height;
 
     errorOccurredFlag = JNI_FALSE;
-    EXEC_WITH_XERROR_HANDLER(env, "sun/awt/X11/XErrorHandler$GLXBadAllocHandler",
-        "()Lsun/awt/X11/XErrorHandler$GLXBadAllocHandler;", JNI_TRUE,
-        errorHandlerRef, errorOccurredFlag,
-        pbuffer = j2d_glXCreatePbuffer(awt_display, glxinfo->fbconfig, attrlist));
+    WITH_XERROR_HANDLER(env, "sun/awt/X11/XErrorHandler$GLXBadAllocHandler",
+        "()Lsun/awt/X11/XErrorHandler$GLXBadAllocHandler;", JNI_TRUE, errorHandlerRef);
+    pbuffer = j2d_glXCreatePbuffer(awt_display, glxinfo->fbconfig, attrlist);
+    XSync(awt_display, False);
+    RESTORE_XERROR_HANDLER(env, JNI_FALSE);
+    GET_HANDLER_ERROR_OCCURRED_FLAG(env, errorHandlerRef, errorOccurredFlag);
 
     if ((pbuffer == 0) || errorOccurredFlag) {
         J2dRlsTraceLn(J2D_TRACE_ERROR,