8150954: Taking screenshots on x11 composite desktop produce wrong result

Summary: The AWT Robot X11 code that takes screenshots uses the default root window, which may not contain the final composited desktop.
Reviewed-by: alexsch, ssadetsky, prr, dbuck

make/mapfiles/libawt_xawt/mapfile-vers

@@ -158,6 +158,7 @@ SUNWprivate_1.1 {
         Java_sun_awt_X11_XRobotPeer_mouseReleaseImpl;
         Java_sun_awt_X11_XRobotPeer_mouseWheelImpl;
         Java_sun_awt_X11_XRobotPeer_setup;
+        Java_sun_awt_X11_XRobotPeer_loadNativeLibraries;
         Java_sun_awt_X11_XToolkit_getNumberOfButtonsImpl;
         Java_java_awt_Component_initIDs;
         Java_java_awt_Container_initIDs;

src/solaris/classes/sun/awt/X11/XRobotPeer.java

@@ -34,6 +34,10 @@ import sun.awt.X11GraphicsConfig;
 
 class XRobotPeer implements RobotPeer {
 
+    static {
+        loadNativeLibraries();
+    }
+
     private X11GraphicsConfig   xgc = null;
     /*
      * native implementation uses some static shared data (pipes, processes)
@@ -98,4 +102,5 @@ class XRobotPeer implements RobotPeer {
     private static native synchronized void keyReleaseImpl(int keycode);
 
     private static native synchronized void getRGBPixelsImpl(X11GraphicsConfig xgc, int x, int y, int width, int height, int pixelArray[]);
+    private static native void loadNativeLibraries();
 }

src/solaris/native/sun/awt/awt_Robot.c

@@ -27,6 +27,9 @@
     #error This file should not be included in headless library
 #endif
 
+#include "jvm_md.h"
+#include <dlfcn.h>
+
 #include "awt_p.h"
 #include "awt_GraphicsEnv.h"
 #define XK_MISCELLANY
@@ -49,11 +52,46 @@
 #include <sys/socket.h>
 #endif
 
+static Bool   (*compositeQueryExtension)   (Display*, int*, int*);
+static Status (*compositeQueryVersion)     (Display*, int*, int*);
+static Window (*compositeGetOverlayWindow) (Display *, Window);
+
 extern struct X11GraphicsConfigIDs x11GraphicsConfigIDs;
 
 static jint * masks;
 static jint num_buttons;
 
+static void *xCompositeHandle;
+
+static const char* XCOMPOSITE = JNI_LIB_NAME("Xcomposite");
+static const char* XCOMPOSITE_VERSIONED = VERSIONED_JNI_LIB_NAME("Xcomposite", "1");
+
+static Bool checkXCompositeFunctions(void) {
+    return (compositeQueryExtension   != NULL   &&
+            compositeQueryVersion     != NULL   &&
+            compositeGetOverlayWindow != NULL);
+}
+
+static void initXCompositeFunctions(void) {
+
+    if (xCompositeHandle == NULL) {
+        xCompositeHandle = dlopen(XCOMPOSITE, RTLD_LAZY | RTLD_GLOBAL);
+        if (xCompositeHandle == NULL) {
+            xCompositeHandle = dlopen(XCOMPOSITE_VERSIONED, RTLD_LAZY | RTLD_GLOBAL);
+        }
+    }
+    //*(void **)(&asyncGetCallTraceFunction)
+    if (xCompositeHandle != NULL) {
+        *(void **)(&compositeQueryExtension) = dlsym(xCompositeHandle, "XCompositeQueryExtension");
+        *(void **)(&compositeQueryVersion) = dlsym(xCompositeHandle, "XCompositeQueryVersion");
+        *(void **)(&compositeGetOverlayWindow) = dlsym(xCompositeHandle, "XCompositeGetOverlayWindow");
+    }
+
+    if (xCompositeHandle && !checkXCompositeFunctions()) {
+        dlclose(xCompositeHandle);
+    }
+}
+
 static int32_t isXTestAvailable() {
     int32_t major_opcode, first_event, first_error;
     int32_t  event_basep, error_basep, majorp, minorp;
@@ -88,6 +126,35 @@ static int32_t isXTestAvailable() {
     return isXTestAvailable;
 }
 
+static Bool hasXCompositeOverlayExtension(Display *display) {
+
+    int xoverlay = False;
+    int eventBase, errorBase;
+    if (checkXCompositeFunctions() &&
+        compositeQueryExtension(display, &eventBase, &errorBase))
+    {
+        int major = 0;
+        int minor = 0;
+
+        compositeQueryVersion(display, &major, &minor);
+        if (major > 0 || minor >= 3) {
+            xoverlay = True;
+        }
+    }
+
+    return xoverlay;
+}
+
+static jboolean isXCompositeDisplay(Display *display, int screenNumber) {
+
+    char NET_WM_CM_Sn[25];
+    snprintf(NET_WM_CM_Sn, sizeof(NET_WM_CM_Sn), "_NET_WM_CM_S%d\0", screenNumber);
+
+    Atom managerSelection = XInternAtom(display, NET_WM_CM_Sn, 0);
+    Window owner = XGetSelectionOwner(display, managerSelection);
+
+    return owner != 0;
+}
 
 static XImage *getWindowImage(Display * display, Window window,
                               int32_t x, int32_t y,
@@ -232,6 +299,12 @@ Java_sun_awt_X11_XRobotPeer_getRGBPixelsImpl( JNIEnv *env,
     DASSERT(adata != NULL);
 
     rootWindow = XRootWindow(awt_display, adata->awt_visInfo.screen);
+    if (hasXCompositeOverlayExtension(awt_display) &&
+        isXCompositeDisplay(awt_display, adata->awt_visInfo.screen))
+    {
+        rootWindow = compositeGetOverlayWindow(awt_display, rootWindow);
+    }
+
     image = getWindowImage(awt_display, rootWindow, x, y, width, height);
 
     /* Array to use to crunch around the pixel values */
@@ -412,3 +485,8 @@ Java_sun_awt_X11_XRobotPeer_mouseWheelImpl (JNIEnv *env,
 
     AWT_UNLOCK();
 }
+
+JNIEXPORT void JNICALL
+Java_sun_awt_X11_XRobotPeer_loadNativeLibraries (JNIEnv *env, jclass cls) {
+    initXCompositeFunctions();
+}