8171952: [macosx]...

8171952: [macosx] AWT_Modality/Automated/ModalExclusion/NoExclusion/ModelessDialog test fails as DummyButton on Dialog did not gain focus when clicked.
Reviewed-by: ssadetsky, serb

src/macosx/native/sun/awt/AWTWindow.m

@@ -326,10 +326,44 @@ AWT_ASSERT_APPKIT_THREAD;
     return [window isKindOfClass: [AWTWindow_Panel class]] || [window isKindOfClass: [AWTWindow_Normal class]];
 }
 
+// Retrieves the list of possible window layers (levels)
++ (NSArray*) getWindowLayers {
+    static NSArray *windowLayers;
+    static dispatch_once_t token;
+ 
+    // Initialize the list of possible window layers
+    dispatch_once(&token, ^{
+        // The layers are ordered from front to back, (i.e. the toppest one is the first)
+        windowLayers = [NSArray arrayWithObjects:
+                            [NSNumber numberWithInt:CGWindowLevelForKey(kCGPopUpMenuWindowLevelKey)],
+                            [NSNumber numberWithInt:CGWindowLevelForKey(kCGFloatingWindowLevelKey)],
+                            [NSNumber numberWithInt:CGWindowLevelForKey(kCGNormalWindowLevelKey)],
+                            nil
+                       ];
+        [windowLayers retain];
+    });
+    return windowLayers;
+}
+
+
 // returns id for the topmost window under mouse
 + (NSInteger) getTopmostWindowUnderMouseID {
     NSInteger result = -1;
-    
+
+    NSArray *windowLayers = [AWTWindow getWindowLayers];
+    // Looking for the window under mouse starting from the toppest layer
+    for (NSNumber *layer in windowLayers) {
+        result = [AWTWindow getTopmostWindowUnderMouseIDImpl:[layer integerValue]];
+        if (result != -1) {
+            break;
+        }
+    }
+    return result;
+}
+
++ (NSInteger) getTopmostWindowUnderMouseIDImpl:(NSInteger)windowLayer {
+    NSInteger result = -1;    
+
     NSRect screenRect = [[NSScreen mainScreen] frame];
     NSPoint nsMouseLocation = [NSEvent mouseLocation];
     CGPoint cgMouseLocation = CGPointMake(nsMouseLocation.x, screenRect.size.height - nsMouseLocation.y);
@@ -338,7 +372,7 @@ AWT_ASSERT_APPKIT_THREAD;
 
     for (NSDictionary *window in windows) {
         NSInteger layer = [[window objectForKey:(id)kCGWindowLayer] integerValue];
-        if (layer == 0) {
+        if (layer == windowLayer) {
             CGRect rect;
             CGRectMakeWithDictionaryRepresentation((CFDictionaryRef)[window objectForKey:(id)kCGWindowBounds], &rect);
             if (CGRectContainsPoint(rect, cgMouseLocation)) {