8017189: [macosx] AWT program menu disabled on Mac

Reviewed-by: leonidr, anthony

src/macosx/classes/sun/lwawt/macosx/CPlatformWindow.java

@@ -47,7 +47,7 @@ import com.apple.laf.ClientPropertyApplicator.Property;
 import com.sun.awt.AWTUtilities;
 
 public class CPlatformWindow extends CFRetainedResource implements PlatformWindow {
-    private native long nativeCreateNSWindow(long nsViewPtr, long styleBits, double x, double y, double w, double h);
+    private native long nativeCreateNSWindow(long nsViewPtr,long ownerPtr, long styleBits, double x, double y, double w, double h);
     private static native void nativeSetNSWindowStyleBits(long nsWindowPtr, int mask, int data);
     private static native void nativeSetNSWindowMenuBar(long nsWindowPtr, long menuBarPtr);
     private static native Insets nativeGetNSWindowInsets(long nsWindowPtr);
@@ -230,7 +230,8 @@ public class CPlatformWindow extends CFRetainedResource implements PlatformWindo
         contentView = createContentView();
         contentView.initialize(peer, responder);
 
-        final long nativeWindowPtr = nativeCreateNSWindow(contentView.getAWTView(), styleBits, 0, 0, 0, 0);
+        final long ownerPtr = owner != null ? owner.getNSWindowPtr() : 0L;
+        final long nativeWindowPtr = nativeCreateNSWindow(contentView.getAWTView(), ownerPtr, styleBits, 0, 0, 0, 0);
         setPtr(nativeWindowPtr);
 
         if (target instanceof javax.swing.RootPaneContainer) {

src/macosx/native/sun/awt/AWTWindow.h

@@ -44,6 +44,7 @@
     jint styleBits;
     BOOL isEnabled;
     NSWindow *nsWindow;
+    AWTWindow *ownerWindow;
 }
 
 // An instance of either AWTWindow_Normal or AWTWindow_Panel
@@ -51,12 +52,15 @@
 
 @property (nonatomic, retain) JNFWeakJObjectWrapper *javaPlatformWindow;
 @property (nonatomic, retain) CMenuBar *javaMenuBar;
+@property (nonatomic, retain) AWTWindow *ownerWindow;
 @property (nonatomic) NSSize javaMinSize;
 @property (nonatomic) NSSize javaMaxSize;
 @property (nonatomic) jint styleBits;
 @property (nonatomic) BOOL isEnabled;
 
+
 - (id) initWithPlatformWindow:(JNFWeakJObjectWrapper *)javaPlatformWindow
+                  ownerWindow:owner
                     styleBits:(jint)styleBits
                     frameRect:(NSRect)frameRect
                   contentView:(NSView *)contentView;

src/macosx/native/sun/awt/AWTWindow.m

@@ -120,6 +120,7 @@ AWT_NS_WINDOW_IMPLEMENTATION
 @synthesize javaMaxSize;
 @synthesize styleBits;
 @synthesize isEnabled;
+@synthesize ownerWindow;
 
 - (void) updateMinMaxSize:(BOOL)resizable {
     if (resizable) {
@@ -201,6 +202,7 @@ AWT_NS_WINDOW_IMPLEMENTATION
 }
 
 - (id) initWithPlatformWindow:(JNFWeakJObjectWrapper *)platformWindow
+                  ownerWindow:owner
                     styleBits:(jint)bits
                     frameRect:(NSRect)rect
                   contentView:(NSView *)view
@@ -245,6 +247,7 @@ AWT_ASSERT_APPKIT_THREAD;
     self.isEnabled = YES;
     self.javaPlatformWindow = platformWindow;
     self.styleBits = bits;
+    self.ownerWindow = owner;
     [self setPropertiesForStyleBits:styleBits mask:MASK(_METHOD_PROP_BITMASK)];
 
     return self;
@@ -350,7 +353,7 @@ AWT_ASSERT_APPKIT_THREAD;
     [self.javaPlatformWindow setJObject:nil withEnv:env];
 
     self.nsWindow = nil;
-
+    self.ownerWindow = nil;
     [super dealloc];
 }
 
@@ -539,11 +542,18 @@ AWT_ASSERT_APPKIT_THREAD;
 AWT_ASSERT_APPKIT_THREAD;
     [AWTToolkit eventCountPlusPlus];
     AWTWindow *opposite = [AWTWindow lastKeyWindow];
-    if (!IS(self.styleBits, IS_DIALOG)) {
-        [CMenuBar activate:self.javaMenuBar modallyDisabled:NO];
-    } else if ((opposite != NULL) && IS(self.styleBits, IS_MODAL)) {
-        [CMenuBar activate:opposite->javaMenuBar modallyDisabled:YES];        
+    
+    // Finds appropriate menubar in our hierarchy,
+    AWTWindow *awtWindow = self;
+    while (awtWindow.ownerWindow != nil) {
+        awtWindow = awtWindow.ownerWindow;
+    }
+    CMenuBar *menuBar = nil;
+    if ([awtWindow.nsWindow isVisible]){
+        menuBar = awtWindow.javaMenuBar;
     }
+    [CMenuBar activate:menuBar modallyDisabled:!awtWindow.isEnabled];
+
     [AWTWindow setLastKeyWindow:nil];
 
     [self _deliverWindowFocusEvent:YES oppositeWindow: opposite];
@@ -741,7 +751,7 @@ AWT_ASSERT_APPKIT_THREAD;
  * Signature: (JJIIII)J
  */
 JNIEXPORT jlong JNICALL Java_sun_lwawt_macosx_CPlatformWindow_nativeCreateNSWindow
-(JNIEnv *env, jobject obj, jlong contentViewPtr, jlong styleBits, jdouble x, jdouble y, jdouble w, jdouble h)
+(JNIEnv *env, jobject obj, jlong contentViewPtr, jlong ownerPtr, jlong styleBits, jdouble x, jdouble y, jdouble w, jdouble h)
 {
     __block AWTWindow *window = nil;
 
@@ -750,13 +760,14 @@ JNF_COCOA_ENTER(env);
     JNFWeakJObjectWrapper *platformWindow = [JNFWeakJObjectWrapper wrapperWithJObject:obj withEnv:env];
     NSView *contentView = OBJC(contentViewPtr);
     NSRect frameRect = NSMakeRect(x, y, w, h);
-
+    AWTWindow *owner = [OBJC(ownerPtr) delegate];
     [ThreadUtilities performOnMainThreadWaiting:YES block:^(){
 
         window = [[AWTWindow alloc] initWithPlatformWindow:platformWindow
-                                                  styleBits:styleBits
-                                                  frameRect:frameRect
-                                                contentView:contentView];
+                                               ownerWindow:owner
+                                                 styleBits:styleBits
+                                                 frameRect:frameRect
+                                               contentView:contentView];
         // the window is released is CPlatformWindow.nativeDispose()
 
         if (window) CFRetain(window.nsWindow);

src/macosx/native/sun/awt/CMenuBar.m

@@ -63,7 +63,7 @@ static BOOL sSetupHelpMenu = NO;
         if (excludingAppleMenu && ![currMenu isJavaMenu]) {
             continue;
         }
-
+        [currItem setSubmenu:nil];
         [theMainMenu removeItemAtIndex:index];
     }
 
@@ -154,7 +154,10 @@ static BOOL sSetupHelpMenu = NO;
     // Clean up extra items
     NSUInteger removedIndex, removedCount = [removedMenuArray count];
     for (removedIndex=removedCount; removedIndex > 0; removedIndex--) {
-        [theMainMenu removeItemAtIndex:[[removedMenuArray objectAtIndex:(removedIndex-1)] integerValue]];
+        NSUInteger index = [[removedMenuArray objectAtIndex:(removedIndex-1)] integerValue];
+        NSMenuItem *currItem = [theMainMenu itemAtIndex:index];
+        [currItem setSubmenu:nil];
+        [theMainMenu removeItemAtIndex:index];
     }
 
     i = cmenuIndex;