8048549: [macosx] Disable usage of system menu bar if AWT is embedded in FX

Reviewed-by: serb, anthony

src/macosx/classes/com/apple/laf/AquaMenuBarUI.java

@@ -26,11 +26,14 @@
 package com.apple.laf;
 
 import java.awt.*;
+import java.security.AccessController;
 
 import javax.swing.*;
 import javax.swing.plaf.ComponentUI;
 import javax.swing.plaf.basic.BasicMenuBarUI;
 
+import sun.lwawt.macosx.LWCToolkit;
+import sun.security.action.GetBooleanAction;
 import sun.security.action.GetPropertyAction;
 
 // MenuBar implementation for Mac L&F
@@ -131,28 +134,20 @@ public class AquaMenuBarUI extends BasicMenuBarUI implements ScreenMenuBarProvid
     ScreenMenuBar fScreenMenuBar;
     boolean useScreenMenuBar = getScreenMenuBarProperty();
 
-    private static String getPrivSysProp(final String propName) {
-        return java.security.AccessController.doPrivileged(new GetPropertyAction(propName));
-    }
-
     static boolean getScreenMenuBarProperty() {
-        final String props[] = new String[]{""};
-
-        boolean useScreenMenuBar = false;
-        try {
-            props[0] = getPrivSysProp(AquaLookAndFeel.sPropertyPrefix + "useScreenMenuBar");
-
-            if (props[0] != null && props[0].equals("true")) useScreenMenuBar = true;
-            else {
-                props[0] = getPrivSysProp(AquaLookAndFeel.sOldPropertyPrefix + "useScreenMenuBar");
-
-                if (props[0] != null && props[0].equals("true")) {
-                    System.err.println(AquaLookAndFeel.sOldPropertyPrefix + "useScreenMenuBar has been deprecated. Please switch to " + AquaLookAndFeel.sPropertyPrefix + "useScreenMenuBar");
-                    useScreenMenuBar = true;
+        // Do not allow AWT to set the screen menu bar if it's embedded in another UI toolkit
+        if (LWCToolkit.isEmbedded()) return false;
+        if (AccessController.doPrivileged(
+                new GetBooleanAction(AquaLookAndFeel.sPropertyPrefix + "useScreenMenuBar"))) {
+            return true;
         }
+        if (AccessController.doPrivileged(
+                new GetBooleanAction(AquaLookAndFeel.sOldPropertyPrefix + "useScreenMenuBar"))) {
+                System.err.println(AquaLookAndFeel.sOldPropertyPrefix +
+                        "useScreenMenuBar has been deprecated. Please switch to " +
+                        AquaLookAndFeel.sPropertyPrefix + "useScreenMenuBar");
+                return true;
         }
-        } catch(final Throwable t) { };
-
-        return useScreenMenuBar;
+        return false;
     }
 }

src/macosx/classes/sun/lwawt/macosx/LWCToolkit.java

@@ -784,6 +784,13 @@ public final class LWCToolkit extends LWToolkit {
      */
     native boolean isApplicationActive();
 
+    /**
+     * Returns true if AWT toolkit is embedded, false otherwise.
+     *
+     * @return true if AWT toolkit is embedded, false otherwise
+     */
+    public static native boolean isEmbedded();
+
     /************************
      * Native methods section
      ************************/

src/macosx/native/sun/awt/LWCToolkit.m

@@ -452,3 +452,14 @@ Java_sun_font_FontManager_populateFontFileNameMap
 
 }
 
+/*
+ * Class:     sun_lwawt_macosx_LWCToolkit
+ * Method:    isEmbedded
+ * Signature: ()Z
+ */
+JNIEXPORT jboolean JNICALL
+Java_sun_lwawt_macosx_LWCToolkit_isEmbedded
+(JNIEnv *env, jclass klass) {
+    return [ThreadUtilities isAWTEmbedded] ? JNI_TRUE : JNI_FALSE;
+}
+