8025673: [macosx] Disable X11 AWT toolkit

Summary: Disable but not completely remove the XAWT and headless toolkits on Mac OS X
Reviewed-by: anthony, art, ihse, erikj
Contributed-by: david.dehaven@oracle.com

makefiles/CompileJavaClasses.gmk

@@ -96,11 +96,7 @@ ifneq ($(OPENJDK_TARGET_OS), solaris)
   EXCLUDES += com/oracle/security
 endif
 
-# In the old build, this isn't excluded on macosx, even though it probably
-# should be.
-ifneq ($(OPENJDK_TARGET_OS), macosx)
-  EXFILES += WrapperGenerator.java
-endif
+EXFILES += WrapperGenerator.java
 
 ifneq ($(OPENJDK_TARGET_OS), windows)
   # Exclude Window security related files in src/share/classes
@@ -169,7 +165,11 @@ ifeq (, $(filter $(OPENJDK_TARGET_OS), windows macosx))
   EXFILES += sun/awt/AWTCharset.java
 endif
 
-ifneq ($(OPENJDK_TARGET_OS), macosx)
+ifeq ($(OPENJDK_TARGET_OS), macosx)
+  # exclude all X11 on Mac, we can't exclude some like below or we'll have compilation errors
+  EXCLUDES += sun/awt/X11
+else
+  # TBD: figure out how to eliminate this long list
   EXFILES += sun/awt/X11/ScreenFormat.java \
       sun/awt/X11/XArc.java \
       sun/awt/X11/XChar2b.java \

makefiles/GenerateSources.gmk

@@ -66,15 +66,17 @@ include gensrc/GensrcExceptions.gmk
 GENSRC += $(GENSRC_EXCEPTIONS)
 
 ifneq ($(OPENJDK_TARGET_OS), windows)
-include gensrc/GensrcIcons.gmk
-GENSRC += $(GENSRC_AWT_ICONS)
+  include gensrc/GensrcIcons.gmk
+  GENSRC += $(GENSRC_AWT_ICONS)
 
-ifeq ($(OPENJDK_TARGET_OS), macosx)
-GENSRC += $(GENSRC_OSX_ICONS)
-endif
+  ifeq ($(OPENJDK_TARGET_OS), macosx)
+    GENSRC += $(GENSRC_OSX_ICONS)
+  endif
 
-include gensrc/GensrcX11Wrappers.gmk
-GENSRC += $(GENSRC_X11WRAPPERS)
+  ifneq ($(OPENJDK_TARGET_OS), macosx)
+    include gensrc/GensrcX11Wrappers.gmk
+    GENSRC += $(GENSRC_X11WRAPPERS)
+  endif
 endif
 
 include gensrc/GensrcCLDR.gmk

makefiles/Tools.gmk

@@ -32,7 +32,7 @@ TOOLS_SRC := $(JDK_TOPDIR)/make/tools/src \
     $(JDK_TOPDIR)/makefiles/sun/osxapp \
     $(JDK_TOPDIR)/make/tools/swing-beans
 
-ifneq ($(OPENJDK_TARGET_OS), windows)
+ifeq ($(findstring $(OPENJDK_TARGET_OS),windows macosx),)
   TOOLS_SRC += $(JDK_TOPDIR)/src/solaris/classes/sun/awt/X11/generator
 endif
 

makefiles/lib/Awt2dLibraries.gmk

@@ -226,7 +226,7 @@ ifeq ($(OPENJDK_TARGET_OS), windows)
       $(JDK_TOPDIR)/src/$(OPENJDK_TARGET_OS_API_DIR)/native/sun/windows \
       $(JDK_TOPDIR)/src/$(OPENJDK_TARGET_OS_API_DIR)/native/sun/java2d/windows \
       $(JDK_TOPDIR)/src/$(OPENJDK_TARGET_OS_API_DIR)/native/sun/java2d/d3d
-else
+else ifneq ($(OPENJDK_TARGET_OS), macosx)
   LIBAWT_DIRS += \
       $(JDK_TOPDIR)/src/$(OPENJDK_TARGET_OS_API_DIR)/native/sun/java2d/x11
 endif
@@ -509,15 +509,7 @@ BUILD_LIBRARIES += $(BUILD_LIBAWT)
 
 ##########################################################################################
 
-# TODO!!
-# Even though this variable as a general name, it is
-# only used on macos, in fontpath.c, as prefix for where to find fonts.
-#
-# It's used for libawt_headless _and_ libawt_xawt
-#
-X11_PATH := /usr/X11R6
-
-ifneq ($(OPENJDK_TARGET_OS), windows)
+ifeq ($(findstring $(OPENJDK_TARGET_OS),windows macosx),)
   ifndef BUILD_HEADLESS_ONLY
 
     LIBAWT_XAWT_DIRS := \
@@ -532,7 +524,7 @@ ifneq ($(OPENJDK_TARGET_OS), windows)
         $(JDK_TOPDIR)/src/$(OPENJDK_TARGET_OS_API_DIR)/native/sun/xawt \
 
     LIBAWT_XAWT_CFLAGS := -DXAWT -DXAWT_HACK \
-        -DX11_PATH=\"$(X11_PATH)\" -DPACKAGE_PATH=\"$(PACKAGE_PATH)\" \
+        -DPACKAGE_PATH=\"$(PACKAGE_PATH)\" \
         $(CUPS_CFLAGS) \
         $(foreach dir, $(LIBAWT_XAWT_DIRS), -I$(dir)) \
         -I$(JDK_TOPDIR)/src/share/native/sun/java2d \
@@ -627,11 +619,6 @@ ifneq ($(OPENJDK_TARGET_OS), windows)
       LIBAWT_XAWT_LDFLAGS += -lpthread
     endif
 
-    ifeq ($(OPENJDK_TARGET_OS), macosx)
-      LIBAWT_XAWT_LDFLAGS_SUFFIX += -lpthread
-    endif
-
-    # On macosx, the shared library origin is set twice for this lib.
     $(eval $(call SetupNativeCompilation,BUILD_LIBAWT_XAWT, \
         LIBRARY := awt_xawt, \
         OUTPUT_DIR := $(INSTALL_LIBRARIES_HERE), \
@@ -653,10 +640,6 @@ ifneq ($(OPENJDK_TARGET_OS), windows)
             -R/usr/dt/lib$(OPENJDK_TARGET_CPU_ISADIR) \
             $(call SET_SHARED_LIBRARY_ORIGIN) \
             $(call SET_SHARED_LIBRARY_ORIGIN,/..), \
-        LDFLAGS_macosx := $(call SET_SHARED_LIBRARY_ORIGIN) \
-            $(call SET_SHARED_LIBRARY_ORIGIN). \
-            $(call SET_SHARED_LIBRARY_ORIGIN) \
-            $(call SET_SHARED_LIBRARY_ORIGIN)., \
         LDFLAGS_SUFFIX := $(LIBAWT_XAWT_LDFLAGS_SUFFIX), \
         VERSIONINFO_RESOURCE := $(JDK_TOPDIR)/src/windows/resource/version.rc, \
         RC_FLAGS := $(RC_FLAGS) \
@@ -788,6 +771,11 @@ ifeq ($(OPENJDK_TARGET_OS), windows)
   LIBFONTMANAGER_EXCLUDE_FILES += X11FontScaler.c \
       X11TextRenderer.c
   LIBFONTMANAGER_OPTIMIZATION := HIGHEST
+else ifeq ($(OPENJDK_TARGET_OS), macosx)
+  LIBFONTMANAGER_EXCLUDE_FILES += X11FontScaler.c \
+      X11TextRenderer.c \
+      fontpath.c \
+      lcdglyph.c
 else
   LIBFONTMANAGER_EXCLUDE_FILES += fontpath.c \
       lcdglyph.c
@@ -845,7 +833,7 @@ $(eval $(call SetupNativeCompilation,BUILD_LIBFONTMANAGER, \
 
 $(BUILD_LIBFONTMANAGER): $(BUILD_LIBAWT)
 
-ifneq (, $(findstring $(OPENJDK_TARGET_OS), solaris macosx))
+ifeq ($(OPENJDK_TARGET_OS), solaris)
   $(BUILD_LIBFONTMANAGER): $(BUILD_LIBAWT_XAWT)
 endif
 
@@ -947,22 +935,21 @@ ifeq ($(OPENJDK_TARGET_OS), windows)
 
 else # OPENJDK_TARGET_OS not windows
 
-  JAWT_LIBS :=
-  ifneq ($(OPENJDK_TARGET_OS), solaris)
-    JAWT_LIBS += -lawt
-  endif
-
-  ifndef BUILD_HEADLESS_ONLY
-    JAWT_LIBS += -lawt_xawt
-  else
-    JAWT_LIBS += -lawt_headless
-    HEADLESS_CFLAG += -DHEADLESS
-  endif
-
-  JAWT_FILES := jawt.c
   ifeq ($(OPENJDK_TARGET_OS), macosx)
     JAWT_FILES := jawt.m
     JAWT_LIBS := -lawt_lwawt
+  else
+    JAWT_FILES := jawt.c
+    JAWT_LIBS :=
+    ifneq ($(OPENJDK_TARGET_OS), solaris)
+      JAWT_LIBS += -lawt
+    endif
+    ifndef BUILD_HEADLESS_ONLY
+      JAWT_LIBS += -lawt_xawt
+    else
+      JAWT_LIBS += -lawt_headless
+      HEADLESS_CFLAG += -DHEADLESS
+    endif
   endif
 
   $(eval $(call SetupNativeCompilation,BUILD_LIBJAWT, \
@@ -1094,7 +1081,8 @@ endif
 ##########################################################################################
 
 ifeq ($(BUILD_HEADLESS), true)
-  ifneq ($(OPENJDK_TARGET_OS), windows)
+  # Mac and Windows only use the native AWT lib, do not build libawt_headless
+  ifeq ($(findstring $(OPENJDK_TARGET_OS), windows macosx),)
 
     LIBAWT_HEADLESS_DIRS := $(JDK_TOPDIR)/src/share/native/sun/font \
         $(JDK_TOPDIR)/src/share/native/sun/java2d/opengl \
@@ -1108,7 +1096,7 @@ ifeq ($(BUILD_HEADLESS), true)
     endif
 
     LIBAWT_HEADLESS_CFLAGS := -DHEADLESS=true \
-        -DX11_PATH=\"$(X11_PATH)\" -DPACKAGE_PATH=\"$(PACKAGE_PATH)\" \
+        -DPACKAGE_PATH=\"$(PACKAGE_PATH)\" \
         $(CUPS_CFLAGS) \
         $(X_CFLAGS) \
         -I$(JDK_TOPDIR)/src/share/native/sun/java2d \
@@ -1155,16 +1143,6 @@ ifeq ($(BUILD_HEADLESS), true)
         AccelGlyphCache.c \
         CUPSfuncs.c
 
-    ifeq ($(OPENJDK_TARGET_OS), macosx)
-      LIBAWT_HEADLESS_FILES += \
-          AWTFont.m \
-          AWTStrike.m \
-          CCharToGlyphMapper.m \
-          CGGlyphImages.m \
-          CGGlyphOutlines.m \
-          CoreTextSupport.m
-    endif
-
     LIBAWT_HEADLESS_REORDER :=
     ifeq ($(OPENJDK_TARGET_OS), solaris)
       ifneq ($(OPENJDK_TARGET_CPU), x86_64)
@@ -1191,13 +1169,6 @@ ifeq ($(BUILD_HEADLESS), true)
         REORDER := $(LIBAWT_HEADLESS_REORDER), \
         LDFLAGS_SUFFIX_linux := -ljvm -lawt -lm $(LIBDL) -ljava, \
         LDFLAGS_SUFFIX_solaris := $(LIBDL) -ljvm -lawt -lm -ljava $(LIBCXX) -lc, \
-        LDFLAGS_SUFFIX_macosx := -ljvm $(LIBCXX) -lawt $(LIBDL) -ljava \
-            -framework Accelerate \
-            -framework ApplicationServices \
-            -framework Cocoa \
-            -F/System/Library/Frameworks/JavaVM.framework/Frameworks \
-            -framework JavaNativeFoundation \
-            -framework JavaRuntimeSupport, \
         OBJECT_DIR := $(JDK_OUTPUTDIR)/objs/libawt_headless, \
         DEBUG_SYMBOLS := $(DEBUG_ALL_BINARIES)))
 

src/share/native/java/lang/System.c

@@ -206,6 +206,11 @@ Java_java_lang_System_initProperties(JNIEnv *env, jclass cla, jobject props)
     if (sprops->awt_toolkit) {
         PUTPROP(props, "awt.toolkit", sprops->awt_toolkit);
     }
+#ifdef MACOSX
+    if (sprops->awt_headless) {
+        PUTPROP(props, "java.awt.headless", sprops->awt_headless);
+    }
+#endif
 
     /* os properties */
     PUTPROP(props, "os.name", sprops->os_name);

src/share/native/java/lang/java_props.h

@@ -116,6 +116,8 @@ typedef struct {
     char *gopherPort;
 
     char *exceptionList;
+
+    char *awt_headless  /* java.awt.headless setting, if NULL (default) will not be set */
 #endif
 
 } java_props_t;

src/solaris/native/java/lang/java_props_macosx.c

@@ -105,40 +105,17 @@ char *setupMacOSXLocale(int cat) {
     }
 }
 
-/* There are several toolkit options on Mac OS X, so we should try to
- * pick the "best" one, given what we know about the environment Java
- * is running under
- */
-
-static PreferredToolkit getPreferredToolkitFromEnv() {
-    char *envVar = getenv("AWT_TOOLKIT");
-    if (envVar == NULL) return unset;
-
-    if (strcasecmp(envVar, "CToolkit") == 0) return CToolkit;
-    if (strcasecmp(envVar, "XToolkit") == 0) return XToolkit;
-    if (strcasecmp(envVar, "HToolkit") == 0) return HToolkit;
-    return unset;
-}
-
-static bool isInAquaSession() {
+int isInAquaSession() {
     // Is the WindowServer available?
     SecuritySessionId session_id;
     SessionAttributeBits session_info;
     OSStatus status = SessionGetInfo(callerSecuritySession, &session_id, &session_info);
-    if (status != noErr) return false;
-    if (!(session_info & sessionHasGraphicAccess)) return false;
-    return true;
-}
-
-PreferredToolkit getPreferredToolkit() {
-    static PreferredToolkit pref = unset;
-    if (pref != unset) return pref;
-
-    PreferredToolkit prefFromEnv = getPreferredToolkitFromEnv();
-    if (prefFromEnv != unset) return pref = prefFromEnv;
-
-    if (isInAquaSession()) return pref = CToolkit;
-    return pref = HToolkit;
+    if (status == noErr) {
+        if (session_info & sessionHasGraphicAccess) {
+            return 1;
+        }
+    }
+    return 0;
 }
 
 void setOSNameAndVersion(java_props_t *sprops) {

src/solaris/native/java/lang/java_props_macosx.h

@@ -29,10 +29,4 @@ char *setupMacOSXLocale(int cat);
 void setOSNameAndVersion(java_props_t *sprops);
 void setUserHome(java_props_t *sprops);
 void setProxyProperties(java_props_t *sProps);
-
-enum PreferredToolkit_enum {
-    unset = 0, CToolkit, XToolkit, HToolkit
-};
-typedef enum PreferredToolkit_enum PreferredToolkit;
-
-PreferredToolkit getPreferredToolkit();
+int isInAquaSession();

src/solaris/native/java/lang/java_props_md.c

@@ -454,40 +454,21 @@ GetJavaProperties(JNIEnv *env)
     /* patches/service packs installed */
     sprops.patch_level = "unknown";
 
-    /* Java 2D properties */
+    /* Java 2D/AWT properties */
 #ifdef MACOSX
-    PreferredToolkit prefToolkit = getPreferredToolkit();
-    switch (prefToolkit) {
-        case CToolkit:
-        case HToolkit:
-            sprops.graphics_env = "sun.awt.CGraphicsEnvironment";
-            break;
-        case XToolkit:
-#endif
+    // Always the same GraphicsEnvironment and Toolkit on Mac OS X
+    sprops.graphics_env = "sun.awt.CGraphicsEnvironment";
+    sprops.awt_toolkit = "sun.lwawt.macosx.LWCToolkit";
+
+    // check if we're in a GUI login session and set java.awt.headless=true if not
+    sprops.awt_headless = isInAquaSession() ? NULL : "true";
+#else
     sprops.graphics_env = "sun.awt.X11GraphicsEnvironment";
-#ifdef MACOSX
-            break;
-    }
-#endif
-    /* AWT properties */
 #ifdef JAVASE_EMBEDDED
     sprops.awt_toolkit = getEmbeddedToolkit();
     if (sprops.awt_toolkit == NULL) // default as below
-#endif
-#ifdef MACOSX
-        switch (prefToolkit) {
-            case CToolkit:
-                sprops.awt_toolkit = "sun.lwawt.macosx.LWCToolkit";
-                break;
-            case XToolkit:
 #endif
     sprops.awt_toolkit = "sun.awt.X11.XToolkit";
-#ifdef MACOSX
-                break;
-            default:
-                sprops.awt_toolkit = "sun.awt.HToolkit";
-                break;
-        }
 #endif
 
     /* This is used only for debugging of font problems. */

src/solaris/native/sun/awt/fontpath.c

@@ -23,7 +23,7 @@
  * questions.
  */
 
-#if defined(__linux__) || defined(MACOSX)
+#if defined(__linux__)
 #include <string.h>
 #endif /* __linux__ */
 #include <stdio.h>
@@ -59,26 +59,12 @@
 extern Display *awt_display;
 #endif /* !HEADLESS */
 
-#ifdef MACOSX
-
-//
-// XXXDARWIN: Hard-code the path to Apple's fontconfig, as it is
-// not included in the dyld search path by default, and 10.4
-// does not support -rpath.
-//
-// This ignores the build time setting of ALT_FREETYPE_LIB_PATH,
-// and should be replaced with -rpath/@rpath support on 10.5 or later,
-// or via support for a the FREETYPE_LIB_PATH define.
-#define FONTCONFIG_DLL_VERSIONED X11_PATH "/lib/" VERSIONED_JNI_LIB_NAME("fontconfig", "1")
-#define FONTCONFIG_DLL X11_PATH "/lib/" JNI_LIB_NAME("fontconfig")
-#else
 #define FONTCONFIG_DLL_VERSIONED VERSIONED_JNI_LIB_NAME("fontconfig", "1")
 #define FONTCONFIG_DLL JNI_LIB_NAME("fontconfig")
-#endif
 
 #define MAXFDIRS 512    /* Max number of directories that contain fonts */
 
-#if !defined(__linux__) && !defined(MACOSX)
+#if !defined(__linux__)
 /*
  * This can be set in the makefile to "/usr/X11" if so desired.
  */
@@ -128,22 +114,6 @@ static char *fullSolarisFontPath[] = {
     NULL, /* terminates the list */
 };
 
-#elif MACOSX
-static char *full_MACOSX_X11FontPath[] = {
-    X11_PATH "/lib/X11/fonts/TrueType",
-    X11_PATH "/lib/X11/fonts/truetype",
-    X11_PATH "/lib/X11/fonts/tt",
-    X11_PATH "/lib/X11/fonts/TTF",
-    X11_PATH "/lib/X11/fonts/OTF",
-    PACKAGE_PATH "/share/fonts/TrueType",
-    PACKAGE_PATH "/share/fonts/truetype",
-    PACKAGE_PATH "/share/fonts/tt",
-    PACKAGE_PATH "/share/fonts/TTF",
-    PACKAGE_PATH "/share/fonts/OTF",
-    X11_PATH "/lib/X11/fonts/Type1",
-    PACKAGE_PATH "/share/fonts/Type1",
-    NULL, /* terminates the list */
-};
 #else /* __linux */
 /* All the known interesting locations we have discovered on
  * various flavors of Linux
@@ -400,7 +370,7 @@ static char **getX11FontPath ()
 
 #endif /* !HEADLESS */
 
-#if defined(__linux__) || defined(MACOSX)
+#if defined(__linux__)
 /* from awt_LoadLibrary.c */
 JNIEXPORT jboolean JNICALL AWTIsHeadless();
 #endif
@@ -527,8 +497,6 @@ static char *getPlatformFontPathChars(JNIEnv *env, jboolean noType1) {
 
 #if defined(__linux__)
     knowndirs = fullLinuxFontPath;
-#elif defined(MACOSX)
-    knowndirs = full_MACOSX_X11FontPath;
 #else /* IF SOLARIS */
     knowndirs = fullSolarisFontPath;
 #endif
@@ -539,7 +507,7 @@ static char *getPlatformFontPathChars(JNIEnv *env, jboolean noType1) {
      * be initialised.
      */
 #ifndef HEADLESS
-#if defined(__linux__) || defined(MACOSX)
+#if defined(__linux__)
     /* There's no headless build on linux ... */
     if (!AWTIsHeadless()) { /* .. so need to call a function to check */
 #endif
@@ -555,7 +523,7 @@ static char *getPlatformFontPathChars(JNIEnv *env, jboolean noType1) {
         x11dirs = getX11FontPath();
     }
     AWT_UNLOCK();
-#if defined(__linux__) || defined(MACOSX)
+#if defined(__linux__)
     }
 #endif
 #endif /* !HEADLESS */

test/java/awt/Toolkit/Headless/WrappedToolkitTest/WrappedToolkitTest.sh

@@ -77,6 +77,12 @@ case "$OS" in
       FILESEP="\\"
       ;;
 
+    Darwin)
+      VAR="Lets not forget about Mac"
+      DEFAULT_JDK=$(/usr/libexec/java_home)
+      FILESEP="/"
+      ;;
+
    # catch all other OSs
    * )
       echo "Unrecognized system!  $OS"
@@ -171,6 +177,22 @@ case "$OS" in
     fi
     ;;
 
+  Darwin)
+    ${TESTJAVA}/bin/java -Djava.awt.headless=true \
+                         TestWrapped sun.lwawt.macosx.LWCToolkit
+    status=$?
+    if [ ! $status -eq "0" ]; then
+      fail "Test FAILED: toolkit wrapped into HeadlessToolkit is not an instance of sun.lwawt.macosx.LWCToolkit";
+    fi
+    ${TESTJAVA}/bin/java -Djava.awt.headless=true \
+                         -Dawt.toolkit=sun.lwawt.macosx.LWCToolkit \
+                         TestWrapped sun.lwawt.macosx.LWCToolkit
+    status=$?
+    if [ ! $status -eq "0" ]; then
+      fail "Test FAILED: toolkit wrapped into HeadlessToolkit is not an instance of sun.lwawt.macosx.LWCToolkit";
+    fi
+    ;;
+
 esac
 
 pass "All the tests are PASSED";