8022071: Some vm/jvmti tests fail because cannot attach to the Java virtual machine

Reviewed-by: erikj, sspitsyn

makefiles/CompileNativeLibraries.gmk

@@ -199,7 +199,7 @@ LIBJAVA_CFLAGS:=$(foreach dir,$(LIBJAVA_SRC_DIRS),-I$(dir)) \
 LIBJAVA_CFLAGS += -DJDK_MAJOR_VERSION='"$(JDK_MAJOR_VERSION)"' \
                   -DJDK_MINOR_VERSION='"$(JDK_MINOR_VERSION)"' \
                   -DJDK_MICRO_VERSION='"$(JDK_MICRO_VERSION)"' \
-                  -DJDK_BUILD_NUMBER='"$(JDK_BUILD_NUMBER)"' 
+                  -DJDK_BUILD_NUMBER='"$(JDK_BUILD_NUMBER)"'
 
 ifneq (,$(JDK_UPDATE_VERSION))
 	LIBJAVA_CFLAGS += -DJDK_UPDATE_VERSION='"$(JDK_UPDATE_VERSION)"'
@@ -279,7 +279,7 @@ $(BUILD_LIBJAVA) : $(BUILD_LIBFDLIBM)
 BUILD_LIBMLIB_SRC:=$(JDK_TOPDIR)/src/share/native/sun/awt/medialib
 BUILD_LIBMLIB_CFLAGS:=-D__USE_J2D_NAMES -D__MEDIALIB_OLD_NAMES \
 			-I$(BUILD_LIBMLIB_SRC) \
-			-I$(JDK_TOPDIR)/src/$(OPENJDK_TARGET_OS_API_DIR)/native/sun/awt/medialib 
+			-I$(JDK_TOPDIR)/src/$(OPENJDK_TARGET_OS_API_DIR)/native/sun/awt/medialib
 
 BUILD_LIBMLIB_LDLIBS:=
 BUILD_LIBMLIB_IMAGE_MAPFILE:=$(JDK_TOPDIR)/makefiles/mapfiles/libmlib_image/mapfile-vers
@@ -1042,6 +1042,7 @@ $(eval $(call SetupNativeCompilation,BUILD_LIBATTACH,\
 		LANG:=C,\
 		OPTIMIZATION:=LOW, \
 		CFLAGS:=$(CFLAGS_JDKLIB),\
+		CFLAGS_windows:=/Gy,\
 		MAPFILE:=$(JDK_TOPDIR)/makefiles/mapfiles/libattach/mapfile-$(OPENJDK_TARGET_OS), \
 		VERSIONINFO_RESOURCE:=$(JDK_TOPDIR)/src/windows/resource/version.rc,\
 		RC_FLAGS:=$(RC_FLAGS) \
@@ -1051,6 +1052,7 @@ $(eval $(call SetupNativeCompilation,BUILD_LIBATTACH,\
 		LDFLAGS:=$(LDFLAGS_JDKLIB) \
 			 $(call SET_SHARED_LIBRARY_ORIGIN),\
 		LDFLAGS_solaris:=-ldoor,\
+		LDFLAGS_windows:=/ORDER:@$(JDK_TOPDIR)/makefiles/mapfiles/libattach/reorder-windows-$(OPENJDK_TARGET_CPU),\
 		LDFLAGS_SUFFIX:=$(LDFLAGS_JDKLIB_SUFFIX),\
 		LDFLAGS_SUFFIX_windows:=$(WIN_JAVA_LIB) advapi32.lib psapi.lib,\
 		OBJECT_DIR:=$(JDK_OUTPUTDIR)/objs/libattach,\
@@ -1413,10 +1415,10 @@ ifndef OPENJDK
 #    ifeq ($(OPENJDK_TARGET_OS), linux)
 #        ifeq ("$(CC_VER_MAJOR)", "3")
 #            OTHER_LDLIBS  += -Wl,-Bstatic -lgcc_eh -Wl,-Bdynamic
-#        endif                           
+#        endif
 #    endif
 #
-# The resulting size of the t2k lib file is (at least on linux) dependant on the order of 
+# The resulting size of the t2k lib file is (at least on linux) dependant on the order of
 # the input .o files. Because of this the new build will differ in size to the old build.
     BUILD_LIBT2K_CFLAGS_COMMON:=-I$(JDK_TOPDIR)/src/share/native/sun/font \
 		            -I$(JDK_TOPDIR)/src/closed/share/native/sun/font/t2k \
@@ -1590,8 +1592,8 @@ LIBINSTRUMENT_LDFLAGS_SUFFIX:=
 ifeq ($(OPENJDK_TARGET_OS), windows)
     LIBINSTRUMENT_LDFLAGS += $(JDK_OUTPUTDIR)/objs/jli_static.lib $(WIN_JAVA_LIB) \
 				 -export:Agent_OnAttach advapi32.lib
-    # Statically link the C runtime so that there are not dependencies on modules 
-    # not on the search patch when invoked from the Windows system directory 
+    # Statically link the C runtime so that there are not dependencies on modules
+    # not on the search patch when invoked from the Windows system directory
     # (or elsewhere).
     LIBINSTRUMENT_CFLAGS := $(filter-out -MD,$(LIBINSTRUMENT_CFLAGS))
     # equivalent of strcasecmp is stricmp on Windows
@@ -2065,13 +2067,13 @@ endif
 
 ifeq ($(OPENJDK_TARGET_OS), windows)
 	BUILD_LIBJLI_FILES += java_md.c \
-			      cmdtoargs.c	
+			      cmdtoargs.c
         # Staticically link with c runtime on windows.
 	LIBJLI_CFLAGS:=$(filter-out -MD,$(LIBJLI_CFLAGS))
 else ifneq ($(OPENJDK_TARGET_OS), macosx)
 
 	BUILD_LIBJLI_FILES += java_md_common.c
-	BUILD_LIBJLI_FILES += java_md_solinux.c ergo.c 
+	BUILD_LIBJLI_FILES += java_md_solinux.c ergo.c
 
 	ERGO_ARCH_FILE = ergo_$(ERGO_FAMILY).c
 
@@ -2518,7 +2520,7 @@ $(eval $(call SetupNativeCompilation,LIBSPLASHSCREEN,\
 BUILD_LIBRARIES += $(LIBSPLASHSCREEN)
 
 ifeq ($(OPENJDK_TARGET_OS),macosx)
-$(LIBSPLASHSCREEN) :  $(INSTALL_LIBRARIES_HERE)/$(LIBRARY_PREFIX)osxapp$(SHARED_LIBRARY_SUFFIX) 
+$(LIBSPLASHSCREEN) :  $(INSTALL_LIBRARIES_HERE)/$(LIBRARY_PREFIX)osxapp$(SHARED_LIBRARY_SUFFIX)
 endif
 
 endif
@@ -3246,7 +3248,7 @@ $(eval $(call SetupNativeCompilation,BUILD_LIBAWT_LWAWT,\
 
 BUILD_LIBRARIES += $(BUILD_LIBAWT_LWAWT)
 
-$(BUILD_LIBAWT_LWAWT) : $(BUILD_LIBAWT) 
+$(BUILD_LIBAWT_LWAWT) : $(BUILD_LIBAWT)
 
 $(BUILD_LIBAWT_LWAWT) : $(BUILD_LIBMLIB_IMAGE)
 
@@ -3287,7 +3289,7 @@ $(eval $(call SetupNativeCompilation,BUILD_LIBOSXUI,\
 
 BUILD_LIBRARIES += $(BUILD_LIBOSXUI)
 
-$(BUILD_LIBOSXUI) : $(BUILD_LIBAWT) 
+$(BUILD_LIBOSXUI) : $(BUILD_LIBAWT)
 
 $(BUILD_LIBOSXUI) : $(BUILD_LIBOSXAPP)
 

makefiles/mapfiles/libattach/reorder-windows-x86

+jvm_attach_thread_func@4
+jvm_attach_thread_func_end

makefiles/mapfiles/libattach/reorder-windows-x86_64

+jvm_attach_thread_func
+jvm_attach_thread_func_end

src/windows/native/sun/tools/attach/WindowsVirtualMachine.c

 /*
- * Copyright (c) 2005, 2012, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2005, 2013, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -91,7 +91,7 @@ typedef struct {
  * Code copied to target process
  */
 #pragma check_stack (off)
-static DWORD WINAPI thread_func(DataBlock *pData)
+DWORD WINAPI jvm_attach_thread_func(DataBlock *pData)
 {
     HINSTANCE h;
     EnqueueOperationFunc addr;
@@ -117,8 +117,8 @@ static DWORD WINAPI thread_func(DataBlock *pData)
     }
 }
 
-/* This function marks the end of thread_func. */
-static void thread_end (void) {
+/* This function marks the end of jvm_attach_thread_func. */
+void jvm_attach_thread_func_end (void) {
 }
 #pragma check_stack
 
@@ -152,10 +152,10 @@ JNIEXPORT jbyteArray JNICALL Java_sun_tools_attach_WindowsVirtualMachine_generat
     DWORD len;
     jbyteArray array;
 
-    len = (DWORD)((LPBYTE) thread_end - (LPBYTE) thread_func);
+    len = (DWORD)((LPBYTE) jvm_attach_thread_func_end - (LPBYTE) jvm_attach_thread_func);
     array= (*env)->NewByteArray(env, (jsize)len);
     if (array != NULL) {
-        (*env)->SetByteArrayRegion(env, array, 0, (jint)len, (jbyte*)&thread_func);
+        (*env)->SetByteArrayRegion(env, array, 0, (jint)len, (jbyte*)&jvm_attach_thread_func);
     }
     return array;
 }