8189095: Import JMC from artifactory using Jib and main makefiles

Reviewed-by: ihse

make/Bundles.gmk

@@ -70,9 +70,14 @@ define SetupBundleFileBody
   $$(call SetIfEmpty, $1_UNZIP_DEBUGINFO, false)
 
   $(BUNDLES_OUTPUTDIR)/$$($1_BUNDLE_NAME): $$($1_FILES)
+        # If any of the files contain a space in the file name, CacheFind
+        # will have replaced it with ?. Tar does not accept that so need to
+        # switch it back.
 	$$(foreach d, $$($1_BASE_DIRS), \
 	  $$(eval $$(call ListPathsSafely, \
 	      $1_$$d_RELATIVE_FILES, $$($1_$$d_LIST_FILE))) \
+	  $$(CAT) $$($1_$$d_LIST_FILE) | $$(TR) '?' ' ' > $$($1_$$d_LIST_FILE).tmp \
+	      && $(MV) $$($1_$$d_LIST_FILE).tmp $$($1_$$d_LIST_FILE) $$(NEWLINE) \
 	)
 	$$(call MakeDir, $$(@D))
         ifneq ($$($1_SPECIAL_INCLUDES), )

make/MacBundles.gmk

@@ -49,22 +49,17 @@ ifeq ($(OPENJDK_TARGET_OS), macosx)
     BUNDLE_VENDOR := $(COMPANY_NAME)
   endif
 
-  JDK_FILE_LIST := $(shell $(FIND) $(JDK_IMAGE_DIR))
-  JRE_FILE_LIST := $(shell $(FIND) $(JRE_IMAGE_DIR))
-
-  JDK_TARGET_LIST := $(subst $(JDK_IMAGE_DIR)/,$(JDK_MACOSX_CONTENTS_DIR)/Home/,$(JDK_FILE_LIST))
-  JRE_TARGET_LIST := $(subst $(JRE_IMAGE_DIR)/,$(JRE_MACOSX_CONTENTS_DIR)/Home/,$(JRE_FILE_LIST))
-
-  # Copy empty directories (jre/lib/applet).
-  $(JDK_MACOSX_CONTENTS_DIR)/Home/%: $(JDK_IMAGE_DIR)/%
-	$(call LogInfo, Copying $(patsubst $(OUTPUTDIR)/%,%,$@))
-	$(MKDIR) -p $(@D)
-	if [ -d "$<" ]; then $(MKDIR) -p $@; else $(CP) -f -R -P '$<' '$@'; fi
+  $(eval $(call SetupCopyFiles, COPY_JDK_IMAGE, \
+      SRC := $(JDK_IMAGE_DIR), \
+      DEST := $(JDK_MACOSX_CONTENTS_DIR)/Home, \
+      FILES := $(call CacheFind, $(JDK_IMAGE_DIR)), \
+  ))
 
-  $(JRE_MACOSX_CONTENTS_DIR)/Home/%: $(JRE_IMAGE_DIR)/%
-	$(call LogInfo, Copying $(patsubst $(OUTPUTDIR)/%,%,$@))
-	$(MKDIR) -p $(@D)
-	if [ -d "$<" ]; then $(MKDIR) -p $@; else $(CP) -f -R -P '$<' '$@'; fi
+  $(eval $(call SetupCopyFiles, COPY_JRE_IMAGE, \
+      SRC := $(JRE_IMAGE_DIR), \
+      DEST := $(JRE_MACOSX_CONTENTS_DIR)/Home, \
+      FILES := $(call CacheFind, $(JRE_IMAGE_DIR)), \
+  ))
 
   $(JDK_MACOSX_CONTENTS_DIR)/MacOS/libjli.dylib:
 	$(call LogInfo, Creating link $(patsubst $(OUTPUTDIR)/%,%,$@))
@@ -102,11 +97,11 @@ ifeq ($(OPENJDK_TARGET_OS), macosx)
           @@VENDOR@@ => $(BUNDLE_VENDOR) , \
   ))
 
-  jdk-bundle: $(JDK_TARGET_LIST) $(JDK_MACOSX_CONTENTS_DIR)/MacOS/libjli.dylib \
+  jdk-bundle: $(COPY_JDK_IMAGE) $(JDK_MACOSX_CONTENTS_DIR)/MacOS/libjli.dylib \
       $(BUILD_JDK_PLIST)
 	$(SETFILE) -a B $(dir $(JDK_MACOSX_CONTENTS_DIR))
 
-  jre-bundle: $(JRE_TARGET_LIST) $(JRE_MACOSX_CONTENTS_DIR)/MacOS/libjli.dylib \
+  jre-bundle: $(COPY_JRE_IMAGE) $(JRE_MACOSX_CONTENTS_DIR)/MacOS/libjli.dylib \
       $(BUILD_JRE_PLIST)
 	$(SETFILE) -a B $(dir $(JRE_MACOSX_CONTENTS_DIR))
 

make/common/MakeBase.gmk

@@ -462,12 +462,23 @@ define NamedParamsMacroTemplate
   $(call $(0)Body,$(strip $1))
 endef
 
+################################################################################
+# Replace question marks with space in string. This macro needs to be called on
+# files from CacheFind in case any of them contains space in their file name,
+# since CacheFind replaces space with ?.
+# Param 1 - String to replace in
+DecodeSpace = \
+    $(subst ?,$(SPACE),$(strip $1))
+EncodeSpace = \
+    $(subst $(SPACE),?,$(strip $1))
+
 ################################################################################
 # Make directory without forking mkdir if not needed
 # 1: List of directories to create
 MakeDir = \
     $(strip \
-        $(eval MakeDir_dirs_to_make := $(strip $(foreach d, $1, $(if $(wildcard $d), , $d)))) \
+        $(eval MakeDir_dirs_to_make := $(strip $(foreach d, $1, $(if $(wildcard $d), , \
+            "$(call DecodeSpace, $d)")))) \
         $(if $(MakeDir_dirs_to_make), $(shell $(MKDIR) -p $(MakeDir_dirs_to_make))) \
     )
 
@@ -479,6 +490,7 @@ SetIfEmpty = \
     $(if $($(strip $1)),,$(eval $(strip $1) := $2))
 
 ################################################################################
+# All install-file and related macros automatically call DecodeSpace when needed.
 
 ifeq ($(OPENJDK_TARGET_OS),solaris)
   # On Solaris, if the target is a symlink and exists, cp won't overwrite.
@@ -487,19 +499,21 @@ ifeq ($(OPENJDK_TARGET_OS),solaris)
   # If the source and target parent directories are the same, recursive copy doesn't work
   # so we fall back on regular copy, which isn't preserving symlinks.
   define install-file
-	$(MKDIR) -p '$(@D)'
-	$(RM) '$@'
-	if [ "$(@D)" != "$(<D)" ]; then \
-	  $(CP) -f -r -P '$<' '$(@D)'; \
-	  if [ "$(@F)" != "$(<F)" ]; then \
-	    $(MV) '$(@D)/$(<F)' '$@'; \
+	$(call MakeDir, $(@D))
+	$(RM) "$(call DecodeSpace, $@)"
+	if [ "$(call DecodeSpace, $(dir $@))" != \
+	    "$(call DecodeSpace, $(dir $(call EncodeSpace, $<)))" ]; then \
+	  $(CP) -f -r -P "$(call DecodeSpace, $<)" "$(call DecodeSpace, $(@D))"; \
+	  if [ "$(call DecodeSpace, $(@F))" != \
+	      "$(call DecodeSpace, $(notdir $(call EncodeSpace, $(<))))" ]; then \
+	    $(MV) "$(call DecodeSpace, $(@D)/$(<F))" "$(call DecodeSpace, $@)"; \
 	  fi; \
 	else \
-	  if [ -L '$<' ]; then \
+	  if [ -L "$(call DecodeSpace, $<)" ]; then \
 	    $(ECHO) "Source file is a symlink and target is in the same directory: $< $@" ; \
 	    exit 1; \
 	  fi; \
-	  $(CP) -f '$<' '$@'; \
+	  $(CP) -f "$(call DecodeSpace, $<)" "$(call DecodeSpace, $@)"; \
 	fi
   endef
 else ifeq ($(OPENJDK_TARGET_OS),macosx)
@@ -512,22 +526,22 @@ else ifeq ($(OPENJDK_TARGET_OS),macosx)
   # If copying a soft link to a directory, need to delete the target first to avoid
   # weird errors.
   define install-file
-	$(MKDIR) -p '$(@D)'
-	$(RM) '$@'
-	$(CP) -fRP '$<' '$@'
-	if [ -n "`$(XATTR) -l '$@'`" ]; then $(XATTR) -c '$@'; fi
+	$(call MakeDir, $(@D))
+	$(RM) "$(call DecodeSpace, $@)"
+	$(CP) -fRP "$(call DecodeSpace, $<)" "$(call DecodeSpace, $@)"
+	if [ -n "`$(XATTR) -l "$(call DecodeSpace, $@)"`" ]; then $(XATTR) -c "$(call DecodeSpace, $@)"; fi
   endef
 else
   define install-file
 	$(call MakeDir, $(@D))
-	$(CP) -fP '$<' '$@'
+	$(CP) -fP "$(call DecodeSpace, $<)" "$(call DecodeSpace, $@)"
   endef
 endif
 
 # Variant of install file that does not preserve symlinks
 define install-file-nolink
 	$(call MakeDir, $(@D))
-	$(CP) -f '$<' '$@'
+	$(CP) -f "$(call DecodeSpace, $<)" "$(call DecodeSpace, $@)"
 endef
 
 ################################################################################
@@ -577,14 +591,14 @@ RelativePath = \
 # the unix emulation environment.
 define link-file-relative
 	$(call MakeDir, $(@D))
-	$(RM) $@
-	$(LN) -s $(call RelativePath, $<, $(@D)) $@
+	$(RM) "$(call DecodeSpace, $@)"
+	$(LN) -s "$(call DecodeSpace, $(call RelativePath, $<, $(@D)))" "$(call DecodeSpace, $@)"
 endef
 
 define link-file-absolute
 	$(call MakeDir, $(@D))
-	$(RM) $@
-	$(LN) -s $< $@
+	$(RM) "$(call DecodeSpace, $@)"
+	$(LN) -s "$(call DecodeSpace, $<)" "$(call DecodeSpace, $@)"
 endef
 
 ################################################################################
@@ -651,6 +665,13 @@ ifneq ($(DISABLE_CACHE_FIND), true)
   # This macro can be called multiple times to add to the cache. Only finds files
   # with no filters.
   #
+  # Files containing space will get spaces replaced with ? because GNU Make
+  # cannot handle lists of files with space in them. By using ?, make will match
+  # the wildcard to space in many situations so we don't need to replace back
+  # to space on every use. While not a complete solution it does allow some uses
+  # of CacheFind to function with spaces in file names, including for
+  # SetupCopyFiles.
+  #
   # Needs to be called with $(eval )
   #
   # Even if the performance benifit is negligible on other platforms, keep the
@@ -668,7 +689,8 @@ ifneq ($(DISABLE_CACHE_FIND), true)
     ifneq ($$(FIND_CACHE_NEW_DIRS), )
       # Remove any trailing slash from dirs in the cache dir list
       FIND_CACHE_DIRS += $$(patsubst %/,%, $$(FIND_CACHE_NEW_DIRS))
-      FIND_CACHE := $$(sort $$(FIND_CACHE) $$(shell $(FIND) $$(FIND_CACHE_NEW_DIRS) \( -type f -o -type l \) $2))
+      FIND_CACHE := $$(sort $$(FIND_CACHE) $$(shell $(FIND) $$(FIND_CACHE_NEW_DIRS) \
+          \( -type f -o -type l \) $2 | $(TR) ' ' '?'))
     endif
   endef
 
@@ -684,7 +706,8 @@ ifneq ($(DISABLE_CACHE_FIND), true)
   # Param 2 - (optional) specialization. Normally "-a \( ... \)" expression.
   define CacheFind
     $(if $(filter-out $(addsuffix /%,- $(FIND_CACHE_DIRS)) $(FIND_CACHE_DIRS),$1), \
-      $(if $(wildcard $1), $(shell $(FIND) $1 \( -type f -o -type l \) $2)), \
+      $(if $(wildcard $1), $(shell $(FIND) $1 \( -type f -o -type l \) $2 \
+          | $(TR) ' ' '?')), \
       $(filter $(addsuffix /%,$(patsubst %/,%,$1)) $1,$(FIND_CACHE)))
   endef
 
@@ -693,7 +716,7 @@ else
   # Param 1 - Dirs to find in
   # Param 2 - (optional) specialization. Normally "-a \( ... \)" expression.
   define CacheFind
-    $(shell $(FIND) $1 \( -type f -o -type l \) $2)
+    $(shell $(FIND) $1 \( -type f -o -type l \) $2 | $(TR) ' ' '?')
   endef
 endif
 
@@ -707,7 +730,7 @@ define AddFileToCopy
   # 4 : Macro to call for copy operation
   # 5 : Action text to log
   $2: $1
-	$$(call LogInfo, $(strip $5) $$(patsubst $(OUTPUTDIR)/%,%,$$@))
+	$$(call LogInfo, $(strip $5) $$(patsubst $(OUTPUTDIR)/%,%,$$(call DecodeSpace, $$@)))
 	$$($$(strip $4))
 
   $3 += $2

test/make/TestCopyFiles.gmk

+#
+# Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved.
+# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+#
+# This code is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License version 2 only, as
+# published by the Free Software Foundation.  Oracle designates this
+# particular file as subject to the "Classpath" exception as provided
+# by Oracle in the LICENSE file that accompanied this code.
+#
+# This code is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+# version 2 for more details (a copy is included in the LICENSE file that
+# accompanied this code).
+#
+# You should have received a copy of the GNU General Public License version
+# 2 along with this work; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+# or visit www.oracle.com if you need additional information or have any
+# questions.
+#
+
+default: all
+
+include $(SPEC)
+include MakeBase.gmk
+include UtilsForTests.gmk
+
+THIS_FILE := $(TOPDIR)/test/make/TestCopyFiles.gmk
+DEPS := $(THIS_FILE) \
+    $(TOPDIR)/make/common/MakeBase.gmk \
+    #
+
+OUTPUT_DIR := $(TESTMAKE_OUTPUTDIR)/copy-files
+$(call MakeDir, $(OUTPUT_DIR))
+
+################################################################################
+# Test SetupCopyFiles with CacheFind and files with spaces in their names.
+
+SRC_DIR := $(OUTPUT_DIR)/src
+DEST_DIR := $(OUTPUT_DIR)/dest
+
+$(OUTPUT_DIR)/_src_created: $(DEPS)
+	$(RM) -r $(SRC_DIR)
+	$(MKDIR) -p $(SRC_DIR)
+	$(MKDIR) -p $(SRC_DIR)/foo
+	$(TOUCH) $(SRC_DIR)/file
+	$(TOUCH) $(SRC_DIR)/foo/foofile
+	$(TOUCH) "$(SRC_DIR)/foo/foo file"
+        # Spaces in directories only works with gnu make 4.0 or later
+        ifeq (4.0, $(firstword $(sort 4.0 $(MAKE_VERSION))))
+	  $(MKDIR) -p "$(SRC_DIR)/foo bar"
+	  $(TOUCH) "$(SRC_DIR)/foo bar/foobarfile"
+	  $(TOUCH) "$(SRC_DIR)/foo bar/foo bar file"
+        endif
+	$(LN) -s file "$(SRC_DIR)/link to file"
+	$(TOUCH) $@
+
+$(eval $(call SetupCopyFiles, COPY_1, \
+    SRC := $(SRC_DIR), \
+    DEST := $(DEST_DIR), \
+    FILES := $(call CacheFind, $(SRC_DIR)), \
+))
+
+do-copy1: $(COPY_1)
+
+run-test1: $(OUTPUT_DIR)/_src_created
+	+$(MAKE) -f $(THIS_FILE) do-copy1
+	$(DIFF) -r $(SRC_DIR) $(DEST_DIR)
+
+TEST_TARGETS += run-test1
+
+.PHONY: do-copy1 run-test1
+
+################################################################################
+
+all: $(TEST_TARGETS)

test/make/TestIdea.gmk

@@ -35,11 +35,11 @@ clean-idea:
 
 verify-idea:
 	$(MKDIR) -p $(IDEA_OUTPUT_DIR)
-	$(BASH) $(TOPDIR)/common/bin/idea.sh -o $(IDEA_OUTPUT_DIR)/idea1
-	$(BASH) $(TOPDIR)/common/bin/idea.sh -o $(IDEA_OUTPUT_DIR)/idea2 java.base
-	$(BASH) $(TOPDIR)/common/bin/idea.sh -o $(IDEA_OUTPUT_DIR)/idea3 java.base jdk.compiler
+	MAKEFLAGS= MFLAGS= $(BASH) $(TOPDIR)/bin/idea.sh -o $(IDEA_OUTPUT_DIR)/idea1
+	MAKEFLAGS= MFLAGS= $(BASH) $(TOPDIR)/bin/idea.sh -o $(IDEA_OUTPUT_DIR)/idea2 java.base
+	MAKEFLAGS= MFLAGS= $(BASH) $(TOPDIR)/bin/idea.sh -o $(IDEA_OUTPUT_DIR)/idea3 java.base jdk.compiler
 
-TEST_TARGETS += verify-idea 
+TEST_TARGETS += verify-idea
 
 all: $(TEST_TARGETS)
 

test/make/TestJavaCompilation.gmk

@@ -29,6 +29,7 @@ include $(SPEC)
 include MakeBase.gmk
 include JarArchive.gmk
 include JavaCompilation.gmk
+include UtilsForTests.gmk
 
 THIS_FILE := $(TOPDIR)/test/make/TestJavaCompilation.gmk
 DEPS := $(THIS_FILE) \
@@ -90,8 +91,8 @@ $(OUTPUT_DIR)/_jar1_verified: $(BUILD_JAR1)
 create-jar2: $(OUTPUT_DIR)/_jar1_verified
 TEST_TARGETS += $(OUTPUT_DIR)/_jar1_verified
 
-# Change a source file and call this makefile again to force the jar to be 
-# updated. 
+# Change a source file and call this makefile again to force the jar to be
+# updated.
 $(OUTPUT_DIR)/_jar1_updated: $(OUTPUT_DIR)/_jar1_verified
 	$(ECHO) updated > $(JAR1_SRC_ROOT)/dir1/file1.class
 	$(ECHO) updated > $(JAR1_SRC_ROOT)/META-INF/metafile
@@ -104,6 +105,7 @@ update-jar1: $(OUTPUT_DIR)_jar1_updated
 # Change the manifest file and call this makefile again to force the jar
 # to be updated
 $(OUTPUT_DIR)/_jar1_updated_manifest: $(OUTPUT_DIR)/_jar1_updated
+	$(SLEEP_ON_MAC)
 	$(ECHO) "Test-Attribute: foobar" > $(JAR1_MANIFEST)
 	+$(MAKE) -f $(THIS_FILE) $(BUILD_JAR1)
 	$(RM) -r $(JAR1_UNZIP)

test/make/TestMake.gmk

@@ -33,10 +33,13 @@ make-base:
 java-compilation:
 	+$(MAKE) -f TestJavaCompilation.gmk $(TEST_SUBTARGET)
 
+copy-files:
+	+$(MAKE) -f TestCopyFiles.gmk $(TEST_SUBTARGET)
+
 test-idea:
 	+$(MAKE) -f TestIdea.gmk $(TEST_SUBTARGET)
 
 
-all: make-base java-compilation test-idea
+all: make-base java-compilation copy-files test-idea
 
-.PHONY: default all make-base java-compilation test-idea
+.PHONY: default all make-base java-compilation copy-files test-idea

test/make/TestMakeBase.gmk

@@ -27,28 +27,13 @@ default: all
 
 include $(SPEC)
 include MakeBase.gmk
+include UtilsForTests.gmk
 
 THIS_FILE := $(TOPDIR)/test/make/TestMakeBase.gmk
 DEPS := $(THIS_FILE) \
     $(TOPDIR)/make/common/MakeBase.gmk \
     #
 
-# Assert two strings are equal
-# 1 - Tested value
-# 2 - Exepected value
-# 3 - Error message
-define assert-equals
-  ifneq ($$(strip $1),$$(strip $2))
-    $$(error $3 - Expected >$$(strip $2)< - Got >$$(strip $1)<)
-  endif
-endef
-
-# On macosx, file system timestamps only have 1 second resultion so must add
-# sleeps to properly test dependencies.
-ifeq ($(OPENJDK_BUILD_OS), macosx)
-  SLEEP_ON_MAC := sleep 1
-endif
-
 OUTPUT_DIR := $(TESTMAKE_OUTPUTDIR)/make-base
 $(call MakeDir, $(OUTPUT_DIR))
 

test/make/UtilsForTests.gmk

+#
+# Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved.
+# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+#
+# This code is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License version 2 only, as
+# published by the Free Software Foundation.  Oracle designates this
+# particular file as subject to the "Classpath" exception as provided
+# by Oracle in the LICENSE file that accompanied this code.
+#
+# This code is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+# version 2 for more details (a copy is included in the LICENSE file that
+# accompanied this code).
+#
+# You should have received a copy of the GNU General Public License version
+# 2 along with this work; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+# or visit www.oracle.com if you need additional information or have any
+# questions.
+#
+
+# This file contains utilities common for multiple test makefiles.
+
+# Assert two strings are equal
+# 1 - Tested value
+# 2 - Exepected value
+# 3 - Error message
+define assert-equals
+  ifneq ($$(strip $1),$$(strip $2))
+    $$(error $3 - Expected >$$(strip $2)< - Got >$$(strip $1)<)
+  endif
+endef
+
+# On macosx, file system timestamps only have 1 second resultion so must add
+# sleeps to properly test dependencies.
+ifeq ($(OPENJDK_BUILD_OS), macosx)
+  SLEEP_ON_MAC := sleep 1
+endif