Configure et al: move the installation directory logic to Makefiles

The logic to figure out the combinations of --prefix and --openssldir
has stayed in Configure so far, with Unix paths as defaults.

However, since we're making Configure increasingly platform agnostic,
these defaults need to change and adapt to the platform, along with
the logic to combine them.

The easiest to provide for this is to move the logic and the defaults
away from Configure and into the build files.

This also means that the definition of the macros ENGINESDIR and
OPENSSLDIR move away from include/openssl/opensslconf.h and into the
build files.

Makefile.in is adapted accordingly.
Reviewed-by: NRich Salz <rsalz@openssl.org>

Configure

@@ -578,6 +578,8 @@ foreach (@argvcopy)
 		if (/^--prefix=(.*)$/)
 			{
 			$config{prefix}=$1;
+			die "Directory given with --prefix MUST be absolute\n"
+				unless file_name_is_absolute($config{prefix});
 			}
 		elsif (/^--api=(.*)$/)
 			{
@@ -813,11 +815,6 @@ my $make	= $ENV{'MAKE'} || "make";
 $config{cross_compile_prefix} = $ENV{'CROSS_COMPILE'}
     if $config{cross_compile_prefix} eq "";
 
-$config{prefix} = "/usr/local" if !$config{prefix};
-$config{openssldir} = "ssl" if !$config{openssldir};
-$config{openssldir} = catdir($config{prefix}, $config{openssldir})
-    unless file_name_is_absolute($config{openssldir});
-
 # Allow environment CC to override compiler...
 $target{cc} = $ENV{CC} || $target{cc};
 
@@ -850,15 +847,6 @@ $target{build_scheme} = [ $target{build_scheme} ]
 my ($builder, $builder_platform, @builder_opts) =
     @{$target{build_scheme}};
 
-# if $config{prefix}/lib$target{multilib} is not an existing directory, then
-# assume that it's not searched by linker automatically, in
-# which case adding $target{multilib} suffix causes more grief than
-# we're ready to tolerate, so don't...
-$target{multilib}="" if !-d "$config{prefix}/lib$target{multilib}";
-
-$config{libdir}="lib$target{multilib}" if $config{libdir} eq "";
-$config{enginesdir}=$config{prefix} . "/" . $config{libdir}  . "/engines";
-
 push @{$config{defines}},
     map { (my $x = $_) =~ s/^OPENSSL_NO_/OPENSSL_EXPERIMENTAL_/; $x }
         @{$config{openssl_experimental_defines}};
@@ -1007,19 +995,6 @@ if ($builder ne "mk1mf")
 #
 # Platform fix-ups
 #
-# Unlike other OSes (like Solaris, Linux, Tru64, IRIX) BSD run-time
-# linkers (tested OpenBSD, NetBSD and FreeBSD) "demand" RPATH set on
-# .so objects. Apparently application RPATH is not global and does
-# not apply to .so linked with other .so. Problem manifests itself
-# when libssl.so fails to load libcrypto.so. One can argue that we
-# should engrave this into Makefile.shared rules or into BSD-* config
-# lines above. Meanwhile let's try to be cautious and pass -rpath to
-# linker only when --prefix is not /usr.
-if ($target =~ /^BSD-/)
-	{
-	$target{shared_ldflag}.=" -Wl,-rpath,\$\$(LIBRPATH)" if ($config{prefix} !~ m|^/usr[/]*$|);
-	}
-
 if ($target{sys_id} ne "")
 	{
 	push @{$config{openssl_sys_defines}}, "OPENSSL_SYS_$target{sys_id}";

Makefile.in

@@ -28,8 +28,38 @@ INSTALL_PREFIX={- $config{install_prefix} -}
 
 # Do not edit these manually. Use Configure with --prefix or --openssldir
 # to change this!  Short explanation in the top comment in Configure
-INSTALLTOP={- $config{prefix} -}
-OPENSSLDIR={- $config{openssldir} -}
+INSTALLTOP={- # $prefix is used in the OPENSSLDIR perl snippet
+	      #
+	      our $prefix = $config{prefix} || "/usr/local";
+              $prefix -}
+OPENSSLDIR={- #
+	      # The logic here is that if no --openssldir was given,
+	      # OPENSSLDIR will get the value from $prefix plus "/ssl".
+	      # If --openssldir was given and the value is an absolute
+	      # path, OPENSSLDIR will get its value without change.
+	      # If the value from --openssldir is a relative path,
+	      # OPENSSLDIR will get $prefix with the --openssldir
+	      # value appended as a subdirectory.
+	      #
+              use File::Spec::Functions;
+              our $openssldir =
+                  $config{openssldir} ?
+                      (file_name_is_absolute($config{openssldir}) ?
+                           $config{openssldir}
+                           : catdir($prefix, $config{openssldir}))
+                      : catdir($prefix, "ssl");
+              $openssldir -}
+LIBDIR={- #
+          # if $prefix/lib$target{multilib} is not an existing
+          # directory, then assume that it's not searched by linker
+          # automatically, in which case adding $target{multilib} suffix
+          # causes more grief than we're ready to tolerate, so don't...
+          our $multilib =
+              -d "$prefix/lib$target{multilib}" ? $target{multilib} : "";
+          our $libdir = $config{libdir} || "lib$multilib";
+          $libdir -}
+ENGINESDIR={- use File::Spec::Functions;
+              catdir($prefix,$libdir,"engines") -}
 
 # NO_IDEA - Define to build without the IDEA algorithm
 # NO_RC4  - Define to build without the RC4 algorithm
@@ -60,7 +90,7 @@ OPENSSLDIR={- $config{openssldir} -}
 
 CROSS_COMPILE= {- $config{cross_compile_prefix} -}
 CC= $(CROSS_COMPILE){- $target{cc} -}
-CFLAG={- join(" ",map { "-D".$_} @{$config{defines}}) -} {- $config{cflags} -}
+CFLAG={- join(" ",(map { "-D".$_} @{$config{defines}}),"-DOPENSSLDIR=\"\\\"\$(OPENSSLDIR)\\\"\"","-DENGINESDIR=\"\\\"\$(ENGINESDIR)\\\"\"") -} {- $config{cflags} -}
 DEPFLAG= {- join(" ",map { "-D".$_} @{$config{depdefines}}) -}
 LDFLAG= {- $config{lflags} -}
 PLIB_LDFLAG= {- $config{plib_lflags} -}
@@ -75,7 +105,7 @@ PERL= {- $config{perl} -}
 RM= rm -f
 TAR= tar
 TARFLAGS= --no-recursion
-LIBDIR={- $config{libdir} -}
+MAKEDEPPROG=$(CROSS_COMPILE){- $config{makedepprog} -}
 
 # We let the C compiler driver to take care of .s files. This is done in
 # order to be excused from maintaining a separate set of architecture
@@ -160,7 +190,20 @@ LIBS=   libcrypto.a libssl.a
 SHARED_CRYPTO=libcrypto$(SHLIB_EXT)
 SHARED_SSL=libssl$(SHLIB_EXT)
 SHARED_LIBS={- '$(SHARED_CRYPTO) $(SHARED_SSL)' if (!$config{no_shared}) -}
-SHARED_LDFLAG={- $target{shared_ldflag} -}
+SHARED_LDFLAG={- $target{shared_ldflag}
+                 # Unlike other OSes (like Solaris, Linux, Tru64,
+                 # IRIX) BSD run-time linkers (tested OpenBSD, NetBSD
+                 # and FreeBSD) "demand" RPATH set on .so objects.
+                 # Apparently application RPATH is not global and
+                 # does not apply to .so linked with other .so.
+                 # Problem manifests itself when libssl.so fails to
+                 # load libcrypto.so. One can argue that we should
+                 # engrave this into Makefile.shared rules or into
+                 # BSD-* config lines above. Meanwhile let's try to
+                 # be cautious and pass -rpath to linker only when
+                 # $prefix is not /usr.
+                 . ($config{target} =~ m|^BSD-| && $prefix !~ m|^/usr/.*$|
+                    ? " -Wl,-rpath,\$\$(LIBRPATH)" : "") -}
 
 GENERAL=        Makefile
 BASENAME=       openssl

crypto/engine/eng_list.c

@@ -355,13 +355,8 @@ ENGINE *ENGINE_by_id(const char *id)
      * Prevent infinite recursion if we're looking for the dynamic engine.
      */
     if (strcmp(id, "dynamic")) {
-# ifdef OPENSSL_SYS_VMS
-        if ((load_dir = getenv("OPENSSL_ENGINES")) == 0)
-            load_dir = "SSLROOT:[ENGINES]";
-# else
         if ((load_dir = getenv("OPENSSL_ENGINES")) == 0)
             load_dir = ENGINESDIR;
-# endif
         iterator = ENGINE_by_id("dynamic");
         if (!iterator || !ENGINE_ctrl_cmd_string(iterator, "ID", id, 0) ||
             !ENGINE_ctrl_cmd_string(iterator, "DIR_LOAD", "2", 0) ||

include/openssl/opensslconf.h.in

@@ -123,13 +123,6 @@ EOF
 /* Generate 80386 code? */
 {- $config{processor} eq "386" ? "#define" : "#undef" -} I386_ONLY
 
-#if !defined(VMS) && !defined(__VMS) /* VMS uses logical names instead */
-# if defined(HEADER_CRYPTLIB_H) && !defined(OPENSSLDIR)
-#  define ENGINESDIR {- quotify1($config{enginesdir}) -}
-#  define OPENSSLDIR {- quotify1($config{openssldir}) -}
-# endif
-#endif
-
 #undef OPENSSL_UNISTD
 #define OPENSSL_UNISTD {- $target{unistd} -}