Streamline dependency generation

It seems that only gcc -MMD produces dependency files that are "sane"
for our needs.  For all other methods, some post processing is needed:

- 'makedepend' (Unix) insists that object files are located in the
  same spot as the source file.
- 'cl /Zs /showIncludes' (Visual C) has "Note: including file: " where
  we'd like to see the object.
- 'CC/DECC' (VMS) insists that the object file is located in the
  current directory, i.e. it strips away all directory information.

So far, we've managed this (except for the VMS case) with individual
uncommented perl command lines directly in the build file template.
We're now collecting these diverse hacks into one perl script that
takes an argument to tell what kind of input to expect and that
massages whatever it gets on STDIN and outputs the result on STDOUT.
Reviewed-by: NAndy Polyakov <appro@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/5591)

Configurations/descrip.mms.tmpl

@@ -3,6 +3,7 @@
 ## {- join("\n## ", @autowarntext) -}
 {-
   use File::Spec::Functions qw/:DEFAULT abs2rel rel2abs/;
+  use File::Basename;
 
   # Our prefix, claimed when speaking with the VSI folks Tuesday
   # January 26th 2016
@@ -878,6 +879,9 @@ EOF
       my $incs_off = join("\n\t\@ ", @{$incs_cmds[1]}) || '!';
       my $depbuild = $disabled{makedepend} ? ""
           : " /MMS=(FILE=${objd}${objn}.tmp-D,TARGET=$obj.OBJ)";
+      my $postprocess_makedepend =
+          sourcefile("util", "postprocess-makedepend.pl");
+      my $objdir = dirname($obj);
 
       return <<"EOF"
 $obj.OBJ : $deps
@@ -891,9 +895,8 @@ $obj.OBJ : $deps
         - PURGE $obj.OBJ
 EOF
       . ($disabled{makedepend} ? "" : <<"EOF"
-        \@ PIPE ( \$(PERL) -e "use File::Compare qw/compare_text/; my \$x = compare_text(""$obj.D"",""$obj.tmp-D""); exit(0x10000000 + (\$x == 0));" || -
-                 RENAME $obj.tmp-D $obj.d )
-        \@ IF F\$SEARCH("$obj.tmp-D") .NES. "" THEN DELETE $obj.tmp-D;*
+        \$(PERL) $postprocess_makedepend "VMS C" $objdir < $obj.tmp-D > $obj.d
+        - DELETE $obj.tmp-D;*
 EOF
         );
   }

Configurations/unix-Makefile.tmpl

@@ -1032,14 +1032,9 @@ $obj$objext: $deps
 EOF
           if (defined $makedepprog  && $makedepprog =~ /\/makedepend/) {
               $recipe .= <<"EOF";
-	-\$(MAKEDEPEND) -f- -o"|\$\@" -- $incs $cmdflags -- $srcs \\
-	    >$obj$depext.tmp 2>/dev/null
-	-\$(PERL) -i -pe 's/^.*\\|//; s/ \\/(\\\\.|[^ ])*//; \$\$_ = undef if (/: *\$\$/ || /^(#.*| *)\$\$/); \$\$_.="\\n" unless !defined(\$\$_) or /\\R\$\$/g;' $obj$depext.tmp
-	\@if cmp $obj$depext.tmp $obj$depext > /dev/null 2> /dev/null; then \\
-		rm -f $obj$depext.tmp; \\
-	else \\
-		mv $obj$depext.tmp $obj$depext; \\
-	fi
+	\$(MAKEDEPEND) -f- -o"|\$\@" -- $incs $cmdflags -- $srcs 2>/dev/null \\
+	    | \$(PERL) \$(SRCDIR)/util/postprocess-makedepend.pl \\
+	               'makedepend' > $obj$depext
 EOF
           }
       }

Configurations/windows-makefile.tmpl

@@ -599,13 +599,9 @@ EOF
      }
      return <<"EOF"	if (!$disabled{makedepend});
 $obj$depext: $deps
-	\$(CC) $cflags /Zs /showIncludes $srcs 2>&1 | \\
-	    "\$(PERL)" -n << > $obj$depext
-chomp;
-s/^Note: including file: *//;
-\$\$collect{\$\$_} = 1;
-END { print '$obj$objext: ',join(" ", sort keys \%collect),"\\n" }
-<<
+	\$(CC) $cflags /Zs /showIncludes $srcs 2>&1 \\
+	    | "\$(PERL)" "\$(SRCDIR)\\util\\postprocess-makedepend.pl" \\
+	                 "VC" "$obj$objext" > $obj$depext
 $obj$objext: $obj$depext
 	\$(CC) $cflags -c \$(COUTFLAG)\$\@ @<<
 $srcs

util/postprocess-makedepend.pl

+#! /usr/bin/env perl
+# Copyright 2018 The OpenSSL Project Authors. All Rights Reserved.
+#
+# Licensed under the OpenSSL license (the "License").  You may not use
+# this file except in compliance with the License.  You can obtain a copy
+# in the file LICENSE in the source distribution or at
+# https://www.openssl.org/source/license.html
+
+use strict;
+use warnings;
+
+my $producer = shift @ARGV;
+
+die "Producer not given\n" unless $producer;
+
+my $procedure = {
+    'makedepend' =>
+        sub {
+            my $line = shift;
+
+            # makedepend, in its infinite wisdom, wants to have the object file
+            # in the same directory as the source file.  This doesn't work too
+            # well with out-of-source-tree builds, so we must resort to tricks
+            # to get things right.  The trick is to call makedepend with an
+            # extra suffix that contains the desired object file path, like
+            # this:
+            #
+            #   makedepend -f- -o"|dir/foo.o" -- $(CFLAGS) -- ../some/foo.c
+            #
+            # The result will look something like this:
+            #
+            #   ../somewhere/foo|dir/foo.o: deps...
+            #
+            # Which is easy to massage by removing everything up to the first |
+
+            # Remove everything up to the first |
+            $line =~ s/^.*\|//;
+            # Also, remove any dependency that starts with a /, because those
+            # are typically system headers
+            $line =~ s/\s+\/(\\.|\S)*//g;
+            # Finally, discard all empty lines or comment lines
+            return undef if $line =~ /:\s*$/ || $line =~ /^(#.*|\s*)$/;
+
+            $line.="\n" unless $line =~ /\R$/g;
+
+            return $line;
+        },
+    'VMS C' =>
+        sub {
+            my $line = shift;
+
+            # current versions of DEC / Compaq / HP / VSI C strips away all
+            # directory information from the object file, so we must insert it
+            # back. Just to be safe against future changes, we check that there
+            # really is no directory information.
+            my $directory = shift;
+
+            # The pattern for target and dependencies will always take this
+            # form:
+            #
+            #   target SPACE : SPACE deps
+            #
+            # This is so a volume delimiter (a : without any spaces around it)
+            # won't get mixed up with the target / deps delimiter.  We use this
+            # fact in the regexp below to make sure we do look at the target.
+            $line =~ s/^/$directory/ unless /^\S+[:>\]]\S+\s+:/;
+
+            # We know that VMS has system header files in text libraries,
+            # extension .TLB.  We also know that our header files aren't stored
+            # in text libraries.  Finally, we know that VMS C produces exactly
+            # one dependency per line, so we simply discard any line ending with
+            # .TLB.
+            return undef if /\.TLB\s*$/;
+
+            return $line;
+        },
+    'VC' =>
+        sub {
+            my $line = shift;
+            my $object = shift;
+
+            # For the moment, we only support Visual C on native Windows, or
+            # compatible compilers.  With those, the flags /Zs /showIncludes
+            # give us the necessary output to be able to create dependencies
+            # that nmake (or any 'make' implementation) should be able to read,
+            # with a bit of help.  The output we're interested in looks like
+            # this (it always starts the same)
+            #
+            #   Note: including file: {whatever header file}
+            #
+            # So all we really have to do is to is to replace the start of the
+            # line with an object file specification, given to us as an extra
+            # argument (passed from $ARGV[1]);
+            #
+            # There are also other lines mixed in, for example compiler
+            # warnings, so we simply discard anything that doesn't start with
+            # the Note:
+
+            if (/^Note: including file: */) {
+                (my $tail = $') =~ s/\s*\R$//;
+                return "${object}: \"$tail\"\n";
+            }
+
+            return undef;
+        },
+} -> {$producer};
+
+die "Producer unrecognised: $producer\n" unless defined $procedure;
+
+while (<STDIN>) {
+    if ($_ = $procedure->($_, @ARGV)) {
+        print or die "$!\n";
+    }
+}