apps/cms.c: Correct -sign output and -verify input with -binary

Also add related warnings on irrelevant use of -nodetach and -content options.

Fixes #15347
Reviewed-by: NTomas Mraz <tomas@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/15358)

apps/cms.c

@@ -287,10 +287,11 @@ static void warn_binary(const char *file)
                 BIO_printf(bio_err, "Warning: input file '%s' contains %s"
                            " character; better use -binary option\n",
                            file, *cur == '\0' ? "NUL" : "8-bit");
-                break;
+                goto end;
             }
         }
     }
+ end:
     BIO_free(bio);
 }
 
@@ -320,7 +321,8 @@ int cms_main(int argc, char **argv)
     char *originatorfile = NULL, *recipfile = NULL, *ciphername = NULL;
     char *to = NULL, *from = NULL, *subject = NULL, *prog;
     cms_key_param *key_first = NULL, *key_param = NULL;
-    int flags = CMS_DETACHED, noout = 0, print = 0, keyidx = -1, vpmtouched = 0;
+    int flags = CMS_DETACHED, binary_files = 0;
+    int noout = 0, print = 0, keyidx = -1, vpmtouched = 0;
     int informat = FORMAT_SMIME, outformat = FORMAT_SMIME;
     int operation = 0, ret = 1, rr_print = 0, rr_allorfirst = -1;
     int verify_retcode = 0, rctformat = FORMAT_SMIME, keyform = FORMAT_UNDEF;
@@ -811,14 +813,26 @@ int cms_main(int argc, char **argv)
 
     ret = 2;
 
-    if (!(operation & SMIME_SIGNERS))
+    if ((operation & SMIME_SIGNERS) == 0) {
+        if ((flags & CMS_DETACHED) == 0)
+            BIO_printf(bio_err,
+                       "Warning: -nodetach option is ignored for non-signing operation\n");
+
         flags &= ~CMS_DETACHED;
+    }
+    if ((operation & SMIME_IP) == 0 && contfile != NULL)
+        BIO_printf(bio_err,
+                   "Warning: -contfile option is ignored for the given operation\n");
 
     if ((flags & CMS_BINARY) != 0) {
         if (!(operation & SMIME_OP))
             outformat = FORMAT_BINARY;
         if (!(operation & SMIME_IP))
             informat = FORMAT_BINARY;
+        if ((operation & SMIME_SIGNERS) != 0 && (flags & CMS_DETACHED) != 0)
+            binary_files = 1;
+        if ((operation & SMIME_IP) != 0 && contfile == NULL)
+            binary_files = 1;
     }
 
     if (operation == SMIME_ENCRYPT) {
@@ -902,7 +916,7 @@ int cms_main(int argc, char **argv)
     if ((flags & CMS_BINARY) == 0)
         warn_binary(infile);
     in = bio_open_default(infile, 'r',
-                          (flags & CMS_BINARY) != 0 ? FORMAT_BINARY : informat);
+                          binary_files ? FORMAT_BINARY : informat);
     if (in == NULL)
         goto end;
 
@@ -945,7 +959,8 @@ int cms_main(int argc, char **argv)
             goto end;
     }
 
-    out = bio_open_default(outfile, 'w', outformat);
+    out = bio_open_default(outfile, 'w',
+                           binary_files ? FORMAT_BINARY : outformat);
     if (out == NULL)
         goto end;
 

doc/man1/openssl-cms.pod.in

@@ -507,8 +507,8 @@ will be written to this file if the verification was successful.
 
 =item B<-content> I<filename>
 
-This specifies a file containing the detached content, this is only
-useful with the B<-verify> command. This is only usable if the CMS
+This specifies a file containing the detached content for operations taking
+S/MIME input, such as the B<-verify> command. This is only usable if the CMS
 structure is using the detached signature form where the content is
 not included. This option will override any content if the input format
 is S/MIME and it uses the multipart/signed MIME content type.

test/recipes/80-test_cms.t

@@ -818,7 +818,6 @@ subtest "CMS binary input tests\n" => sub {
     my $cert = srctop_file("test", "certs", "ee-self-signed.pem");
     my $key = srctop_file("test", "certs", "ee-key.pem");
 
-    plan skip_all => "Binary input tests currently disabled on Windows" if $^O =~ /^MSWin32$/;
     plan tests => 11;
 
     ok(run(app(["openssl", "cms", "-sign", "-md", "sha256",
@@ -829,29 +828,31 @@ subtest "CMS binary input tests\n" => sub {
                 "-binary", "-in", $signed, "-out", $verified])),
        "verify binary input with -binary");
     is(compare($input, $verified), 0, "binary input retained with -binary");
+
     ok(run(app(["openssl", "cms", "-sign", "-md", "sha256",
                 "-signer", $cert, "-inkey", $key,
-                "-in", $input, "-out", $signed])),
+                "-in", $input, "-out", $signed.".nobin"])),
        "sign binary input without -binary");
     ok(run(app(["openssl", "cms", "-verify", "-CAfile", $cert,
-                "-in", $signed, "-out", $verified])),
+                "-in", $signed.".nobin", "-out", $verified.".nobin"])),
        "verify binary input without -binary");
-    is(compare($input, $verified), 1, "binary input not retained without -binary");
+    is(compare($input, $verified.".nobin"), 1, "binary input not retained without -binary");
     ok(!run(app(["openssl", "cms", "-verify", "-CAfile", $cert, "-crlfeol",
-                "-binary", "-in", $signed, "-out", $verified])),
+                "-binary", "-in", $signed, "-out", $verified.".crlfeol"])),
        "verify binary input wrong crlfeol");
 
     ok(run(app(["openssl", "cms", "-sign", "-md", "sha256", "-crlfeol",
                 "-signer", $cert, "-inkey", $key,
                 "-binary", "-in", $input, "-out", $signed.".crlf"])),
-       "sign binary input crlfeol");
+       "sign binary input with -binary -crlfeol");
     ok(run(app(["openssl", "cms", "-verify", "-CAfile", $cert, "-crlfeol",
                 "-binary", "-in", $signed.".crlf", "-out", $verified.".crlf"])),
-       "verify binary input crlfeol");
-    is(compare($input, $verified.".crlf"), 0);
+       "verify binary input with -binary -crlfeol");
+    is(compare($input, $verified.".crlf"), 0,
+       "binary input retained with -binary -crlfeol");
     ok(!run(app(["openssl", "cms", "-verify", "-CAfile", $cert,
-                "-binary", "-in", $signed.".crlf", "-out", $verified.".crlf"])),
-       "verify binary input missing crlfeol");
+                "-binary", "-in", $signed.".crlf", "-out", $verified.".crlf2"])),
+       "verify binary input with -binary missing -crlfeol");
 };
 
 sub check_availability {