Make the handling of output and input formats consistent

Most of all, we needed to sort out which ones are binary and which
ones are text, and make sure they are treated accordingly and
consistently so
Reviewed-by: NTim Hudson <tjh@openssl.org>

apps/apps.h

@@ -163,8 +163,9 @@ int app_load_modules(const CONF *config);
 void unbuffer(FILE *fp);
 
 /* Often used in calls to bio_open_default. */
-# define RB(xformat)  ((xformat) == FORMAT_ASN1 ? "rb" : "r")
-# define WB(xformat)  ((xformat) == FORMAT_ASN1 ? "wb" : "w")
+# define RB(xformat)  (((xformat) & B_FORMAT_TEXT) ? "rb" : "r")
+# define WB(xformat)  (((xformat) & B_FORMAT_TEXT) ? "wb" : "w")
+# define AB(xformat)  (((xformat) & B_FORMAT_TEXT) ? "ab" : "a")
 
 /*
  * Common verification options.
@@ -535,14 +536,15 @@ void print_cert_checks(BIO *bio, X509 *x,
 void store_setup_crl_download(X509_STORE *st);
 
 /* See OPT_FMT_xxx, above. */
+# define B_FORMAT_TEXT   0x8000
 # define FORMAT_UNDEF    0
 # define FORMAT_ASN1     1
-# define FORMAT_TEXT     2
-# define FORMAT_PEM      3
+# define FORMAT_TEXT    (2 | B_FORMAT_TEXT)
+# define FORMAT_PEM     (3 | B_FORMAT_TEXT)
 # define FORMAT_PKCS12   5
-# define FORMAT_SMIME    6
+# define FORMAT_SMIME   (6 | B_FORMAT_TEXT)
 # define FORMAT_ENGINE   7
-# define FORMAT_PEMRSA   9      /* PEM RSAPubicKey format */
+# define FORMAT_PEMRSA  (9 | B_FORMAT_TEXT)     /* PEM RSAPubicKey format */
 # define FORMAT_ASN1RSA  10                     /* DER RSAPubicKey format */
 # define FORMAT_MSBLOB   11                     /* MS Key blob format */
 # define FORMAT_PVK      12                     /* MS PVK file format */

apps/asn1pars.c

@@ -197,7 +197,7 @@ int asn1parse_main(int argc, char **argv)
         BIO_free(in);
     }
 
-    if ((in = bio_open_default(infile, "r")) == NULL)
+    if ((in = bio_open_default(infile, RB(informat))) == NULL)
         goto end;
 
     if (derfile && (derout = bio_open_default(derfile, "wb")) == NULL)

apps/cms.c

@@ -690,16 +690,14 @@ int cms_main(int argc, char **argv)
         flags &= ~CMS_DETACHED;
 
     if (operation & SMIME_OP) {
-        if (outformat == FORMAT_ASN1)
-            outmode = "wb";
+        outmode = WB(outformat);
     } else {
         if (flags & CMS_BINARY)
             outmode = "wb";
     }
 
     if (operation & SMIME_IP) {
-        if (informat == FORMAT_ASN1)
-            inmode = "rb";
+        inmode = RB(informat);
     } else {
         if (flags & CMS_BINARY)
             inmode = "rb";

apps/crl.c

@@ -346,7 +346,7 @@ int crl_main(int argc, char **argv)
             }
         }
     }
-    out = bio_open_default(outfile, "w");
+    out = bio_open_default(outfile, WB(outformat));
     if (out == NULL)
         goto end;
 

apps/dhparam.c

@@ -352,7 +352,7 @@ int dhparam_main(int argc, char **argv)
         /* dh != NULL */
     }
 
-    out = bio_open_default(outfile, "w");
+    out = bio_open_default(outfile, WB(outformat));
     if (out == NULL)
         goto end;
 

apps/dsa.c

@@ -225,7 +225,7 @@ int dsa_main(int argc, char **argv)
         goto end;
     }
 
-    out = bio_open_owner(outfile, "w", private);
+    out = bio_open_owner(outfile, WB(outformat), private);
     if (out == NULL)
         goto end;
 

apps/dsaparam.c

@@ -195,10 +195,10 @@ int dsaparam_main(int argc, char **argv)
     }
     private = genkey ? 1 : 0;
 
-    in = bio_open_default(infile, "r");
+    in = bio_open_default(infile, RB(informat));
     if (in == NULL)
         goto end;
-    out = bio_open_owner(outfile, "w", private);
+    out = bio_open_owner(outfile, WB(outformat), private);
     if (out == NULL)
         goto end;
 

apps/enc.c

@@ -330,7 +330,7 @@ int enc_main(int argc, char **argv)
         unbuffer(stdin);
         in = dup_bio_in();
     } else
-        in = bio_open_default(infile, "r");
+        in = bio_open_default(infile, base64 ? "r" : "rb");
     if (in == NULL)
         goto end;
 
@@ -366,7 +366,7 @@ int enc_main(int argc, char **argv)
         }
     }
 
-    out = bio_open_default(outfile, "w");
+    out = bio_open_default(outfile, base64 ? "w" : "wb");
     if (out == NULL)
         goto end;
 

apps/genpkey.c

@@ -184,7 +184,7 @@ int genpkey_main(int argc, char **argv)
     if (!app_load_modules(NULL))
         goto end;
 
-    out = bio_open_owner(outfile, "wb", private);
+    out = bio_open_owner(outfile, WB(outformat), private);
     if (out == NULL)
         goto end;
 

apps/pkcs8.c

@@ -239,10 +239,10 @@ int pkcs8_main(int argc, char **argv)
     if ((pbe_nid == -1) && !cipher)
         pbe_nid = NID_pbeWithMD5AndDES_CBC;
 
-    in = bio_open_default(infile, "rb");
+    in = bio_open_default(infile, RB(informat));
     if (in == NULL)
         goto end;
-    out = bio_open_owner(outfile, "wb", private);
+    out = bio_open_owner(outfile, WB(outformat), private);
     if (out == NULL)
         goto end;
 

apps/pkey.c

@@ -172,7 +172,7 @@ int pkey_main(int argc, char **argv)
     if (!app_load_modules(NULL))
         goto end;
 
-    out = bio_open_owner(outfile, "wb", private);
+    out = bio_open_owner(outfile, WB(outformat), private);
     if (out == NULL)
         goto end;
 

apps/rand.c

@@ -134,7 +134,7 @@ int rand_main(int argc, char **argv)
         BIO_printf(bio_err, "%ld semi-random bytes loaded\n",
                    app_RAND_load_files(inrand));
 
-    out = bio_open_default(outfile, "w");
+    out = bio_open_default(outfile, base64 ? "w" : "wb");
     if (out == NULL)
         goto end;
 

apps/req.c

@@ -764,7 +764,8 @@ int req_main(int argc, char **argv)
 
     out = bio_open_default(outfile,
                            keyout != NULL && outfile != NULL &&
-                           strcmp(keyout, outfile) == 0 ? "a" : "w");
+                           strcmp(keyout, outfile) == 0
+                           ? AB(outformat) : WB(outformat));
     if (out == NULL)
         goto end;
 

apps/rsa.c

@@ -292,7 +292,7 @@ int rsa_main(int argc, char **argv)
         goto end;
     }
 
-    out = bio_open_owner(outfile, "w", private);
+    out = bio_open_owner(outfile, WB(outformat), private);
     if (out == NULL)
         goto end;
 

apps/sess_id.c

@@ -160,10 +160,7 @@ int sess_id_main(int argc, char **argv)
     }
 
     if (!noout || text) {
-        const char* modeflag = "w";
-        if (outformat == FORMAT_ASN1 || outformat == FORMAT_NSS)
-            modeflag = "wb";
-        out = bio_open_default(outfile, modeflag);
+        out = bio_open_default(outfile, WB(outformat));
         if (out == NULL)
             goto end;
     }

apps/smime.c

@@ -427,16 +427,14 @@ int smime_main(int argc, char **argv)
         flags &= ~PKCS7_DETACHED;
 
     if (operation & SMIME_OP) {
-        if (outformat == FORMAT_ASN1)
-            outmode = "wb";
+        outmode = WB(outformat);
     } else {
         if (flags & PKCS7_BINARY)
             outmode = "wb";
     }
 
     if (operation & SMIME_IP) {
-        if (informat == FORMAT_ASN1)
-            inmode = "rb";
+        inmode = RB(informat);
     } else {
         if (flags & PKCS7_BINARY)
             inmode = "rb";

apps/x509.c

@@ -496,7 +496,7 @@ int x509_main(int argc, char **argv)
     if (!app_load_modules(NULL))
         goto end;
 
-    out = bio_open_default(outfile, "w");
+    out = bio_open_default(outfile, WB(outformat));
     if (out == NULL)
         goto end;
 
@@ -556,7 +556,7 @@ int x509_main(int argc, char **argv)
             BIO_printf(bio_err, "We need a private key to sign with\n");
             goto end;
         }
-        in = bio_open_default(infile, "r");
+        in = bio_open_default(infile, RB(informat));
         if (in == NULL)
             goto end;
         req = PEM_read_bio_X509_REQ(in, NULL, NULL, NULL);