Change the way apps open their input and output files

The different apps had the liberty to decide whether they would open their
input and output files in binary mode or not, which could be confusing if
two different apps were handling the same type of file in different ways.

The solution is to centralise the decision of low level file organisation,
and that the apps would use a selection of formats to state the intent of
the file.
Reviewed-by: NTim Hudson <tjh@openssl.org>

apps/apps.c

@@ -514,7 +514,7 @@ CONF *app_load_config(const char *filename)
     BIO *in;
     CONF *conf;
 
-    in = bio_open_default(filename, "r");
+    in = bio_open_default(filename, 'r', FORMAT_TEXT);
     if (in == NULL)
         return NULL;
 
@@ -527,7 +527,7 @@ CONF *app_load_config_quiet(const char *filename)
     BIO *in;
     CONF *conf;
 
-    in = bio_open_default_quiet(filename, "r");
+    in = bio_open_default_quiet(filename, 'r', FORMAT_TEXT);
     if (in == NULL)
         return NULL;
 
@@ -683,7 +683,7 @@ X509 *load_cert(const char *file, int format,
         unbuffer(stdin);
         cert = dup_bio_in();
     } else
-        cert = bio_open_default(file, RB(format));
+        cert = bio_open_default(file, 'r', format);
     if (cert == NULL)
         goto end;
 
@@ -718,7 +718,7 @@ X509_CRL *load_crl(const char *infile, int format)
         return x;
     }
 
-    in = bio_open_default(infile, RB(format));
+    in = bio_open_default(infile, 'r', format);
     if (in == NULL)
         goto end;
     if (format == FORMAT_ASN1)
@@ -772,7 +772,7 @@ EVP_PKEY *load_key(const char *file, int format, int maybe_stdin,
         unbuffer(stdin);
         key = dup_bio_in();
     } else
-        key = bio_open_default(file, RB(format));
+        key = bio_open_default(file, 'r', format);
     if (key == NULL)
         goto end;
     if (format == FORMAT_ASN1) {
@@ -808,13 +808,6 @@ EVP_PKEY *load_key(const char *file, int format, int maybe_stdin,
     return (pkey);
 }
 
-static const char *key_file_format(int format)
-{
-    if (format == FORMAT_PEM || format == FORMAT_PEMRSA)
-        return "r";
-    return "rb";
-}
-
 EVP_PKEY *load_pubkey(const char *file, int format, int maybe_stdin,
                       const char *pass, ENGINE *e, const char *key_descrip)
 {
@@ -842,7 +835,7 @@ EVP_PKEY *load_pubkey(const char *file, int format, int maybe_stdin,
         unbuffer(stdin);
         key = dup_bio_in();
     } else
-        key = bio_open_default(file, key_file_format(format));
+        key = bio_open_default(file, 'r', format);
     if (key == NULL)
         goto end;
     if (format == FORMAT_ASN1) {
@@ -909,7 +902,7 @@ static int load_certs_crls(const char *file, int format,
         return 0;
     }
 
-    bio = bio_open_default(file, "r");
+    bio = bio_open_default(file, 'r', FORMAT_PEM);
     if (bio == NULL)
         return 0;
 

apps/apps.h

@@ -154,19 +154,14 @@ extern BIO *bio_out;
 extern BIO *bio_err;
 BIO *dup_bio_in(void);
 BIO *dup_bio_out(void);
-BIO *bio_open_owner(const char *filename, const char *mode, int private);
-BIO *bio_open_default(const char *filename, const char *mode);
-BIO *bio_open_default_quiet(const char *filename, const char *mode);
+BIO *bio_open_owner(const char *filename, int format, int private);
+BIO *bio_open_default(const char *filename, char mode, int format);
+BIO *bio_open_default_quiet(const char *filename, char mode, int format);
 CONF *app_load_config(const char *filename);
 CONF *app_load_config_quiet(const char *filename);
 int app_load_modules(const CONF *config);
 void unbuffer(FILE *fp);
 
-/* Often used in calls to bio_open_default. */
-# define RB(xformat)  (((xformat) & B_FORMAT_TEXT) ? "rb" : "r")
-# define WB(xformat)  (((xformat) & B_FORMAT_TEXT) ? "wb" : "w")
-# define AB(xformat)  (((xformat) & B_FORMAT_TEXT) ? "ab" : "a")
-
 /*
  * Common verification options.
  */
@@ -536,14 +531,21 @@ void print_cert_checks(BIO *bio, X509 *x,
 void store_setup_crl_download(X509_STORE *st);
 
 /* See OPT_FMT_xxx, above. */
+/* On some platforms, it's important to distinguish between text and binary
+ * files.  On some, there might even be specific file formats for different
+ * contents.  The FORMAT_xxx macros are meant to express an intent with the
+ * file being read or created.
+ */
 # define B_FORMAT_TEXT   0x8000
 # define FORMAT_UNDEF    0
-# define FORMAT_ASN1     1
-# define FORMAT_TEXT    (2 | B_FORMAT_TEXT)
-# define FORMAT_PEM     (3 | B_FORMAT_TEXT)
-# define FORMAT_PKCS12   5
-# define FORMAT_SMIME   (6 | B_FORMAT_TEXT)
-# define FORMAT_ENGINE   7
+# define FORMAT_TEXT    (1 | B_FORMAT_TEXT)     /* Generic text */
+# define FORMAT_BINARY   2                      /* Generic binary */
+# define FORMAT_BASE64  (3 | B_FORMAT_TEXT)     /* Base64 */
+# define FORMAT_ASN1     4                      /* ASN.1/DER */
+# define FORMAT_PEM     (5 | B_FORMAT_TEXT)
+# define FORMAT_PKCS12   6
+# define FORMAT_SMIME   (7 | B_FORMAT_TEXT)
+# define FORMAT_ENGINE   8                      /* Not really a file format */
 # define FORMAT_PEMRSA  (9 | B_FORMAT_TEXT)     /* PEM RSAPubicKey format */
 # define FORMAT_ASN1RSA  10                     /* DER RSAPubicKey format */
 # define FORMAT_MSBLOB   11                     /* MS Key blob format */

apps/asn1pars.c

@@ -190,17 +190,17 @@ int asn1parse_main(int argc, char **argv)
         goto end;
 
     if (oidfile != NULL) {
-      in = bio_open_default(oidfile, "r");
+        in = bio_open_default(oidfile, 'r', FORMAT_TEXT);
         if (in == NULL)
             goto end;
         OBJ_create_objects(in);
         BIO_free(in);
     }
 
-    if ((in = bio_open_default(infile, RB(informat))) == NULL)
+    if ((in = bio_open_default(infile, 'r', informat)) == NULL)
         goto end;
 
-    if (derfile && (derout = bio_open_default(derfile, "wb")) == NULL)
+    if (derfile && (derout = bio_open_default(derfile, 'w', FORMAT_ASN1)) == NULL)
         goto end;
 
     if (strictpem) {

apps/ca.c

@@ -795,7 +795,8 @@ end_of_options:
 
     /*****************************************************************/
     if (req || gencrl) {
-        Sout = bio_open_default(outfile, "w");
+        /* FIXME: Is it really always text? */
+        Sout = bio_open_default(outfile, 'w', FORMAT_TEXT);
         if (Sout == NULL)
             goto end;
     }

apps/cms.c

@@ -247,7 +247,6 @@ int cms_main(int argc, char **argv)
         NULL;
     char *to = NULL, *from = NULL, *subject = NULL, *prog;
     cms_key_param *key_first = NULL, *key_param = NULL;
-    const char *inmode = "r", *outmode = "w";
     int flags = CMS_DETACHED, noout = 0, print = 0, keyidx = -1, vpmtouched =
         0;
     int informat = FORMAT_SMIME, outformat = FORMAT_SMIME;
@@ -689,18 +688,14 @@ int cms_main(int argc, char **argv)
     if (!(operation & SMIME_SIGNERS))
         flags &= ~CMS_DETACHED;
 
-    if (operation & SMIME_OP) {
-        outmode = WB(outformat);
-    } else {
+    if (!(operation & SMIME_OP)) {
         if (flags & CMS_BINARY)
-            outmode = "wb";
+            outformat = FORMAT_BINARY;
     }
 
-    if (operation & SMIME_IP) {
-        inmode = RB(informat);
-    } else {
+    if (!(operation & SMIME_IP)) {
         if (flags & CMS_BINARY)
-            inmode = "rb";
+            informat = FORMAT_BINARY;
     }
 
     if (operation == SMIME_ENCRYPT) {
@@ -770,7 +765,7 @@ int cms_main(int argc, char **argv)
             goto end;
     }
 
-    in = bio_open_default(infile, inmode);
+    in = bio_open_default(infile, 'r', informat);
     if (in == NULL)
         goto end;
 
@@ -834,7 +829,7 @@ int cms_main(int argc, char **argv)
         }
     }
 
-    out = bio_open_default(outfile, outmode);
+    out = bio_open_default(outfile, 'w', outformat);
     if (out == NULL)
         goto end;
 

apps/crl.c

@@ -346,7 +346,7 @@ int crl_main(int argc, char **argv)
             }
         }
     }
-    out = bio_open_default(outfile, WB(outformat));
+    out = bio_open_default(outfile, 'w', outformat);
     if (out == NULL)
         goto end;
 

apps/crl2p7.c

@@ -152,7 +152,7 @@ int crl2pkcs7_main(int argc, char **argv)
         goto end;
 
     if (!nocrl) {
-        in = bio_open_default(infile, RB(informat));
+        in = bio_open_default(infile, 'r', informat);
         if (in == NULL)
             goto end;
 
@@ -201,7 +201,7 @@ int crl2pkcs7_main(int argc, char **argv)
 
     sk_OPENSSL_STRING_free(certflst);
 
-    out = bio_open_default(outfile, WB(outformat));
+    out = bio_open_default(outfile, 'w', outformat);
     if (out == NULL)
         goto end;
 

apps/dgst.c

@@ -275,7 +275,7 @@ int dgst_main(int argc, char **argv)
     if (randfile)
         app_RAND_load_file(randfile, 0);
 
-    out = bio_open_default(outfile, out_bin ? "wb" : "w");
+    out = bio_open_default(outfile, 'w', out_bin ? FORMAT_BINARY : FORMAT_TEXT);
     if (out == NULL)
         goto end;
 

apps/dhparam.c

@@ -309,7 +309,7 @@ int dhparam_main(int argc, char **argv)
         app_RAND_write_file(NULL);
     } else {
 
-        in = bio_open_default(infile, RB(informat));
+        in = bio_open_default(infile, 'r', informat);
         if (in == NULL)
             goto end;
 
@@ -352,7 +352,7 @@ int dhparam_main(int argc, char **argv)
         /* dh != NULL */
     }
 
-    out = bio_open_default(outfile, WB(outformat));
+    out = bio_open_default(outfile, 'w', outformat);
     if (out == NULL)
         goto end;
 

apps/dsa.c

@@ -225,7 +225,7 @@ int dsa_main(int argc, char **argv)
         goto end;
     }
 
-    out = bio_open_owner(outfile, WB(outformat), private);
+    out = bio_open_owner(outfile, outformat, private);
     if (out == NULL)
         goto end;
 

apps/dsaparam.c

@@ -195,10 +195,10 @@ int dsaparam_main(int argc, char **argv)
     }
     private = genkey ? 1 : 0;
 
-    in = bio_open_default(infile, RB(informat));
+    in = bio_open_default(infile, 'r', informat);
     if (in == NULL)
         goto end;
-    out = bio_open_owner(outfile, WB(outformat), private);
+    out = bio_open_owner(outfile, outformat, private);
     if (out == NULL)
         goto end;
 

apps/ec.c

@@ -205,7 +205,7 @@ int ec_main(int argc, char **argv)
     if (!app_load_modules(NULL))
         goto end;
 
-    in = bio_open_default(infile, RB(informat));
+    in = bio_open_default(infile, 'r', informat);
     if (in == NULL)
         goto end;
 
@@ -227,7 +227,7 @@ int ec_main(int argc, char **argv)
         goto end;
     }
 
-    out = bio_open_owner(outfile, WB(outformat), private);
+    out = bio_open_owner(outfile, outformat, private);
     if (out == NULL)
         goto end;
 

apps/ecparam.c

@@ -223,10 +223,10 @@ int ecparam_main(int argc, char **argv)
     if (!app_load_modules(NULL))
         goto end;
 
-    in = bio_open_default(infile, RB(informat));
+    in = bio_open_default(infile, 'r', informat);
     if (in == NULL)
         goto end;
-    out = bio_open_owner(outfile, WB(outformat), private);
+    out = bio_open_owner(outfile, outformat, private);
     if (out == NULL)
         goto end;
 

apps/enc.c

@@ -138,7 +138,7 @@ int enc_main(int argc, char **argv)
     char mbuf[sizeof magic - 1];
     OPTION_CHOICE o;
     int bsize = BSIZE, verbose = 0, debug = 0, olb64 = 0, nosalt = 0;
-    int enc = 1, printkey = 0, i, k, base64 = 0;
+    int enc = 1, printkey = 0, i, k, format = FORMAT_BINARY;
     int ret = 1, inl, nopad = 0, non_fips_allow = 0;
     unsigned char key[EVP_MAX_KEY_LENGTH], iv[EVP_MAX_IV_LENGTH];
     unsigned char *buff = NULL, salt[PKCS5_SALT_LEN];
@@ -151,7 +151,7 @@ int enc_main(int argc, char **argv)
     /* first check the program name */
     prog = opt_progname(argv[0]);
     if (strcmp(prog, "base64") == 0)
-        base64 = 1;
+        format = FORMAT_BASE64;
 #ifdef ZLIB
     else if (strcmp(prog, "zlib") == 0)
         do_zlib = 1;
@@ -223,7 +223,7 @@ int enc_main(int argc, char **argv)
             olb64 = 1;
             break;
         case OPT_A:
-            base64 = 1;
+            format = FORMAT_BASE64;
             break;
         case OPT_Z:
 #ifdef ZLIB
@@ -246,7 +246,7 @@ int enc_main(int argc, char **argv)
             str = opt_arg();
             break;
         case OPT_KFILE:
-            in = bio_open_default(opt_arg(), "r");
+            in = bio_open_default(opt_arg(), 'r', FORMAT_TEXT);
             if (in == NULL)
                 goto opthelp;
             i = BIO_gets(in, buf, sizeof buf);
@@ -311,7 +311,7 @@ int enc_main(int argc, char **argv)
         dgst = EVP_md5();
 
     /* It must be large enough for a base64 encoded line */
-    if (base64 && bsize < 80)
+    if (format == FORMAT_BASE64 && bsize < 80)
         bsize = 80;
     if (verbose)
         BIO_printf(bio_err, "bufsize=%d\n", bsize);
@@ -330,7 +330,7 @@ int enc_main(int argc, char **argv)
         unbuffer(stdin);
         in = dup_bio_in();
     } else
-        in = bio_open_default(infile, base64 ? "r" : "rb");
+        in = bio_open_default(infile, 'r', format);
     if (in == NULL)
         goto end;
 
@@ -366,7 +366,7 @@ int enc_main(int argc, char **argv)
         }
     }
 
-    out = bio_open_default(outfile, base64 ? "w" : "wb");
+    out = bio_open_default(outfile, 'w', format);
     if (out == NULL)
         goto end;
 
@@ -384,7 +384,7 @@ int enc_main(int argc, char **argv)
     }
 #endif
 
-    if (base64) {
+    if (format == FORMAT_BASE64) {
         if ((b64 = BIO_new(BIO_f_base64())) == NULL)
             goto end;
         if (debug) {

apps/gendsa.c

@@ -147,7 +147,7 @@ int gendsa_main(int argc, char **argv)
     if (!app_load_modules(NULL))
         goto end;
 
-    in = bio_open_default(dsaparams, "r");
+    in = bio_open_default(dsaparams, 'r', FORMAT_PEM);
     if (in == NULL)
         goto end2;
 
@@ -158,7 +158,7 @@ int gendsa_main(int argc, char **argv)
     BIO_free(in);
     in = NULL;
 
-    out = bio_open_owner(outfile, "w", private);
+    out = bio_open_owner(outfile, FORMAT_PEM, private);
     if (out == NULL)
         goto end2;
 

apps/genpkey.c

@@ -184,7 +184,7 @@ int genpkey_main(int argc, char **argv)
     if (!app_load_modules(NULL))
         goto end;
 
-    out = bio_open_owner(outfile, WB(outformat), private);
+    out = bio_open_owner(outfile, outformat, private);
     if (out == NULL)
         goto end;
 

apps/genrsa.c

@@ -172,7 +172,7 @@ int genrsa_main(int argc, char **argv)
     if (!app_load_modules(NULL))
         goto end;
 
-    out = bio_open_owner(outfile, "w", private);
+    out = bio_open_owner(outfile, FORMAT_PEM, private);
     if (out == NULL)
         goto end;
 

apps/nseq.c

@@ -112,10 +112,10 @@ int nseq_main(int argc, char **argv)
     if (!app_load_modules(NULL))
         goto end;
 
-    in = bio_open_default(infile, "r");
+    in = bio_open_default(infile, 'r', FORMAT_PEM);
     if (in == NULL)
         goto end;
-    out = bio_open_default(outfile, "w");
+    out = bio_open_default(outfile, 'w', FORMAT_PEM);
     if (out == NULL)
         goto end;
 

apps/ocsp.c

@@ -486,7 +486,7 @@ int ocsp_main(int argc, char **argv)
     if (!app_load_modules(NULL))
         goto end;
 
-    out = bio_open_default(outfile, "w");
+    out = bio_open_default(outfile, 'w', FORMAT_TEXT);
     if (out == NULL)
         goto end;
 
@@ -494,7 +494,7 @@ int ocsp_main(int argc, char **argv)
         add_nonce = 0;
 
     if (!req && reqin) {
-        derbio = bio_open_default(reqin, "rb");
+        derbio = bio_open_default(reqin, 'r', FORMAT_ASN1);
         if (derbio == NULL)
             goto end;
         req = d2i_OCSP_REQUEST_bio(derbio, NULL);
@@ -589,7 +589,7 @@ int ocsp_main(int argc, char **argv)
         OCSP_REQUEST_print(out, req, 0);
 
     if (reqout) {
-        derbio = bio_open_default(reqout, "wb");
+        derbio = bio_open_default(reqout, 'w', FORMAT_ASN1);
         if (derbio == NULL)
             goto end;
         i2d_OCSP_REQUEST_bio(derbio, req);
@@ -627,7 +627,7 @@ int ocsp_main(int argc, char **argv)
         goto end;
 # endif
     } else if (respin) {
-        derbio = bio_open_default(respin, "rb");
+        derbio = bio_open_default(respin, 'r', FORMAT_ASN1);
         if (derbio == NULL)
             goto end;
         resp = d2i_OCSP_RESPONSE_bio(derbio, NULL);
@@ -644,7 +644,7 @@ int ocsp_main(int argc, char **argv)
  done_resp:
 
     if (respout) {
-        derbio = bio_open_default(respout, "wb");
+        derbio = bio_open_default(respout, 'w', FORMAT_ASN1);
         if (derbio == NULL)
             goto end;
         i2d_OCSP_RESPONSE_bio(derbio, resp);

apps/openssl.c

@@ -299,17 +299,46 @@ void unbuffer(FILE *fp)
     setbuf(fp, NULL);
 }
 
+static const char *modestr(char mode, int format)
+{
+    OPENSSL_assert(mode == 'a' || mode == 'r' || mode == 'w');
+
+    switch (mode) {
+    case 'a':
+        return (format) & B_FORMAT_TEXT ? "ab" : "a";
+    case 'r':
+        return (format) & B_FORMAT_TEXT ? "rb" : "r";
+    case 'w':
+        return (format) & B_FORMAT_TEXT ? "wb" : "w";
+    }
+    /* The assert above should make sure we never reach this point */
+    return NULL;
+}
+
+static const char *modeverb(char mode)
+{
+    switch (mode) {
+    case 'a':
+        return "appending";
+    case 'r':
+        return "reading";
+    case 'w':
+        return "writing";
+    }
+    return "(doing something)";
+}
+
 /*
  * Open a file for writing, owner-read-only.
  */
-BIO *bio_open_owner(const char *filename, const char *modestr, int private)
+BIO *bio_open_owner(const char *filename, int format, int private)
 {
     FILE *fp = NULL;
     BIO *b = NULL;
     int fd = -1, bflags, mode, binmode;
 
     if (!private || filename == NULL || strcmp(filename, "-") == 0)
-        return bio_open_default(filename, modestr);
+        return bio_open_default(filename, 'w', format);
 
     mode = O_WRONLY;
 #ifdef O_CREAT
@@ -318,7 +347,7 @@ BIO *bio_open_owner(const char *filename, const char *modestr, int private)
 #ifdef O_TRUNC
     mode |= O_TRUNC;
 #endif
-    binmode = strchr(modestr, 'b') != NULL;
+    binmode = !(format & B_FORMAT_TEXT);
     if (binmode) {
 #ifdef O_BINARY
         mode |= O_BINARY;
@@ -330,7 +359,7 @@ BIO *bio_open_owner(const char *filename, const char *modestr, int private)
     fd = open(filename, mode, 0600);
     if (fd < 0)
         goto err;
-    fp = fdopen(fd, modestr);
+    fp = fdopen(fd, modestr('w', format));
     if (fp == NULL)
         goto err;
     bflags = BIO_CLOSE;
@@ -352,12 +381,13 @@ BIO *bio_open_owner(const char *filename, const char *modestr, int private)
     return NULL;
 }
 
-static BIO *bio_open_default_(const char *filename, const char *mode, int quiet)
+static BIO *bio_open_default_(const char *filename, char mode, int format,
+                              int quiet)
 {
     BIO *ret;
 
     if (filename == NULL || strcmp(filename, "-") == 0) {
-        ret = *mode == 'r' ? dup_bio_in() : dup_bio_out();
+        ret = mode == 'r' ? dup_bio_in() : dup_bio_out();
         if (quiet) {
             ERR_clear_error();
             return ret;
@@ -366,9 +396,9 @@ static BIO *bio_open_default_(const char *filename, const char *mode, int quiet)
             return ret;
         BIO_printf(bio_err,
                    "Can't open %s, %s\n",
-                   *mode == 'r' ? "stdin" : "stdout", strerror(errno));
+                   mode == 'r' ? "stdin" : "stdout", strerror(errno));
     } else {
-        ret = BIO_new_file(filename, mode);
+        ret = BIO_new_file(filename, modestr(mode, format));
         if (quiet) {
             ERR_clear_error();
             return ret;
@@ -377,21 +407,20 @@ static BIO *bio_open_default_(const char *filename, const char *mode, int quiet)
             return ret;
         BIO_printf(bio_err,
                    "Can't open %s for %s, %s\n",
-                   filename,
-                   *mode == 'r' ? "reading" : "writing", strerror(errno));
+                   filename, modeverb(mode), strerror(errno));
     }
     ERR_print_errors(bio_err);
     return NULL;
 }
 
-BIO *bio_open_default(const char *filename, const char *mode)
+BIO *bio_open_default(const char *filename, char mode, int format)
 {
-    return bio_open_default_(filename, mode, 0);
+    return bio_open_default_(filename, mode, format, 0);
 }
 
-BIO *bio_open_default_quiet(const char *filename, const char *mode)
+BIO *bio_open_default_quiet(const char *filename, char mode, int format)
 {
-    return bio_open_default_(filename, mode, 1);
+    return bio_open_default_(filename, mode, format, 1);
 }
 
 #if defined( OPENSSL_SYS_VMS)

apps/passwd.c

@@ -209,7 +209,7 @@ int passwd_main(int argc, char **argv)
         goto end;
     }
 
-    in = bio_open_default(infile, "r");
+    in = bio_open_default(infile, 'r', FORMAT_TEXT);
     if (in == NULL)
         goto end;
 

apps/pkcs12.c

@@ -353,13 +353,6 @@ int pkcs12_main(int argc, char **argv)
                        app_RAND_load_files(inrand));
     }
 
-    in = bio_open_default(infile, "rb");
-    if (in == NULL)
-        goto end;
-    out = bio_open_owner(outfile, "wb", private);
-    if (out == NULL)
-        goto end;
-
     if (twopass) {
         if (EVP_read_pw_string
             (macpass, sizeof macpass, "Enter MAC Password:", export_cert)) {
@@ -501,6 +494,11 @@ int pkcs12_main(int argc, char **argv)
             PKCS12_set_mac(p12, mpass, -1, NULL, 0, maciter, macmd);
 
         assert(private);
+
+        out = bio_open_owner(outfile, FORMAT_PKCS12, private);
+        if (out == NULL)
+            goto end;
+
         i2d_PKCS12_bio(out, p12);
 
         ret = 0;
@@ -515,6 +513,13 @@ int pkcs12_main(int argc, char **argv)
 
     }
 
+    in = bio_open_default(infile, 'r', FORMAT_PKCS12);
+    if (in == NULL)
+        goto end;
+    out = bio_open_owner(outfile, FORMAT_PEM, private);
+    if (out == NULL)
+        goto end;
+
     if ((p12 = d2i_PKCS12_bio(in, NULL)) == NULL) {
         ERR_print_errors(bio_err);
         goto end;

apps/pkcs7.c

@@ -196,7 +196,7 @@ int pkcs7_main(int argc, char **argv)
     if (!app_load_modules(NULL))
         goto end;
 
-    in = bio_open_default(infile, RB(informat));
+    in = bio_open_default(infile, 'r', informat);
     if (in == NULL)
         goto end;
 
@@ -210,7 +210,7 @@ int pkcs7_main(int argc, char **argv)
         goto end;
     }
 
-    out = bio_open_default(outfile, WB(outformat));
+    out = bio_open_default(outfile, 'w', outformat);
     if (out == NULL)
         goto end;
 

apps/pkcs8.c

@@ -239,10 +239,10 @@ int pkcs8_main(int argc, char **argv)
     if ((pbe_nid == -1) && !cipher)
         pbe_nid = NID_pbeWithMD5AndDES_CBC;
 
-    in = bio_open_default(infile, RB(informat));
+    in = bio_open_default(infile, 'r', informat);
     if (in == NULL)
         goto end;
-    out = bio_open_owner(outfile, WB(outformat), private);
+    out = bio_open_owner(outfile, outformat, private);
     if (out == NULL)
         goto end;
 

apps/pkey.c

@@ -172,7 +172,7 @@ int pkey_main(int argc, char **argv)
     if (!app_load_modules(NULL))
         goto end;
 
-    out = bio_open_owner(outfile, WB(outformat), private);
+    out = bio_open_owner(outfile, outformat, private);
     if (out == NULL)
         goto end;
 

apps/pkeyparam.c

@@ -121,10 +121,10 @@ int pkeyparam_main(int argc, char **argv)
     if (!app_load_modules(NULL))
         goto end;
 
-    in = bio_open_default(infile, "r");
+    in = bio_open_default(infile, 'r', FORMAT_PEM);
     if (in == NULL)
         goto end;
-    out = bio_open_default(outfile, "w");
+    out = bio_open_default(outfile, 'w', FORMAT_PEM);
     if (out == NULL)
         goto end;
     pkey = PEM_read_bio_Parameters(in, NULL);

apps/pkeyutl.c

@@ -249,11 +249,11 @@ int pkeyutl_main(int argc, char **argv)
     app_RAND_load_file(NULL, 0);
 
     if (pkey_op != EVP_PKEY_OP_DERIVE) {
-        in = bio_open_default(infile, "rb");
+        in = bio_open_default(infile, 'r', FORMAT_BINARY);
         if (in == NULL)
             goto end;
     }
-    out = bio_open_default(outfile, "wb");
+    out = bio_open_default(outfile, 'w', FORMAT_BINARY);
     if (out == NULL)
         goto end;
 

apps/rand.c

@@ -87,7 +87,7 @@ int rand_main(int argc, char **argv)
     BIO *out = NULL;
     char *inrand = NULL, *outfile = NULL, *prog;
     OPTION_CHOICE o;
-    int base64 = 0, hex = 0, i, num = -1, r, ret = 1;
+    int format = FORMAT_BINARY, i, num = -1, r, ret = 1;
 
     prog = opt_init(argc, argv, rand_options);
     while ((o = opt_next()) != OPT_EOF) {
@@ -111,17 +111,17 @@ int rand_main(int argc, char **argv)
             inrand = opt_arg();
             break;
         case OPT_BASE64:
-            base64 = 1;
+            format = FORMAT_BASE64;
             break;
         case OPT_HEX:
-            hex = 1;
+            format = FORMAT_TEXT;
             break;
         }
     }
     argc = opt_num_rest();
     argv = opt_rest();
 
-    if (argc != 1 || (hex && base64))
+    if (argc != 1)
         goto opthelp;
     if (sscanf(argv[0], "%d", &num) != 1 || num < 0)
         goto opthelp;
@@ -134,11 +134,11 @@ int rand_main(int argc, char **argv)
         BIO_printf(bio_err, "%ld semi-random bytes loaded\n",
                    app_RAND_load_files(inrand));
 
-    out = bio_open_default(outfile, base64 ? "w" : "wb");
+    out = bio_open_default(outfile, 'w', format);
     if (out == NULL)
         goto end;
 
-    if (base64) {
+    if (format == FORMAT_BASE64) {
         BIO *b64 = BIO_new(BIO_f_base64());
         if (b64 == NULL)
             goto end;
@@ -155,7 +155,7 @@ int rand_main(int argc, char **argv)
         r = RAND_bytes(buf, chunk);
         if (r <= 0)
             goto end;
-        if (!hex)
+        if (format != FORMAT_TEXT) /* hex */
             BIO_write(out, buf, chunk);
         else {
             for (i = 0; i < chunk; i++)
@@ -163,7 +163,7 @@ int rand_main(int argc, char **argv)
         }
         num -= chunk;
     }
-    if (hex)
+    if (format == FORMAT_TEXT)
         BIO_puts(out, "\n");
     (void)BIO_flush(out);
 

apps/req.c

@@ -566,7 +566,7 @@ int req_main(int argc, char **argv)
             BIO_printf(bio_err, "writing new private key to stdout\n");
         else
             BIO_printf(bio_err, "writing new private key to '%s'\n", keyout);
-        out = bio_open_owner(keyout, "w", private);
+        out = bio_open_owner(keyout, outformat, private);
         if (out == NULL)
             goto end;
 
@@ -601,7 +601,7 @@ int req_main(int argc, char **argv)
     }
 
     if (!newreq) {
-        in = bio_open_default(infile, RB(informat));
+        in = bio_open_default(infile, 'r', informat);
         if (in == NULL)
             goto end;
 
@@ -764,8 +764,8 @@ int req_main(int argc, char **argv)
 
     out = bio_open_default(outfile,
                            keyout != NULL && outfile != NULL &&
-                           strcmp(keyout, outfile) == 0
-                           ? AB(outformat) : WB(outformat));
+                           strcmp(keyout, outfile) == 0 ? 'a' : 'w',
+                           outformat);
     if (out == NULL)
         goto end;
 

apps/rsa.c

@@ -292,7 +292,7 @@ int rsa_main(int argc, char **argv)
         goto end;
     }
 
-    out = bio_open_owner(outfile, WB(outformat), private);
+    out = bio_open_owner(outfile, outformat, private);
     if (out == NULL)
         goto end;
 

apps/rsautl.c

@@ -251,10 +251,10 @@ int rsautl_main(int argc, char **argv)
         goto end;
     }
 
-    in = bio_open_default(infile, "rb");
+    in = bio_open_default(infile, 'r', FORMAT_BINARY);
     if (in == NULL)
         goto end;
-    out = bio_open_default(outfile, "wb");
+    out = bio_open_default(outfile, 'w', FORMAT_BINARY);
     if (out == NULL)
         goto end;
 

apps/sess_id.c

@@ -160,7 +160,7 @@ int sess_id_main(int argc, char **argv)
     }
 
     if (!noout || text) {
-        out = bio_open_default(outfile, WB(outformat));
+        out = bio_open_default(outfile, 'w', outformat);
         if (out == NULL)
             goto end;
     }
@@ -217,7 +217,7 @@ static SSL_SESSION *load_sess_id(char *infile, int format)
     SSL_SESSION *x = NULL;
     BIO *in = NULL;
 
-    in = bio_open_default(infile, RB(format));
+    in = bio_open_default(infile, 'r', format);
     if (in == NULL)
         goto end;
     if (format == FORMAT_ASN1)

apps/smime.c

@@ -170,7 +170,6 @@ int smime_main(int argc, char **argv)
         NULL;
     char *passinarg = NULL, *passin = NULL, *to = NULL, *from =
         NULL, *subject = NULL;
-    const char *inmode = "r", *outmode = "w";
     OPTION_CHOICE o;
     int flags = PKCS7_DETACHED, operation = 0, ret = 0, need_rand = 0, indef =
         0;
@@ -426,18 +425,14 @@ int smime_main(int argc, char **argv)
     if (!(operation & SMIME_SIGNERS))
         flags &= ~PKCS7_DETACHED;
 
-    if (operation & SMIME_OP) {
-        outmode = WB(outformat);
-    } else {
+    if (!(operation & SMIME_OP)) {
         if (flags & PKCS7_BINARY)
-            outmode = "wb";
+            outformat = FORMAT_BINARY;
     }
 
-    if (operation & SMIME_IP) {
-        inmode = RB(informat);
-    } else {
+    if (!(operation & SMIME_IP)) {
         if (flags & PKCS7_BINARY)
-            inmode = "rb";
+            informat = FORMAT_BINARY;
     }
 
     if (operation == SMIME_ENCRYPT) {
@@ -494,7 +489,7 @@ int smime_main(int argc, char **argv)
             goto end;
     }
 
-    in = bio_open_default(infile, inmode);
+    in = bio_open_default(infile, 'r', informat);
     if (in == NULL)
         goto end;
 
@@ -523,7 +518,7 @@ int smime_main(int argc, char **argv)
         }
     }
 
-    out = bio_open_default(outfile, outmode);
+    out = bio_open_default(outfile, 'w', outformat);
     if (out == NULL)
         goto end;
 

apps/spkac.c

@@ -175,7 +175,7 @@ int spkac_main(int argc, char **argv)
         NETSCAPE_SPKI_sign(spki, pkey, EVP_md5());
         spkstr = NETSCAPE_SPKI_b64_encode(spki);
 
-        out = bio_open_default(outfile, "w");
+        out = bio_open_default(outfile, 'w', FORMAT_TEXT);
         if (out == NULL)
             goto end;
         BIO_printf(out, "SPKAC=%s\n", spkstr);
@@ -205,7 +205,7 @@ int spkac_main(int argc, char **argv)
         goto end;
     }
 
-    out = bio_open_default(outfile, "w");
+    out = bio_open_default(outfile, 'w', FORMAT_TEXT);
     if (out == NULL)
         goto end;
 

apps/ts.c

@@ -429,13 +429,13 @@ static int query_command(const char *data, char *digest, const EVP_MD *md,
 
     /* Build query object either from file or from scratch. */
     if (in != NULL) {
-        if ((in_bio = BIO_new_file(in, "rb")) == NULL)
+        if ((in_bio = bio_open_default(in, 'r', FORMAT_ASN1)) == NULL)
             goto end;
         query = d2i_TS_REQ_bio(in_bio, NULL);
     } else {
         /* Open the file if no explicit digest bytes were specified. */
         if (digest == NULL
-            && (data_bio = bio_open_default(data, "rb")) == NULL)
+            && (data_bio = bio_open_default(data, 'r', FORMAT_ASN1)) == NULL)
             goto end;
         query = create_query(data_bio, digest, md, policy, no_nonce, cert);
     }
@@ -443,14 +443,16 @@ static int query_command(const char *data, char *digest, const EVP_MD *md,
         goto end;
 
     /* Write query either in ASN.1 or in text format. */
-    if ((out_bio = bio_open_default(out, "wb")) == NULL)
-        goto end;
     if (text) {
         /* Text output. */
+        if ((out_bio = bio_open_default(out, 'w', FORMAT_TEXT)) == NULL)
+            goto end;
         if (!TS_REQ_print_bio(out_bio, query))
             goto end;
     } else {
         /* ASN.1 output. */
+        if ((out_bio = bio_open_default(out, 'w', FORMAT_ASN1)) == NULL)
+            goto end;
         if (!i2d_TS_REQ_bio(out_bio, query))
             goto end;
     }
@@ -662,10 +664,10 @@ static int reply_command(CONF *conf, char *section, char *engine,
         goto end;
 
     /* Write response either in ASN.1 or text format. */
-    if ((out_bio = bio_open_default(out, "wb")) == NULL)
-        goto end;
     if (text) {
         /* Text output. */
+        if ((out_bio = bio_open_default(out, 'w', FORMAT_TEXT)) == NULL)
+        goto end;
         if (token_out) {
             TS_TST_INFO *tst_info = TS_RESP_get_tst_info(response);
             if (!TS_TST_INFO_print_bio(out_bio, tst_info))
@@ -676,6 +678,8 @@ static int reply_command(CONF *conf, char *section, char *engine,
         }
     } else {
         /* ASN.1 DER output. */
+        if ((out_bio = bio_open_default(out, 'w', FORMAT_ASN1)) == NULL)
+            goto end;
         if (token_out) {
             PKCS7 *token = TS_RESP_get_token(response);
             if (!i2d_PKCS7_bio(out_bio, token))

apps/x509.c

@@ -496,7 +496,7 @@ int x509_main(int argc, char **argv)
     if (!app_load_modules(NULL))
         goto end;
 
-    out = bio_open_default(outfile, WB(outformat));
+    out = bio_open_default(outfile, 'w', outformat);
     if (out == NULL)
         goto end;
 
@@ -556,7 +556,7 @@ int x509_main(int argc, char **argv)
             BIO_printf(bio_err, "We need a private key to sign with\n");
             goto end;
         }
-        in = bio_open_default(infile, RB(informat));
+        in = bio_open_default(infile, 'r', informat);
         if (in == NULL)
             goto end;
         req = PEM_read_bio_X509_REQ(in, NULL, NULL, NULL);