Avoid erroneous "assert(private)" failures.

When processing a public key input via "-pubin", "private" was
sometimes erroneously set, or else not set and incorrectly asserted.
Reviewed-by: NRich salz <rsalz@openssl.org>

apps/dsa.c

@@ -194,7 +194,7 @@ int dsa_main(int argc, char **argv)
     argc = opt_num_rest();
     argv = opt_rest();
     private = pubin || pubout ? 0 : 1;
-    if (text)
+    if (text && !pubin)
         private = 1;
 
     if (!app_passwd(passinarg, passoutarg, &passin, &passout)) {
@@ -227,7 +227,7 @@ int dsa_main(int argc, char **argv)
         goto end;
 
     if (text) {
-        assert(private);
+        assert(pubin || private);
         if (!DSA_print(out, dsa, 0)) {
             perror(outfile);
             ERR_print_errors(bio_err);
@@ -267,6 +267,11 @@ int dsa_main(int argc, char **argv)
         pk = EVP_PKEY_new();
         EVP_PKEY_set1_DSA(pk, dsa);
         if (outformat == FORMAT_PVK) {
+            if (pubin) {
+                BIO_printf(bio_err, "PVK form impossible with public key input\n");
+                EVP_PKEY_free(pk);
+                goto end;
+            }
             assert(private);
             i = i2b_PVK_bio(out, pk, pvk_encr, 0, passout);
         }

apps/ec.c

@@ -194,7 +194,7 @@ int ec_main(int argc, char **argv)
     argc = opt_num_rest();
     argv = opt_rest();
     private = param_out || pubin || pubout ? 0 : 1;
-    if (text)
+    if (text && !pubin)
         private = 1;
 
     if (!app_passwd(passinarg, passoutarg, &passin, &passout)) {
@@ -237,7 +237,7 @@ int ec_main(int argc, char **argv)
         EC_KEY_set_asn1_flag(eckey, asn1_flag);
 
     if (text) {
-        assert(private);
+        assert(pubin || private);
         if (!EC_KEY_print(out, eckey, 0)) {
             perror(outfile);
             ERR_print_errors(bio_err);

apps/pkey.c

@@ -182,18 +182,20 @@ int pkey_main(int argc, char **argv)
 
     if (!noout) {
         if (outformat == FORMAT_PEM) {
-            assert(private);
             if (pubout)
                 PEM_write_bio_PUBKEY(out, pkey);
-            else
+            else {
+                assert(private);
                 PEM_write_bio_PrivateKey(out, pkey, cipher,
                                          NULL, 0, NULL, passout);
+            }
         } else if (outformat == FORMAT_ASN1) {
-            assert(private);
             if (pubout)
                 i2d_PUBKEY_bio(out, pkey);
-            else
+            else {
+                assert(private);
                 i2d_PrivateKey_bio(out, pkey);
+            }
         } else {
             BIO_printf(bio_err, "Bad format specified for key\n");
             goto end;

apps/rsa.c

@@ -250,7 +250,7 @@ int rsa_main(int argc, char **argv)
     }
     argc = opt_num_rest();
     argv = opt_rest();
-    private = text || (!pubout && !noout) ? 1 : 0;
+    private = (text && !pubin) || (!pubout && !noout) ? 1 : 0;
 
     if (!app_passwd(passinarg, passoutarg, &passin, &passout)) {
         BIO_printf(bio_err, "Error getting passwords\n");
@@ -293,7 +293,7 @@ int rsa_main(int argc, char **argv)
         goto end;
 
     if (text) {
-        assert(private);
+        assert(pubin || private);
         if (!RSA_print(out, rsa, 0)) {
             perror(outfile);
             ERR_print_errors(bio_err);
@@ -364,11 +364,17 @@ int rsa_main(int argc, char **argv)
         EVP_PKEY *pk;
         pk = EVP_PKEY_new();
         EVP_PKEY_set1_RSA(pk, rsa);
-        if (outformat == FORMAT_PVK)
+        if (outformat == FORMAT_PVK) {
+            if (pubin) {
+                BIO_printf(bio_err, "PVK form impossible with public key input\n");
+                EVP_PKEY_free(pk);
+                goto end;
+            }
+            assert(private);
             i = i2b_PVK_bio(out, pk, pvk_encr, 0, passout);
-        else if (pubin || pubout)
+        } else if (pubin || pubout) {
             i = i2b_PublicKey_bio(out, pk);
-        else {
+        } else {
             assert(private);
             i = i2b_PrivateKey_bio(out, pk);
         }