Add a note and better error if using Ed25519/Ed448 in dgst

Fixes #5873
Reviewed-by: NRich Salz <rsalz@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/5880)

apps/dgst.c

@@ -235,6 +235,8 @@ int dgst_main(int argc, char **argv)
     }
 
     if (keyfile != NULL) {
+        int type;
+
         if (want_pub)
             sigkey = load_pubkey(keyfile, keyform, 0, NULL, e, "key file");
         else
@@ -245,6 +247,15 @@ int dgst_main(int argc, char **argv)
              */
             goto end;
         }
+        type = EVP_PKEY_id(sigkey);
+        if (type == EVP_PKEY_ED25519 || type == EVP_PKEY_ED448) {
+            /*
+             * We implement PureEdDSA for these which doesn't have a separate
+             * digest, and only supports one shot.
+             */
+            BIO_printf(bio_err, "Key type not supported for this operation\n");
+            goto end;
+        }
     }
 
     if (mac_name != NULL) {

doc/man1/dgst.pod

@@ -86,7 +86,9 @@ Filename to output to, or standard output by default.
 
 =item B<-sign filename>
 
-Digitally sign the digest using the private key in "filename".
+Digitally sign the digest using the private key in "filename". Note this option
+does not support Ed25519 or Ed448 private keys. Use the B<pkeyutl> command
+instead for this.
 
 =item B<-keyform arg>