Adapt X509_PUBKEY_set() for use with provided implementations

We do this by letting a serializer serialize the provider side key to
a DER blob formatted according to the SubjectPublicKeyInfo structure
(see RFC 5280), and deserialize it in libcrypto using the usual d2i
function.
Reviewed-by: NMatt Caswell <matt@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/10851)

crypto/x509/x_pubkey.c

@@ -16,6 +16,7 @@
 #include "crypto/x509.h"
 #include <openssl/rsa.h>
 #include <openssl/dsa.h>
+#include <openssl/serializer.h>
 
 struct X509_pubkey_st {
     X509_ALGOR *algor;
@@ -66,11 +67,15 @@ int X509_PUBKEY_set(X509_PUBKEY **x, EVP_PKEY *pkey)
     if (x == NULL)
         return 0;
 
-    if ((pk = X509_PUBKEY_new()) == NULL)
-        goto error;
+    if (pkey == NULL)
+        goto unsupported;
 
-    if (pkey != NULL && pkey->ameth) {
-        if (pkey->ameth->pub_encode) {
+    if (pkey->ameth != NULL) {
+        if ((pk = X509_PUBKEY_new()) == NULL) {
+            X509err(X509_F_X509_PUBKEY_SET, ERR_R_MALLOC_FAILURE);
+            goto error;
+        }
+        if (pkey->ameth->pub_encode != NULL) {
             if (!pkey->ameth->pub_encode(pk, pkey)) {
                 X509err(X509_F_X509_PUBKEY_SET,
                         X509_R_PUBLIC_KEY_ENCODE_ERROR);
@@ -80,15 +85,37 @@ int X509_PUBKEY_set(X509_PUBKEY **x, EVP_PKEY *pkey)
             X509err(X509_F_X509_PUBKEY_SET, X509_R_METHOD_NOT_SUPPORTED);
             goto error;
         }
-    } else {
-        X509err(X509_F_X509_PUBKEY_SET, X509_R_UNSUPPORTED_ALGORITHM);
-        goto error;
+    } else if (pkey->pkeys[0].keymgmt != NULL) {
+        BIO *bmem = BIO_new(BIO_s_mem());
+        const char *serprop = "format=der,type=public";
+        OSSL_SERIALIZER_CTX *sctx =
+            OSSL_SERIALIZER_CTX_new_by_EVP_PKEY(pkey, serprop);
+
+        if (OSSL_SERIALIZER_to_bio(sctx, bmem)) {
+            const unsigned char *der = NULL;
+            long derlen = BIO_get_mem_data(bmem, (char **)&der);
+
+            pk = d2i_X509_PUBKEY(NULL, &der, derlen);
+        }
+
+        OSSL_SERIALIZER_CTX_free(sctx);
+        BIO_free(bmem);
     }
 
+    if (pk == NULL)
+        goto unsupported;
+
     X509_PUBKEY_free(*x);
+    if (!EVP_PKEY_up_ref(pkey)) {
+        X509err(X509_F_X509_PUBKEY_SET, ERR_R_INTERNAL_ERROR);
+        goto error;
+    }
     *x = pk;
     pk->pkey = pkey;
-    return EVP_PKEY_up_ref(pkey);
+    return 1;
+
+ unsupported:
+    X509err(X509_F_X509_PUBKEY_SET, X509_R_UNSUPPORTED_ALGORITHM);
 
  error:
     X509_PUBKEY_free(pk);