Make EVP_PKEY_asn1_add0() stricter about its input

It turns out that the strictness that was implemented in
EVP_PKEY_asn1_new() (see Github openssl/openssl#6880) was badly placed
for some usages, and that it's better to do this check only when the
method is getting registered.

Fixes #7758
Reviewed-by: NTim Hudson <tjh@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/7847)

(cherry picked from commit a86003162138031137727147c9b642d99db434b1)

CHANGES

@@ -9,6 +9,9 @@
 
  Changes between 1.1.1a and 1.1.1b [xx XXX xxxx]
 
+  *) Move strictness check from EVP_PKEY_asn1_new() to EVP_PKEY_asn1_add0().
+     [Richard Levitte]
+
   *) Remove the 'dist' target and add a tarball building script.  The
      'dist' target has fallen out of use, and it shouldn't be
      necessary to configure just to create a source distribution.

crypto/asn1/ameth_lib.c

@@ -140,6 +140,22 @@ int EVP_PKEY_asn1_add0(const EVP_PKEY_ASN1_METHOD *ameth)
 {
     EVP_PKEY_ASN1_METHOD tmp = { 0, };
 
+    /*
+     * One of the following must be true:
+     *
+     * pem_str == NULL AND ASN1_PKEY_ALIAS is set
+     * pem_str != NULL AND ASN1_PKEY_ALIAS is clear
+     *
+     * Anything else is an error and may lead to a corrupt ASN1 method table
+     */
+    if (!((ameth->pem_str == NULL
+           && (ameth->pkey_flags & ASN1_PKEY_ALIAS) != 0)
+          || (ameth->pem_str != NULL
+              && (ameth->pkey_flags & ASN1_PKEY_ALIAS) == 0))) {
+        EVPerr(EVP_F_EVP_PKEY_ASN1_ADD0, ERR_R_PASSED_INVALID_ARGUMENT);
+        return 0;
+    }
+
     if (app_methods == NULL) {
         app_methods = sk_EVP_PKEY_ASN1_METHOD_new(ameth_cmp);
         if (app_methods == NULL)
@@ -216,18 +232,6 @@ EVP_PKEY_ASN1_METHOD *EVP_PKEY_asn1_new(int id, int flags,
             goto err;
     }
 
-    /*
-     * One of the following must be true:
-     *
-     * pem_str == NULL AND ASN1_PKEY_ALIAS is set
-     * pem_str != NULL AND ASN1_PKEY_ALIAS is clear
-     *
-     * Anything else is an error and may lead to a corrupt ASN1 method table
-     */
-    if (!((pem_str == NULL && (flags & ASN1_PKEY_ALIAS) != 0)
-          || (pem_str != NULL && (flags & ASN1_PKEY_ALIAS) == 0)))
-        goto err;
-
     if (pem_str) {
         ameth->pem_str = OPENSSL_strdup(pem_str);
         if (!ameth->pem_str)