Extend default method string to include public key methods.

Add missing prototypes.

Fix engine method lookup.

crypto/engine/eng_fat.c

@@ -89,6 +89,8 @@ int ENGINE_set_default(ENGINE *e, unsigned int flags)
 #endif
 	if((flags & ENGINE_METHOD_RAND) && !ENGINE_set_default_RAND(e))
 		return 0;
+	if((flags & ENGINE_METHOD_PKEY_METHS) && !ENGINE_set_default_pkey_meths(e))
+		return 0;
 	return 1;
 	}
 
@@ -115,6 +117,8 @@ static int int_def_cb(const char *alg, int len, void *arg)
 		*pflags |= ENGINE_METHOD_CIPHERS;
 	else if (!strncmp(alg, "DIGESTS", len))
 		*pflags |= ENGINE_METHOD_DIGESTS;
+	else if (!strncmp(alg, "PKEY", len))
+		*pflags |= ENGINE_METHOD_PKEY_METHS;
 	else
 		return 0;
 	return 1;

crypto/engine/engine.h

@@ -111,6 +111,7 @@ extern "C" {
 #define ENGINE_METHOD_CIPHERS		(unsigned int)0x0040
 #define ENGINE_METHOD_DIGESTS		(unsigned int)0x0080
 #define ENGINE_METHOD_STORE		(unsigned int)0x0100
+#define ENGINE_METHOD_PKEY_METHS	(unsigned int)0x0200
 /* Obvious all-or-nothing cases. */
 #define ENGINE_METHOD_ALL		(unsigned int)0xFFFF
 #define ENGINE_METHOD_NONE		(unsigned int)0x0000
@@ -568,6 +569,7 @@ int ENGINE_set_default_DH(ENGINE *e);
 int ENGINE_set_default_RAND(ENGINE *e);
 int ENGINE_set_default_ciphers(ENGINE *e);
 int ENGINE_set_default_digests(ENGINE *e);
+int ENGINE_set_default_pkey_meths(ENGINE *e);
 
 /* The combination "set" - the flags are bitwise "OR"d from the
  * ENGINE_METHOD_*** defines above. As with the "ENGINE_register_complete()"

crypto/evp/pmeth_lib.c

@@ -126,9 +126,9 @@ static EVP_PKEY_CTX *int_ctx_new(EVP_PKEY *pkey, ENGINE *e, int id)
 			EVPerr(EVP_F_INT_CTX_NEW,ERR_R_ENGINE_LIB);
 			return NULL;
 			}
-		else
-			e = ENGINE_get_pkey_meth_engine(id);
 		}
+	else
+		e = ENGINE_get_pkey_meth_engine(id);
 
 	/* If an ENGINE handled this method look it up. Othewise
 	 * use internal tables.
@@ -143,6 +143,13 @@ static EVP_PKEY_CTX *int_ctx_new(EVP_PKEY *pkey, ENGINE *e, int id)
 		return NULL;
 
 	ret = OPENSSL_malloc(sizeof(EVP_PKEY_CTX));
+	if (!ret)
+		{
+		if (e)
+			ENGINE_finish(e);
+		EVPerr(EVP_F_INT_CTX_NEW,ERR_R_MALLOC_FAILURE);
+		return NULL;
+		}
 	ret->engine = e;
 	ret->pmeth = pmeth;
 	ret->operation = EVP_PKEY_OP_UNDEFINED;