Error if memory limit exceeded.

Set a specific error if the parameters are otherwise valid but exceed the
memory limit.
Reviewed-by: NRich Salz <rsalz@openssl.org>

crypto/evp/evp_err.c

@@ -100,6 +100,7 @@ static ERR_STRING_DATA EVP_str_functs[] = {
     {ERR_FUNC(EVP_F_EVP_PBE_ALG_ADD), "EVP_PBE_alg_add"},
     {ERR_FUNC(EVP_F_EVP_PBE_ALG_ADD_TYPE), "EVP_PBE_alg_add_type"},
     {ERR_FUNC(EVP_F_EVP_PBE_CIPHERINIT), "EVP_PBE_CipherInit"},
+    {ERR_FUNC(EVP_F_EVP_PBE_SCRYPT), "EVP_PBE_scrypt"},
     {ERR_FUNC(EVP_F_EVP_PKCS82PKEY), "EVP_PKCS82PKEY"},
     {ERR_FUNC(EVP_F_EVP_PKCS82PKEY_BROKEN), "EVP_PKCS82PKEY_BROKEN"},
     {ERR_FUNC(EVP_F_EVP_PKEY2PKCS8_BROKEN), "EVP_PKEY2PKCS8_broken"},
@@ -197,6 +198,7 @@ static ERR_STRING_DATA EVP_str_reasons[] = {
     {ERR_REASON(EVP_R_INVALID_OPERATION), "invalid operation"},
     {ERR_REASON(EVP_R_IV_TOO_LARGE), "iv too large"},
     {ERR_REASON(EVP_R_KEYGEN_FAILURE), "keygen failure"},
+    {ERR_REASON(EVP_R_MEMORY_LIMIT_EXCEEDED), "memory limit exceeded"},
     {ERR_REASON(EVP_R_MESSAGE_DIGEST_IS_NULL), "message digest is null"},
     {ERR_REASON(EVP_R_METHOD_NOT_SUPPORTED), "method not supported"},
     {ERR_REASON(EVP_R_MISSING_PARAMETERS), "missing parameters"},

crypto/evp/scrypt.c

@@ -61,6 +61,7 @@
 #include <stdio.h>
 #include <string.h>
 #include <openssl/evp.h>
+#include <openssl/err.h>
 #include <internal/numbers.h>
 
 #define R(a,b) (((a) << (b)) | ((a) >> (32 - (b))))
@@ -255,8 +256,10 @@ int EVP_PBE_scrypt(const char *pass, size_t passlen,
     if (maxmem == 0)
         maxmem = SCRYPT_MAX_MEM;
 
-    if (Blen + Vlen > maxmem)
+    if (Blen + Vlen > maxmem) {
+        EVPerr(EVP_F_EVP_PBE_SCRYPT, EVP_R_MEMORY_LIMIT_EXCEEDED);
         return 0;
+    }
 
     /* If no key return to indicate parameters are OK */
     if (key == NULL)

include/openssl/evp.h

@@ -1447,6 +1447,7 @@ void ERR_load_EVP_strings(void);
 # define EVP_F_EVP_PBE_ALG_ADD                            115
 # define EVP_F_EVP_PBE_ALG_ADD_TYPE                       160
 # define EVP_F_EVP_PBE_CIPHERINIT                         116
+# define EVP_F_EVP_PBE_SCRYPT                             181
 # define EVP_F_EVP_PKCS82PKEY                             111
 # define EVP_F_EVP_PKCS82PKEY_BROKEN                      136
 # define EVP_F_EVP_PKEY2PKCS8_BROKEN                      113
@@ -1538,6 +1539,7 @@ void ERR_load_EVP_strings(void);
 # define EVP_R_INVALID_OPERATION                          148
 # define EVP_R_IV_TOO_LARGE                               102
 # define EVP_R_KEYGEN_FAILURE                             120
+# define EVP_R_MEMORY_LIMIT_EXCEEDED                      172
 # define EVP_R_MESSAGE_DIGEST_IS_NULL                     159
 # define EVP_R_METHOD_NOT_SUPPORTED                       144
 # define EVP_R_MISSING_PARAMETERS                         103