Pretty-print large INTEGERs and ENUMERATEDs in hex.

This avoids taking quadratic time to pretty-print certificates with
excessively large integer fields. Very large integers aren't any more
readable in decimal than hexadecimal anyway, and the i2s_* functions
will parse either form.

Found by libFuzzer.
Reviewed-by: NRich Salz <rsalz@openssl.org>
Reviewed-by: NRichard Levitte <levitte@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/4790)

crypto/err/openssl.txt

@@ -1390,6 +1390,7 @@ X509V3_F_A2I_GENERAL_NAME:164:a2i_GENERAL_NAME
 X509V3_F_ADDR_VALIDATE_PATH_INTERNAL:166:addr_validate_path_internal
 X509V3_F_ASIDENTIFIERCHOICE_CANONIZE:161:ASIdentifierChoice_canonize
 X509V3_F_ASIDENTIFIERCHOICE_IS_CANONICAL:162:ASIdentifierChoice_is_canonical
+X509V3_F_BIGNUM_TO_STRING:167:bignum_to_string
 X509V3_F_COPY_EMAIL:122:copy_email
 X509V3_F_COPY_ISSUER:123:copy_issuer
 X509V3_F_DO_DIRNAME:144:do_dirname

crypto/x509v3/v3_utl.c

@@ -14,6 +14,7 @@
 #include <stdio.h>
 #include "internal/ctype.h"
 #include <openssl/conf.h>
+#include <openssl/crypto.h>
 #include <openssl/x509v3.h>
 #include "internal/x509_int.h"
 #include <openssl/bn.h>
@@ -100,6 +101,43 @@ int X509V3_add_value_bool_nf(const char *name, int asn1_bool,
     return 1;
 }
 
+static char *bignum_to_string(const BIGNUM *bn)
+{
+    char *tmp, *ret;
+    size_t len;
+
+    /*
+     * Display large numbers in hex and small numbers in decimal. Converting to
+     * decimal takes quadratic time and is no more useful than hex for large
+     * numbers.
+     */
+    if (BN_num_bits(bn) < 128)
+        return BN_bn2dec(bn);
+
+    tmp = BN_bn2hex(bn);
+    if (tmp == NULL)
+        return NULL;
+
+    len = strlen(tmp) + 3;
+    ret = OPENSSL_malloc(len);
+    if (ret == NULL) {
+        X509V3err(X509V3_F_BIGNUM_TO_STRING, ERR_R_MALLOC_FAILURE);
+        OPENSSL_free(tmp);
+        return NULL;
+    }
+
+    /* Prepend "0x", but place it after the "-" if negative. */
+    if (tmp[0] == '-') {
+        OPENSSL_strlcpy(ret, "-0x", len);
+        OPENSSL_strlcat(ret, tmp + 1, len);
+    } else {
+        OPENSSL_strlcpy(ret, "0x", len);
+        OPENSSL_strlcat(ret, tmp, len);
+    }
+    OPENSSL_free(tmp);
+    return ret;
+}
+
 char *i2s_ASN1_ENUMERATED(X509V3_EXT_METHOD *method, const ASN1_ENUMERATED *a)
 {
     BIGNUM *bntmp = NULL;
@@ -108,7 +146,7 @@ char *i2s_ASN1_ENUMERATED(X509V3_EXT_METHOD *method, const ASN1_ENUMERATED *a)
     if (!a)
         return NULL;
     if ((bntmp = ASN1_ENUMERATED_to_BN(a, NULL)) == NULL
-        || (strtmp = BN_bn2dec(bntmp)) == NULL)
+        || (strtmp = bignum_to_string(bntmp)) == NULL)
         X509V3err(X509V3_F_I2S_ASN1_ENUMERATED, ERR_R_MALLOC_FAILURE);
     BN_free(bntmp);
     return strtmp;
@@ -122,7 +160,7 @@ char *i2s_ASN1_INTEGER(X509V3_EXT_METHOD *method, const ASN1_INTEGER *a)
     if (!a)
         return NULL;
     if ((bntmp = ASN1_INTEGER_to_BN(a, NULL)) == NULL
-        || (strtmp = BN_bn2dec(bntmp)) == NULL)
+        || (strtmp = bignum_to_string(bntmp)) == NULL)
         X509V3err(X509V3_F_I2S_ASN1_INTEGER, ERR_R_MALLOC_FAILURE);
     BN_free(bntmp);
     return strtmp;

crypto/x509v3/v3err.c

@@ -22,6 +22,8 @@ static const ERR_STRING_DATA X509V3_str_functs[] = {
      "ASIdentifierChoice_canonize"},
     {ERR_PACK(ERR_LIB_X509V3, X509V3_F_ASIDENTIFIERCHOICE_IS_CANONICAL, 0),
      "ASIdentifierChoice_is_canonical"},
+    {ERR_PACK(ERR_LIB_X509V3, X509V3_F_BIGNUM_TO_STRING, 0),
+     "bignum_to_string"},
     {ERR_PACK(ERR_LIB_X509V3, X509V3_F_COPY_EMAIL, 0), "copy_email"},
     {ERR_PACK(ERR_LIB_X509V3, X509V3_F_COPY_ISSUER, 0), "copy_issuer"},
     {ERR_PACK(ERR_LIB_X509V3, X509V3_F_DO_DIRNAME, 0), "do_dirname"},

include/openssl/x509v3err.h

@@ -26,6 +26,7 @@ int ERR_load_X509V3_strings(void);
 # define X509V3_F_ADDR_VALIDATE_PATH_INTERNAL             166
 # define X509V3_F_ASIDENTIFIERCHOICE_CANONIZE             161
 # define X509V3_F_ASIDENTIFIERCHOICE_IS_CANONICAL         162
+# define X509V3_F_BIGNUM_TO_STRING                        167
 # define X509V3_F_COPY_EMAIL                              122
 # define X509V3_F_COPY_ISSUER                             123
 # define X509V3_F_DO_DIRNAME                              144