Reject decoding of an INT64 with a value >INT64_MAX

Reviewed-by: NRichard Levitte <levitte@openssl.org>
Reviewed-by: NAndy Polyakov <appro@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/3159)

crypto/asn1/x_int64.c

@@ -71,6 +71,11 @@ static int uint64_c2i(ASN1_VALUE **pval, const unsigned char *cont, int len,
         ASN1err(ASN1_F_UINT64_C2I, ASN1_R_ILLEGAL_NEGATIVE_VALUE);
         return 0;
     }
+    if ((it->size & INTxx_FLAG_SIGNED) == INTxx_FLAG_SIGNED
+            && !neg && utmp > INT64_MAX) {
+        ASN1err(ASN1_F_UINT64_C2I, ASN1_R_TOO_LARGE);
+        return 0;
+    }
     memcpy(cp, &utmp, sizeof(utmp));
     return 1;
 }

test/asn1_encode_test.c

@@ -372,7 +372,7 @@ static ASN1_INT64_DATA int64_expected[] = {
     CUSTOM_EXPECTED_SUCCESS(ASN1_LONG_UNDEF, ASN1_LONG_UNDEF), /* t_zero */
     CUSTOM_EXPECTED_SUCCESS(1, 1), /* t_one */
     CUSTOM_EXPECTED_FAILURE,     /* t_9bytes_1 */
-    CUSTOM_EXPECTED_SUCCESS(INT64_MIN, INT64_MIN), /* t_8bytes_1 */
+    CUSTOM_EXPECTED_FAILURE,     /* t_8bytes_1 (too large positive) */
     CUSTOM_EXPECTED_SUCCESS(INT64_MAX, INT64_MAX), /* t_8bytes_2 */
     CUSTOM_EXPECTED_FAILURE,     /* t_8bytes_3_pad (illegal padding) */
     CUSTOM_EXPECTED_SUCCESS(INT64_MIN, INT64_MIN), /* t_8bytes_4_neg */