Add a test for underflow in ecp_nistp521.c

The previous commit fixed an underflow that may occur in ecp_nistp521.c.
This commit adds a test for that condition. It is heavily based on an
original test harness by Billy Brumley.
Reviewed-by: NNicola Tuveri <nic.tuv@gmail.com>
(Merged from https://github.com/openssl/openssl/pull/8405)

(cherry picked from commit 6855b496b205c067ecb276221c31c6212f4fdbae)

test/ectest.c

@@ -1403,6 +1403,74 @@ err:
     BN_CTX_free(ctx);
     return r;
 }
+
+/*
+ * Tests a point known to cause an incorrect underflow in an old version of
+ * ecp_nist521.c
+ */
+static int underflow_test(void)
+{
+    BN_CTX *ctx = NULL;
+    EC_GROUP *grp = NULL;
+    EC_POINT *P = NULL, *Q = NULL, *R = NULL;
+    BIGNUM *x1 = NULL, *y1 = NULL, *z1 = NULL, *x2 = NULL, *y2 = NULL;
+    BIGNUM *k = NULL;
+    int testresult = 0;
+    const char *x1str =
+        "1534f0077fffffe87e9adcfe000000000000000000003e05a21d2400002e031b1f4"
+        "b80000c6fafa4f3c1288798d624a247b5e2ffffffffffffffefe099241900004";
+    const char *p521m1 =
+        "1ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"
+        "fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe";
+
+    ctx = BN_CTX_new();
+    if (!TEST_ptr(ctx))
+        return 0;
+
+    BN_CTX_start(ctx);
+    x1 = BN_CTX_get(ctx);
+    y1 = BN_CTX_get(ctx);
+    z1 = BN_CTX_get(ctx);
+    x2 = BN_CTX_get(ctx);
+    y2 = BN_CTX_get(ctx);
+    k = BN_CTX_get(ctx);
+    if (!TEST_ptr(k))
+        goto err;
+
+    grp = EC_GROUP_new_by_curve_name(NID_secp521r1);
+    P = EC_POINT_new(grp);
+    Q = EC_POINT_new(grp);
+    R = EC_POINT_new(grp);
+    if (!TEST_ptr(grp) || !TEST_ptr(P) || !TEST_ptr(Q) || !TEST_ptr(R))
+        goto err;
+
+    if (!TEST_int_gt(BN_hex2bn(&x1, x1str), 0)
+            || !TEST_int_gt(BN_hex2bn(&y1, p521m1), 0)
+            || !TEST_int_gt(BN_hex2bn(&z1, p521m1), 0)
+            || !TEST_int_gt(BN_hex2bn(&k, "02"), 0)
+            || !TEST_true(EC_POINT_set_Jprojective_coordinates_GFp(grp, P, x1,
+                                                                   y1, z1, ctx))
+            || !TEST_true(EC_POINT_mul(grp, Q, NULL, P, k, ctx))
+            || !TEST_true(EC_POINT_get_affine_coordinates(grp, Q, x1, y1, ctx))
+            || !TEST_true(EC_POINT_dbl(grp, R, P, ctx))
+            || !TEST_true(EC_POINT_get_affine_coordinates(grp, R, x2, y2, ctx)))
+        goto err;
+
+    if (!TEST_int_eq(BN_cmp(x1, x2), 0)
+            || !TEST_int_eq(BN_cmp(y1, y2), 0))
+        goto err;
+
+    testresult = 1;
+
+ err:
+    BN_CTX_end(ctx);
+    EC_POINT_free(P);
+    EC_POINT_free(Q);
+    EC_GROUP_free(grp);
+    BN_CTX_free(ctx);
+
+    return testresult;
+}
 # endif
 
 static const unsigned char p521_named[] = {
@@ -1510,6 +1578,7 @@ int setup_tests(void)
 # endif
 # ifndef OPENSSL_NO_EC_NISTP_64_GCC_128
     ADD_ALL_TESTS(nistp_single_test, OSSL_NELEM(nistp_tests_params));
+    ADD_TEST(underflow_test);
 # endif
     ADD_ALL_TESTS(internal_curve_test, crv_len);
     ADD_ALL_TESTS(internal_curve_test_method, crv_len);