Add a test case for rsaz_512_sqr overflow handling

[extended tests]
Reviewed-by: NPaul Dale <paul.dale@oracle.com>
(Merged from https://github.com/openssl/openssl/pull/10575)

test/bntest.c

@@ -2404,6 +2404,288 @@ static int test_gcd_prime(void)
     return st;
 }
 
+typedef struct mod_exp_test_st
+{
+  const char *base;
+  const char *exp;
+  const char *mod;
+  const char *res;
+} MOD_EXP_TEST;
+
+static const MOD_EXP_TEST ModExpTests[] = {
+   /* original test vectors for rsaz_512_sqr bug, by OSS-Fuzz */
+   {
+       "1166180238001879113042182292626169621106255558914000595999312084"
+       "4627946820899490684928760491249738643524880720584249698100907201"
+       "002086675047927600340800371",
+       "8000000000000000000000000000000000000000000000000000000000000000"
+       "0000000000000000000000000000000000000000000000000000000000000000"
+       "00000000",
+       "1340780792684523720980737645613191762604395855615117867483316354"
+       "3294276330515137663421134775482798690129946803802212663956180562"
+       "088664022929883876655300863",
+       "8243904058268085430037326628480645845409758077568738532059032482"
+       "8294114415890603594730158120426756266457928475330450251339773498"
+       "26758407619521544102068438"
+   },
+   {
+       "4974270041410803822078866696159586946995877618987010219312844726"
+       "0284386121835740784990869050050504348861513337232530490826340663"
+       "197278031692737429054",
+       "4974270041410803822078866696159586946995877428188754995041148539"
+       "1663243362592271353668158565195557417149981094324650322556843202"
+       "946445882670777892608",
+       "1340780716511420227215592830971452482815377482627251725537099028"
+       "4429769497230131760206012644403029349547320953206103351725462999"
+       "947509743623340557059752191",
+       "5296244594780707015616522701706118082963369547253192207884519362"
+       "1767869984947542695665420219028522815539559194793619684334900442"
+       "49304558011362360473525933"
+   },
+   /* test vectors for rsaz_512_srq bug, with rcx/rbx=1 */
+   {   /* between first and second iteration */
+       "5148719036160389201525610950887605325980251964889646556085286545"
+       "3931548809178823413169359635978762036512397113080988070677858033"
+       "36463909753993540214027190",
+       "6703903964971298549787012499102923063739682910296196688861780721"
+       "8608820150367734884009371490834517138450159290932430254268769414"
+       "05973284973216824503042158",
+       "6703903964971298549787012499102923063739682910296196688861780721"
+       "8608820150367734884009371490834517138450159290932430254268769414"
+       "05973284973216824503042159",
+       "1"
+   },
+   {   /* between second and third iteration */
+       "8908340854353752577419678771330460827942371434853054158622636544"
+       "8151360109722890949471912566649465436296659601091730745087014189"
+       "2672764191218875181826063",
+       "6703903964971298549787012499102923063739682910296196688861780721"
+       "8608820150367734884009371490834517138450159290932430254268769414"
+       "05973284973216824503042158",
+       "6703903964971298549787012499102923063739682910296196688861780721"
+       "8608820150367734884009371490834517138450159290932430254268769414"
+       "05973284973216824503042159",
+       "1"
+   },
+   {   /* between third and fourth iteration */
+       "3427446396505596330634350984901719674479522569002785244080234738"
+       "4288743635435746136297299366444548736533053717416735379073185344"
+       "26985272974404612945608761",
+       "6703903964971298549787012499102923063739682910296196688861780721"
+       "8608820150367734884009371490834517138450159290932430254268769414"
+       "05973284973216824503042158",
+       "6703903964971298549787012499102923063739682910296196688861780721"
+       "8608820150367734884009371490834517138450159290932430254268769414"
+       "05973284973216824503042159",
+       "1"
+   },
+   {   /* between fourth and fifth iteration */
+       "3472743044917564564078857826111874560045331237315597383869652985"
+       "6919870028890895988478351133601517365908445058405433832718206902"
+       "4088133164805266956353542",
+       "6703903964971298549787012499102923063739682910296196688861780721"
+       "8608820150367734884009371490834517138450159290932430254268769414"
+       "05973284973216824503042158",
+       "6703903964971298549787012499102923063739682910296196688861780721"
+       "8608820150367734884009371490834517138450159290932430254268769414"
+       "05973284973216824503042159",
+       "1"
+   },
+   {   /* between fifth and sixth iteration */
+       "3608632990153469264412378349742339216742409743898601587274768025"
+       "0110772032985643555192767717344946174122842255204082586753499651"
+       "14483434992887431333675068",
+       "6703903964971298549787012499102923063739682910296196688861780721"
+       "8608820150367734884009371490834517138450159290932430254268769414"
+       "05973284973216824503042158",
+       "6703903964971298549787012499102923063739682910296196688861780721"
+       "8608820150367734884009371490834517138450159290932430254268769414"
+       "05973284973216824503042159",
+       "1"
+   },
+   {   /* between sixth and seventh iteration */
+       "8455374370234070242910508226941981520235709767260723212165264877"
+       "8689064388017521524568434328264431772644802567028663962962025746"
+       "9283458217850119569539086",
+       "6703903964971298549787012499102923063739682910296196688861780721"
+       "8608820150367734884009371490834517138450159290932430254268769414"
+       "05973284973216824503042158",
+       "6703903964971298549787012499102923063739682910296196688861780721"
+       "8608820150367734884009371490834517138450159290932430254268769414"
+       "05973284973216824503042159",
+       "1"
+   },
+   {   /* between seventh and eighth iteration */
+       "5155371529688532178421209781159131443543419764974688878527112131"
+       "7446518205609427412336183157918981038066636807317733319323257603"
+       "04416292040754017461076359",
+       "1005585594745694782468051874865438459560952436544429503329267108"
+       "2791323022555160232601405723625177570767523893639864538140315412"
+       "108959927459825236754563832",
+       "1005585594745694782468051874865438459560952436544429503329267108"
+       "2791323022555160232601405723625177570767523893639864538140315412"
+       "108959927459825236754563833",
+       "1"
+   },
+   /* test vectors for rsaz_512_srq bug, with rcx/rbx=2 */
+   {   /* between first and second iteration */
+       "3155666506033786929967309937640790361084670559125912405342594979"
+       "4345142818528956285490897841406338022378565972533508820577760065"
+       "58494345853302083699912572",
+       "6703903964971298549787012499102923063739682910296196688861780721"
+       "8608820150367734884009371490834517138450159290932430254268769414"
+       "05973284973216824503042158",
+       "6703903964971298549787012499102923063739682910296196688861780721"
+       "8608820150367734884009371490834517138450159290932430254268769414"
+       "05973284973216824503042159",
+       "1"
+   },
+   {   /* between second and third iteration */
+       "3789819583801342198190405714582958759005991915505282362397087750"
+       "4213544724644823098843135685133927198668818185338794377239590049"
+       "41019388529192775771488319",
+       "6703903964971298549787012499102923063739682910296196688861780721"
+       "8608820150367734884009371490834517138450159290932430254268769414"
+       "05973284973216824503042158",
+       "6703903964971298549787012499102923063739682910296196688861780721"
+       "8608820150367734884009371490834517138450159290932430254268769414"
+       "05973284973216824503042159",
+       "1"
+   },
+   {   /* between third and forth iteration */
+       "4695752552040706867080542538786056470322165281761525158189220280"
+       "4025547447667484759200742764246905647644662050122968912279199065"
+       "48065034299166336940507214",
+       "6703903964971298549787012499102923063739682910296196688861780721"
+       "8608820150367734884009371490834517138450159290932430254268769414"
+       "05973284973216824503042158",
+       "6703903964971298549787012499102923063739682910296196688861780721"
+       "8608820150367734884009371490834517138450159290932430254268769414"
+       "05973284973216824503042159",
+       "1"
+   },
+   {   /* between forth and fifth iteration */
+       "2159140240970485794188159431017382878636879856244045329971239574"
+       "8919691133560661162828034323196457386059819832804593989740268964"
+       "74502911811812651475927076",
+       "6703903964971298549787012499102923063739682910296196688861780721"
+       "8608820150367734884009371490834517138450159290932430254268769414"
+       "05973284973216824503042158",
+       "6703903964971298549787012499102923063739682910296196688861780721"
+       "8608820150367734884009371490834517138450159290932430254268769414"
+       "05973284973216824503042159",
+       "1"
+   },
+   {   /* between fifth and sixth iteration */
+       "5239312332984325668414624633307915097111691815000872662334695514"
+       "5436533521392362443557163429336808208137221322444780490437871903"
+       "99972784701334569424519255",
+       "6703903964971298549787012499102923063739682910296196688861780721"
+       "8608820150367734884009371490834517138450159290932430254268769414"
+       "05973284973216824503042158",
+       "6703903964971298549787012499102923063739682910296196688861780721"
+       "8608820150367734884009371490834517138450159290932430254268769414"
+       "05973284973216824503042159",
+       "1"
+   },
+   {   /* between sixth and seventh iteration */
+       "1977953647322612860406858017869125467496941904523063466791308891"
+       "1172796739058531929470539758361774569875505293428856181093904091"
+       "33788264851714311303725089",
+       "6703903964971298549787012499102923063739682910296196688861780721"
+       "8608820150367734884009371490834517138450159290932430254268769414"
+       "05973284973216824503042158",
+       "6703903964971298549787012499102923063739682910296196688861780721"
+       "8608820150367734884009371490834517138450159290932430254268769414"
+       "05973284973216824503042159",
+       "1"
+   },
+   {   /* between seventh and eighth iteration */
+       "6456987954117763835533395796948878140715006860263624787492985786"
+       "8514630216966738305923915688821526449499763719943997120302368211"
+       "04813318117996225041943964",
+       "1340780792994259709957402499820584612747936582059239337772356144"
+       "3721764030073546976801874298166903427690031858186486050853753882"
+       "811946551499689575296532556",
+       "1340780792994259709957402499820584612747936582059239337772356144"
+       "3721764030073546976801874298166903427690031858186486050853753882"
+       "811946551499689575296532557",
+       "1"
+   }
+};
+
+static int test_mod_exp(int i)
+{
+    const MOD_EXP_TEST *test = &ModExpTests[i];
+    int res = 0;
+    BIGNUM* result = NULL;
+    BIGNUM *base = NULL, *exponent = NULL, *modulo = NULL;
+    char *s = NULL;
+
+    if (!TEST_ptr(result = BN_new())
+            || !TEST_true(BN_dec2bn(&base, test->base))
+            || !TEST_true(BN_dec2bn(&exponent, test->exp))
+            || !TEST_true(BN_dec2bn(&modulo, test->mod)))
+        goto err;
+
+    if (!TEST_int_eq(BN_mod_exp(result, base, exponent, modulo, ctx), 1))
+        goto err;
+
+    if (!TEST_ptr(s = BN_bn2dec(result)))
+        goto err;
+
+    if (!TEST_mem_eq(s, strlen(s), test->res, strlen(test->res)))
+        goto err;
+
+    res = 1;
+
+ err:
+    OPENSSL_free(s);
+    BN_free(result);
+    BN_free(base);
+    BN_free(exponent);
+    BN_free(modulo);
+    return res;
+}
+
+static int test_mod_exp_consttime(int i)
+{
+    const MOD_EXP_TEST *test = &ModExpTests[i];
+    int res = 0;
+    BIGNUM* result = NULL;
+    BIGNUM *base = NULL, *exponent = NULL, *modulo = NULL;
+    char *s = NULL;
+
+    if (!TEST_ptr(result = BN_new())
+            || !TEST_true(BN_dec2bn(&base, test->base))
+            || !TEST_true(BN_dec2bn(&exponent, test->exp))
+            || !TEST_true(BN_dec2bn(&modulo, test->mod)))
+        goto err;
+
+    BN_set_flags(base, BN_FLG_CONSTTIME);
+    BN_set_flags(exponent, BN_FLG_CONSTTIME);
+    BN_set_flags(modulo, BN_FLG_CONSTTIME);
+
+    if (!TEST_int_eq(BN_mod_exp(result, base, exponent, modulo, ctx), 1))
+        goto err;
+
+    if (!TEST_ptr(s = BN_bn2dec(result)))
+        goto err;
+
+    if (!TEST_mem_eq(s, strlen(s), test->res, strlen(test->res)))
+        goto err;
+
+    res = 1;
+
+ err:
+    OPENSSL_free(s);
+    BN_free(result);
+    BN_free(base);
+    BN_free(exponent);
+    BN_free(modulo);
+    return res;
+}
+
 static int file_test_run(STANZA *s)
 {
     static const FILETEST filetests[] = {
@@ -2508,6 +2790,8 @@ int setup_tests(void)
         ADD_ALL_TESTS(test_is_prime, (int)OSSL_NELEM(primes));
         ADD_ALL_TESTS(test_not_prime, (int)OSSL_NELEM(not_primes));
         ADD_TEST(test_gcd_prime);
+        ADD_ALL_TESTS(test_mod_exp, (int)OSSL_NELEM(ModExpTests));
+        ADD_ALL_TESTS(test_mod_exp_consttime, (int)OSSL_NELEM(ModExpTests));
     } else {
         ADD_ALL_TESTS(run_file_tests, n);
     }