ecp_nistz256.c 49.9 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30
/******************************************************************************
 *                                                                            *
 * Copyright 2014 Intel Corporation                                           *
 *                                                                            *
 * Licensed under the Apache License, Version 2.0 (the "License");            *
 * you may not use this file except in compliance with the License.           *
 * You may obtain a copy of the License at                                    *
 *                                                                            *
 *    http://www.apache.org/licenses/LICENSE-2.0                              *
 *                                                                            *
 * Unless required by applicable law or agreed to in writing, software        *
 * distributed under the License is distributed on an "AS IS" BASIS,          *
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.   *
 * See the License for the specific language governing permissions and        *
 * limitations under the License.                                             *
 *                                                                            *
 ******************************************************************************
 *                                                                            *
 * Developers and authors:                                                    *
 * Shay Gueron (1, 2), and Vlad Krasnov (1)                                   *
 * (1) Intel Corporation, Israel Development Center                           *
 * (2) University of Haifa                                                    *
 * Reference:                                                                 *
 * S.Gueron and V.Krasnov, "Fast Prime Field Elliptic Curve Cryptography with *
 *                          256 Bit Primes"                                   *
 *                                                                            *
 ******************************************************************************/

#include <string.h>

31
#include "internal/cryptlib.h"
32
#include "internal/bn_int.h"
33 34 35
#include "ec_lcl.h"

#if BN_BITS2 != 64
36
# define TOBN(hi,lo)    lo,hi
37
#else
38
# define TOBN(hi,lo)    ((BN_ULONG)hi<<32|lo)
39 40 41
#endif

#if defined(__GNUC__)
42
# define ALIGN32        __attribute((aligned(32)))
43
#elif defined(_MSC_VER)
44
# define ALIGN32        __declspec(align(32))
45 46 47 48
#else
# define ALIGN32
#endif

49 50
#define ALIGNPTR(p,N)   ((unsigned char *)p+N-(size_t)p%N)
#define P256_LIMBS      (256/BN_BITS2)
51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67

typedef unsigned short u16;

typedef struct {
    BN_ULONG X[P256_LIMBS];
    BN_ULONG Y[P256_LIMBS];
    BN_ULONG Z[P256_LIMBS];
} P256_POINT;

typedef struct {
    BN_ULONG X[P256_LIMBS];
    BN_ULONG Y[P256_LIMBS];
} P256_POINT_AFFINE;

typedef P256_POINT_AFFINE PRECOMP256_ROW[64];

/* structure for precomputed multiples of the generator */
68
struct nistz256_pre_comp_st {
69 70
    const EC_GROUP *group;      /* Parent EC_GROUP object */
    size_t w;                   /* Window size */
71 72
    /*
     * Constant time access to the X and Y coordinates of the pre-computed,
73
     * generator multiplies, in the Montgomery domain. Pre-calculated
74 75
     * multiplies are stored in affine form.
     */
76 77 78
    PRECOMP256_ROW *precomp;
    void *precomp_storage;
    int references;
79
};
80 81 82 83 84 85 86 87 88 89 90

/* Functions implemented in assembly */
/* Modular mul by 2: res = 2*a mod P */
void ecp_nistz256_mul_by_2(BN_ULONG res[P256_LIMBS],
                           const BN_ULONG a[P256_LIMBS]);
/* Modular div by 2: res = a/2 mod P */
void ecp_nistz256_div_by_2(BN_ULONG res[P256_LIMBS],
                           const BN_ULONG a[P256_LIMBS]);
/* Modular mul by 3: res = 3*a mod P */
void ecp_nistz256_mul_by_3(BN_ULONG res[P256_LIMBS],
                           const BN_ULONG a[P256_LIMBS]);
91
/* Modular add: res = a+b mod P   */
92 93 94
void ecp_nistz256_add(BN_ULONG res[P256_LIMBS],
                      const BN_ULONG a[P256_LIMBS],
                      const BN_ULONG b[P256_LIMBS]);
95
/* Modular sub: res = a-b mod P   */
96 97 98
void ecp_nistz256_sub(BN_ULONG res[P256_LIMBS],
                      const BN_ULONG a[P256_LIMBS],
                      const BN_ULONG b[P256_LIMBS]);
99
/* Modular neg: res = -a mod P    */
100 101 102 103 104 105 106 107 108 109 110 111 112 113 114
void ecp_nistz256_neg(BN_ULONG res[P256_LIMBS], const BN_ULONG a[P256_LIMBS]);
/* Montgomery mul: res = a*b*2^-256 mod P */
void ecp_nistz256_mul_mont(BN_ULONG res[P256_LIMBS],
                           const BN_ULONG a[P256_LIMBS],
                           const BN_ULONG b[P256_LIMBS]);
/* Montgomery sqr: res = a*a*2^-256 mod P */
void ecp_nistz256_sqr_mont(BN_ULONG res[P256_LIMBS],
                           const BN_ULONG a[P256_LIMBS]);
/* Convert a number from Montgomery domain, by multiplying with 1 */
void ecp_nistz256_from_mont(BN_ULONG res[P256_LIMBS],
                            const BN_ULONG in[P256_LIMBS]);
/* Convert a number to Montgomery domain, by multiplying with 2^512 mod P*/
void ecp_nistz256_to_mont(BN_ULONG res[P256_LIMBS],
                          const BN_ULONG in[P256_LIMBS]);
/* Functions that perform constant time access to the precomputed tables */
115
void ecp_nistz256_scatter_w5(P256_POINT *val,
116
                             const P256_POINT *in_t, int idx);
117
void ecp_nistz256_gather_w5(P256_POINT *val,
118
                            const P256_POINT *in_t, int idx);
119
void ecp_nistz256_scatter_w7(P256_POINT_AFFINE *val,
120
                             const P256_POINT_AFFINE *in_t, int idx);
121
void ecp_nistz256_gather_w7(P256_POINT_AFFINE *val,
122
                            const P256_POINT_AFFINE *in_t, int idx);
123 124 125 126 127 128 129

/* One converted into the Montgomery domain */
static const BN_ULONG ONE[P256_LIMBS] = {
    TOBN(0x00000000, 0x00000001), TOBN(0xffffffff, 0x00000000),
    TOBN(0xffffffff, 0xffffffff), TOBN(0x00000000, 0xfffffffe)
};

130
static NISTZ256_PRE_COMP *ecp_nistz256_pre_comp_new(const EC_GROUP *group);
131 132

/* Precomputed tables for the default generator */
A
Andy Polyakov 已提交
133
extern const PRECOMP256_ROW ecp_nistz256_precomputed[37];
134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162

/* Recode window to a signed digit, see ecp_nistputil.c for details */
static unsigned int _booth_recode_w5(unsigned int in)
{
    unsigned int s, d;

    s = ~((in >> 5) - 1);
    d = (1 << 6) - in - 1;
    d = (d & s) | (in & ~s);
    d = (d >> 1) + (d & 1);

    return (d << 1) + (s & 1);
}

static unsigned int _booth_recode_w7(unsigned int in)
{
    unsigned int s, d;

    s = ~((in >> 7) - 1);
    d = (1 << 8) - in - 1;
    d = (d & s) | (in & ~s);
    d = (d >> 1) + (d & 1);

    return (d << 1) + (s & 1);
}

static void copy_conditional(BN_ULONG dst[P256_LIMBS],
                             const BN_ULONG src[P256_LIMBS], BN_ULONG move)
{
163
    BN_ULONG mask1 = 0-move;
164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222
    BN_ULONG mask2 = ~mask1;

    dst[0] = (src[0] & mask1) ^ (dst[0] & mask2);
    dst[1] = (src[1] & mask1) ^ (dst[1] & mask2);
    dst[2] = (src[2] & mask1) ^ (dst[2] & mask2);
    dst[3] = (src[3] & mask1) ^ (dst[3] & mask2);
    if (P256_LIMBS == 8) {
        dst[4] = (src[4] & mask1) ^ (dst[4] & mask2);
        dst[5] = (src[5] & mask1) ^ (dst[5] & mask2);
        dst[6] = (src[6] & mask1) ^ (dst[6] & mask2);
        dst[7] = (src[7] & mask1) ^ (dst[7] & mask2);
    }
}

static BN_ULONG is_zero(BN_ULONG in)
{
    in |= (0 - in);
    in = ~in;
    in >>= BN_BITS2 - 1;
    return in;
}

static BN_ULONG is_equal(const BN_ULONG a[P256_LIMBS],
                         const BN_ULONG b[P256_LIMBS])
{
    BN_ULONG res;

    res = a[0] ^ b[0];
    res |= a[1] ^ b[1];
    res |= a[2] ^ b[2];
    res |= a[3] ^ b[3];
    if (P256_LIMBS == 8) {
        res |= a[4] ^ b[4];
        res |= a[5] ^ b[5];
        res |= a[6] ^ b[6];
        res |= a[7] ^ b[7];
    }

    return is_zero(res);
}

static BN_ULONG is_one(const BN_ULONG a[P256_LIMBS])
{
    BN_ULONG res;

    res = a[0] ^ ONE[0];
    res |= a[1] ^ ONE[1];
    res |= a[2] ^ ONE[2];
    res |= a[3] ^ ONE[3];
    if (P256_LIMBS == 8) {
        res |= a[4] ^ ONE[4];
        res |= a[5] ^ ONE[5];
        res |= a[6] ^ ONE[6];
    }

    return is_zero(res);
}

#ifndef ECP_NISTZ256_REFERENCE_IMPLEMENTATION
223 224 225 226 227 228
void ecp_nistz256_point_double(P256_POINT *r, const P256_POINT *a);
void ecp_nistz256_point_add(P256_POINT *r,
                            const P256_POINT *a, const P256_POINT *b);
void ecp_nistz256_point_add_affine(P256_POINT *r,
                                   const P256_POINT *a,
                                   const P256_POINT_AFFINE *b);
229 230
#else
/* Point double: r = 2*a */
231
static void ecp_nistz256_point_double(P256_POINT *r, const P256_POINT *a)
232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276
{
    BN_ULONG S[P256_LIMBS];
    BN_ULONG M[P256_LIMBS];
    BN_ULONG Zsqr[P256_LIMBS];
    BN_ULONG tmp0[P256_LIMBS];

    const BN_ULONG *in_x = a->X;
    const BN_ULONG *in_y = a->Y;
    const BN_ULONG *in_z = a->Z;

    BN_ULONG *res_x = r->X;
    BN_ULONG *res_y = r->Y;
    BN_ULONG *res_z = r->Z;

    ecp_nistz256_mul_by_2(S, in_y);

    ecp_nistz256_sqr_mont(Zsqr, in_z);

    ecp_nistz256_sqr_mont(S, S);

    ecp_nistz256_mul_mont(res_z, in_z, in_y);
    ecp_nistz256_mul_by_2(res_z, res_z);

    ecp_nistz256_add(M, in_x, Zsqr);
    ecp_nistz256_sub(Zsqr, in_x, Zsqr);

    ecp_nistz256_sqr_mont(res_y, S);
    ecp_nistz256_div_by_2(res_y, res_y);

    ecp_nistz256_mul_mont(M, M, Zsqr);
    ecp_nistz256_mul_by_3(M, M);

    ecp_nistz256_mul_mont(S, S, in_x);
    ecp_nistz256_mul_by_2(tmp0, S);

    ecp_nistz256_sqr_mont(res_x, M);

    ecp_nistz256_sub(res_x, res_x, tmp0);
    ecp_nistz256_sub(S, S, res_x);

    ecp_nistz256_mul_mont(S, S, M);
    ecp_nistz256_sub(res_y, S, res_y);
}

/* Point addition: r = a+b */
277 278
static void ecp_nistz256_point_add(P256_POINT *r,
                                   const P256_POINT *a, const P256_POINT *b)
279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304
{
    BN_ULONG U2[P256_LIMBS], S2[P256_LIMBS];
    BN_ULONG U1[P256_LIMBS], S1[P256_LIMBS];
    BN_ULONG Z1sqr[P256_LIMBS];
    BN_ULONG Z2sqr[P256_LIMBS];
    BN_ULONG H[P256_LIMBS], R[P256_LIMBS];
    BN_ULONG Hsqr[P256_LIMBS];
    BN_ULONG Rsqr[P256_LIMBS];
    BN_ULONG Hcub[P256_LIMBS];

    BN_ULONG res_x[P256_LIMBS];
    BN_ULONG res_y[P256_LIMBS];
    BN_ULONG res_z[P256_LIMBS];

    BN_ULONG in1infty, in2infty;

    const BN_ULONG *in1_x = a->X;
    const BN_ULONG *in1_y = a->Y;
    const BN_ULONG *in1_z = a->Z;

    const BN_ULONG *in2_x = b->X;
    const BN_ULONG *in2_y = b->Y;
    const BN_ULONG *in2_z = b->Z;

    /* We encode infinity as (0,0), which is not on the curve,
     * so it is OK. */
305 306
    in1infty = (in1_x[0] | in1_x[1] | in1_x[2] | in1_x[3] |
                in1_y[0] | in1_y[1] | in1_y[2] | in1_y[3]);
307
    if (P256_LIMBS == 8)
308 309
        in1infty |= (in1_x[4] | in1_x[5] | in1_x[6] | in1_x[7] |
                     in1_y[4] | in1_y[5] | in1_y[6] | in1_y[7]);
310

311 312
    in2infty = (in2_x[0] | in2_x[1] | in2_x[2] | in2_x[3] |
                in2_y[0] | in2_y[1] | in2_y[2] | in2_y[3]);
313
    if (P256_LIMBS == 8)
314 315
        in2infty |= (in2_x[4] | in2_x[5] | in2_x[6] | in2_x[7] |
                     in2_y[4] | in2_y[5] | in2_y[6] | in2_y[7]);
316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333

    in1infty = is_zero(in1infty);
    in2infty = is_zero(in2infty);

    ecp_nistz256_sqr_mont(Z2sqr, in2_z);        /* Z2^2 */
    ecp_nistz256_sqr_mont(Z1sqr, in1_z);        /* Z1^2 */

    ecp_nistz256_mul_mont(S1, Z2sqr, in2_z);    /* S1 = Z2^3 */
    ecp_nistz256_mul_mont(S2, Z1sqr, in1_z);    /* S2 = Z1^3 */

    ecp_nistz256_mul_mont(S1, S1, in1_y);       /* S1 = Y1*Z2^3 */
    ecp_nistz256_mul_mont(S2, S2, in2_y);       /* S2 = Y2*Z1^3 */
    ecp_nistz256_sub(R, S2, S1);                /* R = S2 - S1 */

    ecp_nistz256_mul_mont(U1, in1_x, Z2sqr);    /* U1 = X1*Z2^2 */
    ecp_nistz256_mul_mont(U2, in2_x, Z1sqr);    /* U2 = X2*Z1^2 */
    ecp_nistz256_sub(H, U2, U1);                /* H = U2 - U1 */

334 335 336
    /*
     * This should not happen during sign/ecdh, so no constant time violation
     */
337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378
    if (is_equal(U1, U2) && !in1infty && !in2infty) {
        if (is_equal(S1, S2)) {
            ecp_nistz256_point_double(r, a);
            return;
        } else {
            memset(r, 0, sizeof(*r));
            return;
        }
    }

    ecp_nistz256_sqr_mont(Rsqr, R);             /* R^2 */
    ecp_nistz256_mul_mont(res_z, H, in1_z);     /* Z3 = H*Z1*Z2 */
    ecp_nistz256_sqr_mont(Hsqr, H);             /* H^2 */
    ecp_nistz256_mul_mont(res_z, res_z, in2_z); /* Z3 = H*Z1*Z2 */
    ecp_nistz256_mul_mont(Hcub, Hsqr, H);       /* H^3 */

    ecp_nistz256_mul_mont(U2, U1, Hsqr);        /* U1*H^2 */
    ecp_nistz256_mul_by_2(Hsqr, U2);            /* 2*U1*H^2 */

    ecp_nistz256_sub(res_x, Rsqr, Hsqr);
    ecp_nistz256_sub(res_x, res_x, Hcub);

    ecp_nistz256_sub(res_y, U2, res_x);

    ecp_nistz256_mul_mont(S2, S1, Hcub);
    ecp_nistz256_mul_mont(res_y, R, res_y);
    ecp_nistz256_sub(res_y, res_y, S2);

    copy_conditional(res_x, in2_x, in1infty);
    copy_conditional(res_y, in2_y, in1infty);
    copy_conditional(res_z, in2_z, in1infty);

    copy_conditional(res_x, in1_x, in2infty);
    copy_conditional(res_y, in1_y, in2infty);
    copy_conditional(res_z, in1_z, in2infty);

    memcpy(r->X, res_x, sizeof(res_x));
    memcpy(r->Y, res_y, sizeof(res_y));
    memcpy(r->Z, res_z, sizeof(res_z));
}

/* Point addition when b is known to be affine: r = a+b */
379 380 381
static void ecp_nistz256_point_add_affine(P256_POINT *r,
                                          const P256_POINT *a,
                                          const P256_POINT_AFFINE *b)
382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402
{
    BN_ULONG U2[P256_LIMBS], S2[P256_LIMBS];
    BN_ULONG Z1sqr[P256_LIMBS];
    BN_ULONG H[P256_LIMBS], R[P256_LIMBS];
    BN_ULONG Hsqr[P256_LIMBS];
    BN_ULONG Rsqr[P256_LIMBS];
    BN_ULONG Hcub[P256_LIMBS];

    BN_ULONG res_x[P256_LIMBS];
    BN_ULONG res_y[P256_LIMBS];
    BN_ULONG res_z[P256_LIMBS];

    BN_ULONG in1infty, in2infty;

    const BN_ULONG *in1_x = a->X;
    const BN_ULONG *in1_y = a->Y;
    const BN_ULONG *in1_z = a->Z;

    const BN_ULONG *in2_x = b->X;
    const BN_ULONG *in2_y = b->Y;

403 404 405 406
    /*
     * In affine representation we encode infty as (0,0), which is not on the
     * curve, so it is OK
     */
407 408
    in1infty = (in1_x[0] | in1_x[1] | in1_x[2] | in1_x[3] |
                in1_y[0] | in1_y[1] | in1_y[2] | in1_y[3]);
409
    if (P256_LIMBS == 8)
410 411
        in1infty |= (in1_x[4] | in1_x[5] | in1_x[6] | in1_x[7] |
                     in1_y[4] | in1_y[5] | in1_y[6] | in1_y[7]);
412

413 414
    in2infty = (in2_x[0] | in2_x[1] | in2_x[2] | in2_x[3] |
                in2_y[0] | in2_y[1] | in2_y[2] | in2_y[3]);
415
    if (P256_LIMBS == 8)
416 417
        in2infty |= (in2_x[4] | in2_x[5] | in2_x[6] | in2_x[7] |
                     in2_y[4] | in2_y[5] | in2_y[6] | in2_y[7]);
418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467

    in1infty = is_zero(in1infty);
    in2infty = is_zero(in2infty);

    ecp_nistz256_sqr_mont(Z1sqr, in1_z);        /* Z1^2 */

    ecp_nistz256_mul_mont(U2, in2_x, Z1sqr);    /* U2 = X2*Z1^2 */
    ecp_nistz256_sub(H, U2, in1_x);             /* H = U2 - U1 */

    ecp_nistz256_mul_mont(S2, Z1sqr, in1_z);    /* S2 = Z1^3 */

    ecp_nistz256_mul_mont(res_z, H, in1_z);     /* Z3 = H*Z1*Z2 */

    ecp_nistz256_mul_mont(S2, S2, in2_y);       /* S2 = Y2*Z1^3 */
    ecp_nistz256_sub(R, S2, in1_y);             /* R = S2 - S1 */

    ecp_nistz256_sqr_mont(Hsqr, H);             /* H^2 */
    ecp_nistz256_sqr_mont(Rsqr, R);             /* R^2 */
    ecp_nistz256_mul_mont(Hcub, Hsqr, H);       /* H^3 */

    ecp_nistz256_mul_mont(U2, in1_x, Hsqr);     /* U1*H^2 */
    ecp_nistz256_mul_by_2(Hsqr, U2);            /* 2*U1*H^2 */

    ecp_nistz256_sub(res_x, Rsqr, Hsqr);
    ecp_nistz256_sub(res_x, res_x, Hcub);
    ecp_nistz256_sub(H, U2, res_x);

    ecp_nistz256_mul_mont(S2, in1_y, Hcub);
    ecp_nistz256_mul_mont(H, H, R);
    ecp_nistz256_sub(res_y, H, S2);

    copy_conditional(res_x, in2_x, in1infty);
    copy_conditional(res_x, in1_x, in2infty);

    copy_conditional(res_y, in2_y, in1infty);
    copy_conditional(res_y, in1_y, in2infty);

    copy_conditional(res_z, ONE, in1infty);
    copy_conditional(res_z, in1_z, in2infty);

    memcpy(r->X, res_x, sizeof(res_x));
    memcpy(r->Y, res_y, sizeof(res_y));
    memcpy(r->Z, res_z, sizeof(res_z));
}
#endif

/* r = in^-1 mod p */
static void ecp_nistz256_mod_inverse(BN_ULONG r[P256_LIMBS],
                                     const BN_ULONG in[P256_LIMBS])
{
468 469 470 471
    /*
     * The poly is ffffffff 00000001 00000000 00000000 00000000 ffffffff
     * ffffffff ffffffff We use FLT and used poly-2 as exponent
     */
472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540
    BN_ULONG p2[P256_LIMBS];
    BN_ULONG p4[P256_LIMBS];
    BN_ULONG p8[P256_LIMBS];
    BN_ULONG p16[P256_LIMBS];
    BN_ULONG p32[P256_LIMBS];
    BN_ULONG res[P256_LIMBS];
    int i;

    ecp_nistz256_sqr_mont(res, in);
    ecp_nistz256_mul_mont(p2, res, in);         /* 3*p */

    ecp_nistz256_sqr_mont(res, p2);
    ecp_nistz256_sqr_mont(res, res);
    ecp_nistz256_mul_mont(p4, res, p2);         /* f*p */

    ecp_nistz256_sqr_mont(res, p4);
    ecp_nistz256_sqr_mont(res, res);
    ecp_nistz256_sqr_mont(res, res);
    ecp_nistz256_sqr_mont(res, res);
    ecp_nistz256_mul_mont(p8, res, p4);         /* ff*p */

    ecp_nistz256_sqr_mont(res, p8);
    for (i = 0; i < 7; i++)
        ecp_nistz256_sqr_mont(res, res);
    ecp_nistz256_mul_mont(p16, res, p8);        /* ffff*p */

    ecp_nistz256_sqr_mont(res, p16);
    for (i = 0; i < 15; i++)
        ecp_nistz256_sqr_mont(res, res);
    ecp_nistz256_mul_mont(p32, res, p16);       /* ffffffff*p */

    ecp_nistz256_sqr_mont(res, p32);
    for (i = 0; i < 31; i++)
        ecp_nistz256_sqr_mont(res, res);
    ecp_nistz256_mul_mont(res, res, in);

    for (i = 0; i < 32 * 4; i++)
        ecp_nistz256_sqr_mont(res, res);
    ecp_nistz256_mul_mont(res, res, p32);

    for (i = 0; i < 32; i++)
        ecp_nistz256_sqr_mont(res, res);
    ecp_nistz256_mul_mont(res, res, p32);

    for (i = 0; i < 16; i++)
        ecp_nistz256_sqr_mont(res, res);
    ecp_nistz256_mul_mont(res, res, p16);

    for (i = 0; i < 8; i++)
        ecp_nistz256_sqr_mont(res, res);
    ecp_nistz256_mul_mont(res, res, p8);

    ecp_nistz256_sqr_mont(res, res);
    ecp_nistz256_sqr_mont(res, res);
    ecp_nistz256_sqr_mont(res, res);
    ecp_nistz256_sqr_mont(res, res);
    ecp_nistz256_mul_mont(res, res, p4);

    ecp_nistz256_sqr_mont(res, res);
    ecp_nistz256_sqr_mont(res, res);
    ecp_nistz256_mul_mont(res, res, p2);

    ecp_nistz256_sqr_mont(res, res);
    ecp_nistz256_sqr_mont(res, res);
    ecp_nistz256_mul_mont(res, res, in);

    memcpy(r, res, sizeof(res));
}

541 542 543 544
/*
 * ecp_nistz256_bignum_to_field_elem copies the contents of |in| to |out| and
 * returns one if it fits. Otherwise it returns zero.
 */
E
Emilia Kasper 已提交
545 546
__owur static int ecp_nistz256_bignum_to_field_elem(BN_ULONG out[P256_LIMBS],
                                                    const BIGNUM *in)
547
{
548
    return bn_copy_words(out, in, P256_LIMBS);
549 550 551
}

/* r = sum(scalar[i]*point[i]) */
E
Emilia Kasper 已提交
552 553 554 555 556
__owur static int ecp_nistz256_windowed_mul(const EC_GROUP *group,
                                            P256_POINT *r,
                                            const BIGNUM **scalar,
                                            const EC_POINT **point,
                                            size_t num, BN_CTX *ctx)
557
{
558
    size_t i;
559
    int j, ret = 0;
560
    unsigned int idx;
561 562 563 564
    unsigned char (*p_str)[33] = NULL;
    const unsigned int window_size = 5;
    const unsigned int mask = (1 << (window_size + 1)) - 1;
    unsigned int wvalue;
565
    P256_POINT *temp;           /* place for 5 temporary points */
566
    const BIGNUM **scalars = NULL;
567
    P256_POINT (*table)[16] = NULL;
568 569
    void *table_storage = NULL;

570 571 572
    if ((num * 16 + 6) > OPENSSL_MALLOC_MAX_NELEMS(P256_POINT)
        || (table_storage =
            OPENSSL_malloc((num * 16 + 5) * sizeof(P256_POINT) + 64)) == NULL
573 574 575
        || (p_str =
            OPENSSL_malloc(num * 33 * sizeof(unsigned char))) == NULL
        || (scalars = OPENSSL_malloc(num * sizeof(BIGNUM *))) == NULL) {
576
        ECerr(EC_F_ECP_NISTZ256_WINDOWED_MUL, ERR_R_MALLOC_FAILURE);
577 578 579
        goto err;
    }

A
Andy Polyakov 已提交
580
    table = (void *)ALIGNPTR(table_storage, 64);
581
    temp = (P256_POINT *)(table + num);
A
Andy Polyakov 已提交
582

583 584 585
    for (i = 0; i < num; i++) {
        P256_POINT *row = table[i];

586
        /* This is an unusual input, we don't guarantee constant-timeness. */
587 588 589 590 591
        if ((BN_num_bits(scalar[i]) > 256) || BN_is_negative(scalar[i])) {
            BIGNUM *mod;

            if ((mod = BN_CTX_get(ctx)) == NULL)
                goto err;
592
            if (!BN_nnmod(mod, scalar[i], group->order, ctx)) {
593
                ECerr(EC_F_ECP_NISTZ256_WINDOWED_MUL, ERR_R_BN_LIB);
594 595 596 597 598 599
                goto err;
            }
            scalars[i] = mod;
        } else
            scalars[i] = scalar[i];

600 601
        for (j = 0; j < bn_get_top(scalars[i]) * BN_BYTES; j += BN_BYTES) {
            BN_ULONG d = bn_get_words(scalars[i])[j / BN_BYTES];
602

603 604 605 606
            p_str[i][j + 0] = (unsigned char)d;
            p_str[i][j + 1] = (unsigned char)(d >> 8);
            p_str[i][j + 2] = (unsigned char)(d >> 16);
            p_str[i][j + 3] = (unsigned char)(d >>= 24);
607 608
            if (BN_BYTES == 8) {
                d >>= 8;
609 610 611 612
                p_str[i][j + 4] = (unsigned char)d;
                p_str[i][j + 5] = (unsigned char)(d >> 8);
                p_str[i][j + 6] = (unsigned char)(d >> 16);
                p_str[i][j + 7] = (unsigned char)(d >> 24);
613 614 615 616 617
            }
        }
        for (; j < 33; j++)
            p_str[i][j] = 0;

618 619 620
        if (!ecp_nistz256_bignum_to_field_elem(temp[0].X, point[i]->X)
            || !ecp_nistz256_bignum_to_field_elem(temp[0].Y, point[i]->Y)
            || !ecp_nistz256_bignum_to_field_elem(temp[0].Z, point[i]->Z)) {
621 622
            ECerr(EC_F_ECP_NISTZ256_WINDOWED_MUL,
                  EC_R_COORDINATES_OUT_OF_RANGE);
623 624 625
            goto err;
        }

626 627 628 629
        /*
	 * row[0] is implicitly (0,0,0) (the point at infinity), therefore it
	 * is not stored. All other values are actually stored with an offset
	 * of -1 in table.
A
Andy Polyakov 已提交
630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662
         */

        ecp_nistz256_scatter_w5  (row, &temp[0], 1);
        ecp_nistz256_point_double(&temp[1], &temp[0]);              /*1+1=2  */
        ecp_nistz256_scatter_w5  (row, &temp[1], 2);
        ecp_nistz256_point_add   (&temp[2], &temp[1], &temp[0]);    /*2+1=3  */
        ecp_nistz256_scatter_w5  (row, &temp[2], 3);
        ecp_nistz256_point_double(&temp[1], &temp[1]);              /*2*2=4  */
        ecp_nistz256_scatter_w5  (row, &temp[1], 4);
        ecp_nistz256_point_double(&temp[2], &temp[2]);              /*2*3=6  */
        ecp_nistz256_scatter_w5  (row, &temp[2], 6);
        ecp_nistz256_point_add   (&temp[3], &temp[1], &temp[0]);    /*4+1=5  */
        ecp_nistz256_scatter_w5  (row, &temp[3], 5);
        ecp_nistz256_point_add   (&temp[4], &temp[2], &temp[0]);    /*6+1=7  */
        ecp_nistz256_scatter_w5  (row, &temp[4], 7);
        ecp_nistz256_point_double(&temp[1], &temp[1]);              /*2*4=8  */
        ecp_nistz256_scatter_w5  (row, &temp[1], 8);
        ecp_nistz256_point_double(&temp[2], &temp[2]);              /*2*6=12 */
        ecp_nistz256_scatter_w5  (row, &temp[2], 12);
        ecp_nistz256_point_double(&temp[3], &temp[3]);              /*2*5=10 */
        ecp_nistz256_scatter_w5  (row, &temp[3], 10);
        ecp_nistz256_point_double(&temp[4], &temp[4]);              /*2*7=14 */
        ecp_nistz256_scatter_w5  (row, &temp[4], 14);
        ecp_nistz256_point_add   (&temp[2], &temp[2], &temp[0]);    /*12+1=13*/
        ecp_nistz256_scatter_w5  (row, &temp[2], 13);
        ecp_nistz256_point_add   (&temp[3], &temp[3], &temp[0]);    /*10+1=11*/
        ecp_nistz256_scatter_w5  (row, &temp[3], 11);
        ecp_nistz256_point_add   (&temp[4], &temp[4], &temp[0]);    /*14+1=15*/
        ecp_nistz256_scatter_w5  (row, &temp[4], 15);
        ecp_nistz256_point_add   (&temp[2], &temp[1], &temp[0]);    /*8+1=9  */
        ecp_nistz256_scatter_w5  (row, &temp[2], 9);
        ecp_nistz256_point_double(&temp[1], &temp[1]);              /*2*8=16 */
        ecp_nistz256_scatter_w5  (row, &temp[1], 16);
663 664
    }

665
    idx = 255;
666

667 668
    wvalue = p_str[0][(idx - 1) / 8];
    wvalue = (wvalue >> ((idx - 1) % 8)) & mask;
669

A
Andy Polyakov 已提交
670 671 672 673 674 675
    /*
     * We gather to temp[0], because we know it's position relative
     * to table
     */
    ecp_nistz256_gather_w5(&temp[0], table[0], _booth_recode_w5(wvalue) >> 1);
    memcpy(r, &temp[0], sizeof(temp[0]));
676

677 678 679
    while (idx >= 5) {
        for (i = (idx == 255 ? 1 : 0); i < num; i++) {
            unsigned int off = (idx - 1) / 8;
680 681

            wvalue = p_str[i][off] | p_str[i][off + 1] << 8;
682
            wvalue = (wvalue >> ((idx - 1) % 8)) & mask;
683 684 685

            wvalue = _booth_recode_w5(wvalue);

A
Andy Polyakov 已提交
686
            ecp_nistz256_gather_w5(&temp[0], table[i], wvalue >> 1);
687

A
Andy Polyakov 已提交
688 689
            ecp_nistz256_neg(temp[1].Y, temp[0].Y);
            copy_conditional(temp[0].Y, temp[1].Y, (wvalue & 1));
690

A
Andy Polyakov 已提交
691
            ecp_nistz256_point_add(r, r, &temp[0]);
692 693
        }

694
        idx -= window_size;
695 696 697 698 699 700 701 702 703 704 705 706 707 708 709

        ecp_nistz256_point_double(r, r);
        ecp_nistz256_point_double(r, r);
        ecp_nistz256_point_double(r, r);
        ecp_nistz256_point_double(r, r);
        ecp_nistz256_point_double(r, r);
    }

    /* Final window */
    for (i = 0; i < num; i++) {
        wvalue = p_str[i][0];
        wvalue = (wvalue << 1) & mask;

        wvalue = _booth_recode_w5(wvalue);

A
Andy Polyakov 已提交
710
        ecp_nistz256_gather_w5(&temp[0], table[i], wvalue >> 1);
711

A
Andy Polyakov 已提交
712 713
        ecp_nistz256_neg(temp[1].Y, temp[0].Y);
        copy_conditional(temp[0].Y, temp[1].Y, wvalue & 1);
714

A
Andy Polyakov 已提交
715
        ecp_nistz256_point_add(r, r, &temp[0]);
716 717
    }

718
    ret = 1;
719
 err:
R
Rich Salz 已提交
720 721 722
    OPENSSL_free(table_storage);
    OPENSSL_free(p_str);
    OPENSSL_free(scalars);
723
    return ret;
724 725 726 727 728 729 730 731 732 733 734 735 736
}

/* Coordinates of G, for which we have precomputed tables */
const static BN_ULONG def_xG[P256_LIMBS] = {
    TOBN(0x79e730d4, 0x18a9143c), TOBN(0x75ba95fc, 0x5fedb601),
    TOBN(0x79fb732b, 0x77622510), TOBN(0x18905f76, 0xa53755c6)
};

const static BN_ULONG def_yG[P256_LIMBS] = {
    TOBN(0xddf25357, 0xce95560a), TOBN(0x8b4ab8e4, 0xba19e45c),
    TOBN(0xd2e88688, 0xdd21f325), TOBN(0x8571ff18, 0x25885d85)
};

737 738 739 740
/*
 * ecp_nistz256_is_affine_G returns one if |generator| is the standard, P-256
 * generator.
 */
741
static int ecp_nistz256_is_affine_G(const EC_POINT *generator)
742
{
743 744 745 746 747 748
    return (bn_get_top(generator->X) == P256_LIMBS) &&
        (bn_get_top(generator->Y) == P256_LIMBS) &&
        (bn_get_top(generator->Z) == (P256_LIMBS - P256_LIMBS / 8)) &&
        is_equal(bn_get_words(generator->X), def_xG) &&
        is_equal(bn_get_words(generator->Y), def_yG) &&
        is_one(bn_get_words(generator->Z));
749 750
}

E
Emilia Kasper 已提交
751
__owur static int ecp_nistz256_mult_precompute(EC_GROUP *group, BN_CTX *ctx)
752
{
753 754
    /*
     * We precompute a table for a Booth encoded exponent (wNAF) based
755
     * computation. Each table holds 64 values for safe access, with an
756 757 758
     * implicit value of infinity at index zero. We use window of size 7, and
     * therefore require ceil(256/7) = 37 tables.
     */
759 760 761
    BIGNUM *order;
    EC_POINT *P = NULL, *T = NULL;
    const EC_POINT *generator;
762
    NISTZ256_PRE_COMP *pre_comp;
763
    BN_CTX *new_ctx = NULL;
764 765 766 767 768 769
    int i, j, k, ret = 0;
    size_t w;

    PRECOMP256_ROW *preComputedTable = NULL;
    unsigned char *precomp_storage = NULL;

770
    /* if there is an old NISTZ256_PRE_COMP object, throw it away */
771
    EC_pre_comp_free(group);
772 773
    generator = EC_GROUP_get0_generator(group);
    if (generator == NULL) {
774
        ECerr(EC_F_ECP_NISTZ256_MULT_PRECOMPUTE, EC_R_UNDEFINED_GENERATOR);
775 776 777 778
        return 0;
    }

    if (ecp_nistz256_is_affine_G(generator)) {
779 780 781 782
        /*
         * No need to calculate tables for the standard generator because we
         * have them statically.
         */
783 784 785
        return 1;
    }

786
    if ((pre_comp = ecp_nistz256_pre_comp_new(group)) == NULL)
787 788 789
        return 0;

    if (ctx == NULL) {
790
        ctx = new_ctx = BN_CTX_new();
791 792 793 794 795 796 797 798 799 800 801 802 803 804
        if (ctx == NULL)
            goto err;
    }

    BN_CTX_start(ctx);
    order = BN_CTX_get(ctx);

    if (order == NULL)
        goto err;

    if (!EC_GROUP_get_order(group, order, ctx))
        goto err;

    if (BN_is_zero(order)) {
805
        ECerr(EC_F_ECP_NISTZ256_MULT_PRECOMPUTE, EC_R_UNKNOWN_ORDER);
806 807 808 809 810 811 812
        goto err;
    }

    w = 7;

    if ((precomp_storage =
         OPENSSL_malloc(37 * 64 * sizeof(P256_POINT_AFFINE) + 64)) == NULL) {
813
        ECerr(EC_F_ECP_NISTZ256_MULT_PRECOMPUTE, ERR_R_MALLOC_FAILURE);
814 815 816
        goto err;
    }

A
Andy Polyakov 已提交
817 818
    preComputedTable = (void *)ALIGNPTR(precomp_storage, 64);

819 820
    P = EC_POINT_new(group);
    T = EC_POINT_new(group);
821 822
    if (P == NULL || T == NULL)
        goto err;
823

824 825 826 827
    /*
     * The zero entry is implicitly infinity, and we skip it, storing other
     * values with -1 offset.
     */
828 829
    if (!EC_POINT_copy(T, generator))
        goto err;
830 831

    for (k = 0; k < 64; k++) {
832 833
        if (!EC_POINT_copy(P, T))
            goto err;
834
        for (j = 0; j < 37; j++) {
A
Andy Polyakov 已提交
835
            P256_POINT_AFFINE temp;
836
            /*
837
             * It would be faster to use EC_POINTs_make_affine and
838 839
             * make multiple points affine at the same time.
             */
840 841 842 843 844 845 846 847
            if (!EC_POINT_make_affine(group, P, ctx))
                goto err;
            if (!ecp_nistz256_bignum_to_field_elem(temp.X, P->X) ||
                !ecp_nistz256_bignum_to_field_elem(temp.Y, P->Y)) {
                ECerr(EC_F_ECP_NISTZ256_MULT_PRECOMPUTE,
                      EC_R_COORDINATES_OUT_OF_RANGE);
                goto err;
            }
A
Andy Polyakov 已提交
848
            ecp_nistz256_scatter_w7(preComputedTable[j], &temp, k);
849 850 851 852
            for (i = 0; i < 7; i++) {
                if (!EC_POINT_dbl(group, P, P, ctx))
                    goto err;
            }
853
        }
854 855
        if (!EC_POINT_add(group, T, T, generator, ctx))
            goto err;
856 857 858 859 860 861 862
    }

    pre_comp->group = group;
    pre_comp->w = w;
    pre_comp->precomp = preComputedTable;
    pre_comp->precomp_storage = precomp_storage;
    precomp_storage = NULL;
863
    SETPRECOMP(group, nistz256, pre_comp);
864 865 866
    pre_comp = NULL;
    ret = 1;

867
 err:
868 869
    if (ctx != NULL)
        BN_CTX_end(ctx);
870 871
    BN_CTX_free(new_ctx);

872
    EC_nistz256_pre_comp_free(pre_comp);
R
Rich Salz 已提交
873
    OPENSSL_free(precomp_storage);
R
Rich Salz 已提交
874 875
    EC_POINT_free(P);
    EC_POINT_free(T);
876 877 878 879 880 881 882 883 884 885 886 887 888
    return ret;
}

/*
 * Note that by default ECP_NISTZ256_AVX2 is undefined. While it's great
 * code processing 4 points in parallel, corresponding serial operation
 * is several times slower, because it uses 29x29=58-bit multiplication
 * as opposite to 64x64=128-bit in integer-only scalar case. As result
 * it doesn't provide *significant* performance improvement. Note that
 * just defining ECP_NISTZ256_AVX2 is not sufficient to make it work,
 * you'd need to compile even asm/ecp_nistz256-avx.pl module.
 */
#if defined(ECP_NISTZ256_AVX2)
A
Andy Polyakov 已提交
889
# if !(defined(__x86_64) || defined(__x86_64__) || \
890 891 892 893 894
       defined(_M_AMD64) || defined(_MX64)) || \
     !(defined(__GNUC__) || defined(_MSC_VER)) /* this is for ALIGN32 */
#  undef ECP_NISTZ256_AVX2
# else
/* Constant time access, loading four values, from four consecutive tables */
895 896 897
void ecp_nistz256_avx2_multi_gather_w7(void *result, const void *in,
                                       int index0, int index1, int index2,
                                       int index3);
898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922
void ecp_nistz256_avx2_transpose_convert(void *RESULTx4, const void *in);
void ecp_nistz256_avx2_convert_transpose_back(void *result, const void *Ax4);
void ecp_nistz256_avx2_point_add_affine_x4(void *RESULTx4, const void *Ax4,
                                           const void *Bx4);
void ecp_nistz256_avx2_point_add_affines_x4(void *RESULTx4, const void *Ax4,
                                            const void *Bx4);
void ecp_nistz256_avx2_to_mont(void *RESULTx4, const void *Ax4);
void ecp_nistz256_avx2_from_mont(void *RESULTx4, const void *Ax4);
void ecp_nistz256_avx2_set1(void *RESULTx4);
int ecp_nistz_avx2_eligible(void);

static void booth_recode_w7(unsigned char *sign,
                            unsigned char *digit, unsigned char in)
{
    unsigned char s, d;

    s = ~((in >> 7) - 1);
    d = (1 << 8) - in - 1;
    d = (d & s) | (in & ~s);
    d = (d >> 1) + (d & 1);

    *sign = s & 1;
    *digit = d;
}

923 924
/*
 * ecp_nistz256_avx2_mul_g performs multiplication by G, using only the
925
 * precomputed table. It does 4 affine point additions in parallel,
926 927
 * significantly speeding up point multiplication for a fixed value.
 */
928
static void ecp_nistz256_avx2_mul_g(P256_POINT *r,
929
                                    unsigned char p_str[33],
930
                                    const P256_POINT_AFFINE(*preComputedTable)[64])
931 932 933 934 935 936 937 938 939
{
    const unsigned int window_size = 7;
    const unsigned int mask = (1 << (window_size + 1)) - 1;
    unsigned int wvalue;
    /* Using 4 windows at a time */
    unsigned char sign0, digit0;
    unsigned char sign1, digit1;
    unsigned char sign2, digit2;
    unsigned char sign3, digit3;
940
    unsigned int idx = 0;
941 942 943 944 945
    BN_ULONG tmp[P256_LIMBS];
    int i;

    ALIGN32 BN_ULONG aX4[4 * 9 * 3] = { 0 };
    ALIGN32 BN_ULONG bX4[4 * 9 * 2] = { 0 };
A
Andy Polyakov 已提交
946 947
    ALIGN32 P256_POINT_AFFINE point_arr[4];
    ALIGN32 P256_POINT res_point_arr[4];
948 949 950 951

    /* Initial four windows */
    wvalue = *((u16 *) & p_str[0]);
    wvalue = (wvalue << 1) & mask;
952
    idx += window_size;
953
    booth_recode_w7(&sign0, &digit0, wvalue);
954 955 956
    wvalue = *((u16 *) & p_str[(idx - 1) / 8]);
    wvalue = (wvalue >> ((idx - 1) % 8)) & mask;
    idx += window_size;
957
    booth_recode_w7(&sign1, &digit1, wvalue);
958 959 960
    wvalue = *((u16 *) & p_str[(idx - 1) / 8]);
    wvalue = (wvalue >> ((idx - 1) % 8)) & mask;
    idx += window_size;
961
    booth_recode_w7(&sign2, &digit2, wvalue);
962 963 964
    wvalue = *((u16 *) & p_str[(idx - 1) / 8]);
    wvalue = (wvalue >> ((idx - 1) % 8)) & mask;
    idx += window_size;
965 966
    booth_recode_w7(&sign3, &digit3, wvalue);

A
Andy Polyakov 已提交
967
    ecp_nistz256_avx2_multi_gather_w7(point_arr, preComputedTable[0],
968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983
                                      digit0, digit1, digit2, digit3);

    ecp_nistz256_neg(tmp, point_arr[0].Y);
    copy_conditional(point_arr[0].Y, tmp, sign0);
    ecp_nistz256_neg(tmp, point_arr[1].Y);
    copy_conditional(point_arr[1].Y, tmp, sign1);
    ecp_nistz256_neg(tmp, point_arr[2].Y);
    copy_conditional(point_arr[2].Y, tmp, sign2);
    ecp_nistz256_neg(tmp, point_arr[3].Y);
    copy_conditional(point_arr[3].Y, tmp, sign3);

    ecp_nistz256_avx2_transpose_convert(aX4, point_arr);
    ecp_nistz256_avx2_to_mont(aX4, aX4);
    ecp_nistz256_avx2_to_mont(&aX4[4 * 9], &aX4[4 * 9]);
    ecp_nistz256_avx2_set1(&aX4[4 * 9 * 2]);

984 985 986
    wvalue = *((u16 *) & p_str[(idx - 1) / 8]);
    wvalue = (wvalue >> ((idx - 1) % 8)) & mask;
    idx += window_size;
987
    booth_recode_w7(&sign0, &digit0, wvalue);
988 989 990
    wvalue = *((u16 *) & p_str[(idx - 1) / 8]);
    wvalue = (wvalue >> ((idx - 1) % 8)) & mask;
    idx += window_size;
991
    booth_recode_w7(&sign1, &digit1, wvalue);
992 993 994
    wvalue = *((u16 *) & p_str[(idx - 1) / 8]);
    wvalue = (wvalue >> ((idx - 1) % 8)) & mask;
    idx += window_size;
995
    booth_recode_w7(&sign2, &digit2, wvalue);
996 997 998
    wvalue = *((u16 *) & p_str[(idx - 1) / 8]);
    wvalue = (wvalue >> ((idx - 1) % 8)) & mask;
    idx += window_size;
999 1000
    booth_recode_w7(&sign3, &digit3, wvalue);

A
Andy Polyakov 已提交
1001
    ecp_nistz256_avx2_multi_gather_w7(point_arr, preComputedTable[4 * 1],
1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019
                                      digit0, digit1, digit2, digit3);

    ecp_nistz256_neg(tmp, point_arr[0].Y);
    copy_conditional(point_arr[0].Y, tmp, sign0);
    ecp_nistz256_neg(tmp, point_arr[1].Y);
    copy_conditional(point_arr[1].Y, tmp, sign1);
    ecp_nistz256_neg(tmp, point_arr[2].Y);
    copy_conditional(point_arr[2].Y, tmp, sign2);
    ecp_nistz256_neg(tmp, point_arr[3].Y);
    copy_conditional(point_arr[3].Y, tmp, sign3);

    ecp_nistz256_avx2_transpose_convert(bX4, point_arr);
    ecp_nistz256_avx2_to_mont(bX4, bX4);
    ecp_nistz256_avx2_to_mont(&bX4[4 * 9], &bX4[4 * 9]);
    /* Optimized when both inputs are affine */
    ecp_nistz256_avx2_point_add_affines_x4(aX4, aX4, bX4);

    for (i = 2; i < 9; i++) {
1020 1021 1022
        wvalue = *((u16 *) & p_str[(idx - 1) / 8]);
        wvalue = (wvalue >> ((idx - 1) % 8)) & mask;
        idx += window_size;
1023
        booth_recode_w7(&sign0, &digit0, wvalue);
1024 1025 1026
        wvalue = *((u16 *) & p_str[(idx - 1) / 8]);
        wvalue = (wvalue >> ((idx - 1) % 8)) & mask;
        idx += window_size;
1027
        booth_recode_w7(&sign1, &digit1, wvalue);
1028 1029 1030
        wvalue = *((u16 *) & p_str[(idx - 1) / 8]);
        wvalue = (wvalue >> ((idx - 1) % 8)) & mask;
        idx += window_size;
1031
        booth_recode_w7(&sign2, &digit2, wvalue);
1032 1033 1034
        wvalue = *((u16 *) & p_str[(idx - 1) / 8]);
        wvalue = (wvalue >> ((idx - 1) % 8)) & mask;
        idx += window_size;
1035 1036
        booth_recode_w7(&sign3, &digit3, wvalue);

A
Andy Polyakov 已提交
1037
        ecp_nistz256_avx2_multi_gather_w7(point_arr,
1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062
                                          preComputedTable[4 * i],
                                          digit0, digit1, digit2, digit3);

        ecp_nistz256_neg(tmp, point_arr[0].Y);
        copy_conditional(point_arr[0].Y, tmp, sign0);
        ecp_nistz256_neg(tmp, point_arr[1].Y);
        copy_conditional(point_arr[1].Y, tmp, sign1);
        ecp_nistz256_neg(tmp, point_arr[2].Y);
        copy_conditional(point_arr[2].Y, tmp, sign2);
        ecp_nistz256_neg(tmp, point_arr[3].Y);
        copy_conditional(point_arr[3].Y, tmp, sign3);

        ecp_nistz256_avx2_transpose_convert(bX4, point_arr);
        ecp_nistz256_avx2_to_mont(bX4, bX4);
        ecp_nistz256_avx2_to_mont(&bX4[4 * 9], &bX4[4 * 9]);

        ecp_nistz256_avx2_point_add_affine_x4(aX4, aX4, bX4);
    }

    ecp_nistz256_avx2_from_mont(&aX4[4 * 9 * 0], &aX4[4 * 9 * 0]);
    ecp_nistz256_avx2_from_mont(&aX4[4 * 9 * 1], &aX4[4 * 9 * 1]);
    ecp_nistz256_avx2_from_mont(&aX4[4 * 9 * 2], &aX4[4 * 9 * 2]);

    ecp_nistz256_avx2_convert_transpose_back(res_point_arr, aX4);
    /* Last window is performed serially */
1063 1064
    wvalue = *((u16 *) & p_str[(idx - 1) / 8]);
    wvalue = (wvalue >> ((idx - 1) % 8)) & mask;
1065
    booth_recode_w7(&sign0, &digit0, wvalue);
1066 1067
    ecp_nistz256_gather_w7((P256_POINT_AFFINE *)r,
                           preComputedTable[36], digit0);
1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079
    ecp_nistz256_neg(tmp, r->Y);
    copy_conditional(r->Y, tmp, sign0);
    memcpy(r->Z, ONE, sizeof(ONE));
    /* Sum the four windows */
    ecp_nistz256_point_add(r, r, &res_point_arr[0]);
    ecp_nistz256_point_add(r, r, &res_point_arr[1]);
    ecp_nistz256_point_add(r, r, &res_point_arr[2]);
    ecp_nistz256_point_add(r, r, &res_point_arr[3]);
}
# endif
#endif

E
Emilia Kasper 已提交
1080 1081 1082
__owur static int ecp_nistz256_set_from_affine(EC_POINT *out, const EC_GROUP *group,
                                               const P256_POINT_AFFINE *in,
                                               BN_CTX *ctx)
1083
{
1084
    BIGNUM *x, *y;
1085 1086 1087
    BN_ULONG d_x[P256_LIMBS], d_y[P256_LIMBS];
    int ret = 0;

1088
    x = BN_new();
1089
    if (x == NULL)
1090 1091
        return 0;
    y = BN_new();
1092
    if (y == NULL) {
1093 1094 1095
        BN_free(x);
        return 0;
    }
1096
    memcpy(d_x, in->X, sizeof(d_x));
1097
    bn_set_static_words(x, d_x, P256_LIMBS);
1098 1099

    memcpy(d_y, in->Y, sizeof(d_y));
1100 1101 1102
    bn_set_static_words(y, d_y, P256_LIMBS);

    ret = EC_POINT_set_affine_coordinates_GFp(group, out, x, y, ctx);
1103

R
Rich Salz 已提交
1104 1105
    BN_free(x);
    BN_free(y);
1106 1107 1108 1109 1110

    return ret;
}

/* r = scalar*G + sum(scalars[i]*points[i]) */
E
Emilia Kasper 已提交
1111 1112 1113 1114 1115 1116
__owur static int ecp_nistz256_points_mul(const EC_GROUP *group,
                                          EC_POINT *r,
                                          const BIGNUM *scalar,
                                          size_t num,
                                          const EC_POINT *points[],
                                          const BIGNUM *scalars[], BN_CTX *ctx)
1117 1118
{
    int i = 0, ret = 0, no_precomp_for_generator = 0, p_is_infinity = 0;
D
Dr. Stephen Henson 已提交
1119
    size_t j;
1120 1121
    unsigned char p_str[33] = { 0 };
    const PRECOMP256_ROW *preComputedTable = NULL;
1122
    const NISTZ256_PRE_COMP *pre_comp = NULL;
1123
    const EC_POINT *generator = NULL;
1124
    BN_CTX *new_ctx = NULL;
1125 1126
    const BIGNUM **new_scalars = NULL;
    const EC_POINT **new_points = NULL;
1127
    unsigned int idx = 0;
1128 1129 1130 1131 1132 1133 1134 1135 1136
    const unsigned int window_size = 7;
    const unsigned int mask = (1 << (window_size + 1)) - 1;
    unsigned int wvalue;
    ALIGN32 union {
        P256_POINT p;
        P256_POINT_AFFINE a;
    } t, p;
    BIGNUM *tmp_scalar;

1137
    if ((num + 1) == 0 || (num + 1) > OPENSSL_MALLOC_MAX_NELEMS(void *)) {
A
Andy Polyakov 已提交
1138 1139 1140 1141
        ECerr(EC_F_ECP_NISTZ256_POINTS_MUL, ERR_R_MALLOC_FAILURE);
        return 0;
    }

1142
    if (group->meth != r->meth) {
1143
        ECerr(EC_F_ECP_NISTZ256_POINTS_MUL, EC_R_INCOMPATIBLE_OBJECTS);
1144 1145
        return 0;
    }
1146

1147 1148 1149
    if ((scalar == NULL) && (num == 0))
        return EC_POINT_set_to_infinity(group, r);

D
Dr. Stephen Henson 已提交
1150 1151
    for (j = 0; j < num; j++) {
        if (group->meth != points[j]->meth) {
1152
            ECerr(EC_F_ECP_NISTZ256_POINTS_MUL, EC_R_INCOMPATIBLE_OBJECTS);
1153 1154 1155 1156
            return 0;
        }
    }

1157 1158 1159 1160 1161 1162 1163
    if (ctx == NULL) {
        ctx = new_ctx = BN_CTX_new();
        if (ctx == NULL)
            goto err;
    }

    BN_CTX_start(ctx);
1164 1165 1166 1167

    if (scalar) {
        generator = EC_GROUP_get0_generator(group);
        if (generator == NULL) {
1168
            ECerr(EC_F_ECP_NISTZ256_POINTS_MUL, EC_R_UNDEFINED_GENERATOR);
1169 1170 1171 1172
            goto err;
        }

        /* look if we can use precomputed multiples of generator */
1173
        pre_comp = group->pre_comp.nistz256;
1174 1175

        if (pre_comp) {
1176 1177 1178 1179
            /*
             * If there is a precomputed table for the generator, check that
             * it was generated with the same generator.
             */
1180 1181 1182 1183
            EC_POINT *pre_comp_generator = EC_POINT_new(group);
            if (pre_comp_generator == NULL)
                goto err;

A
Andy Polyakov 已提交
1184 1185
            if (!ecp_nistz256_set_from_affine(pre_comp_generator,
                                              group, pre_comp->precomp[0],
1186 1187
                                              ctx)) {
                EC_POINT_free(pre_comp_generator);
1188
                goto err;
1189
            }
1190 1191 1192 1193 1194 1195 1196 1197

            if (0 == EC_POINT_cmp(group, generator, pre_comp_generator, ctx))
                preComputedTable = (const PRECOMP256_ROW *)pre_comp->precomp;

            EC_POINT_free(pre_comp_generator);
        }

        if (preComputedTable == NULL && ecp_nistz256_is_affine_G(generator)) {
1198 1199 1200 1201 1202 1203
            /*
             * If there is no precomputed data, but the generator is the
             * default, a hardcoded table of precomputed data is used. This
             * is because applications, such as Apache, do not use
             * EC_KEY_precompute_mult.
             */
A
Andy Polyakov 已提交
1204
            preComputedTable = ecp_nistz256_precomputed;
1205 1206 1207 1208 1209 1210 1211 1212
        }

        if (preComputedTable) {
            if ((BN_num_bits(scalar) > 256)
                || BN_is_negative(scalar)) {
                if ((tmp_scalar = BN_CTX_get(ctx)) == NULL)
                    goto err;

1213
                if (!BN_nnmod(tmp_scalar, scalar, group->order, ctx)) {
1214
                    ECerr(EC_F_ECP_NISTZ256_POINTS_MUL, ERR_R_BN_LIB);
1215 1216 1217 1218 1219
                    goto err;
                }
                scalar = tmp_scalar;
            }

1220 1221
            for (i = 0; i < bn_get_top(scalar) * BN_BYTES; i += BN_BYTES) {
                BN_ULONG d = bn_get_words(scalar)[i / BN_BYTES];
1222

1223 1224 1225 1226
                p_str[i + 0] = (unsigned char)d;
                p_str[i + 1] = (unsigned char)(d >> 8);
                p_str[i + 2] = (unsigned char)(d >> 16);
                p_str[i + 3] = (unsigned char)(d >>= 24);
1227 1228
                if (BN_BYTES == 8) {
                    d >>= 8;
1229 1230 1231 1232
                    p_str[i + 4] = (unsigned char)d;
                    p_str[i + 5] = (unsigned char)(d >> 8);
                    p_str[i + 6] = (unsigned char)(d >> 16);
                    p_str[i + 7] = (unsigned char)(d >> 24);
1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246
                }
            }

            for (; i < 33; i++)
                p_str[i] = 0;

#if defined(ECP_NISTZ256_AVX2)
            if (ecp_nistz_avx2_eligible()) {
                ecp_nistz256_avx2_mul_g(&p.p, p_str, preComputedTable);
            } else
#endif
            {
                /* First window */
                wvalue = (p_str[0] << 1) & mask;
1247
                idx += window_size;
1248 1249 1250

                wvalue = _booth_recode_w7(wvalue);

1251 1252
                ecp_nistz256_gather_w7(&p.a, preComputedTable[0],
                                       wvalue >> 1);
1253 1254 1255 1256 1257 1258 1259

                ecp_nistz256_neg(p.p.Z, p.p.Y);
                copy_conditional(p.p.Y, p.p.Z, wvalue & 1);

                memcpy(p.p.Z, ONE, sizeof(ONE));

                for (i = 1; i < 37; i++) {
1260
                    unsigned int off = (idx - 1) / 8;
1261
                    wvalue = p_str[off] | p_str[off + 1] << 8;
1262 1263
                    wvalue = (wvalue >> ((idx - 1) % 8)) & mask;
                    idx += window_size;
1264 1265 1266

                    wvalue = _booth_recode_w7(wvalue);

A
Andy Polyakov 已提交
1267
                    ecp_nistz256_gather_w7(&t.a,
1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283
                                           preComputedTable[i], wvalue >> 1);

                    ecp_nistz256_neg(t.p.Z, t.a.Y);
                    copy_conditional(t.a.Y, t.p.Z, wvalue & 1);

                    ecp_nistz256_point_add_affine(&p.p, &p.p, &t.a);
                }
            }
        } else {
            p_is_infinity = 1;
            no_precomp_for_generator = 1;
        }
    } else
        p_is_infinity = 1;

    if (no_precomp_for_generator) {
1284 1285 1286 1287
        /*
         * Without a precomputed table for the generator, it has to be
         * handled like a normal point.
         */
1288
        new_scalars = OPENSSL_malloc((num + 1) * sizeof(BIGNUM *));
1289
        if (new_scalars == NULL) {
1290
            ECerr(EC_F_ECP_NISTZ256_POINTS_MUL, ERR_R_MALLOC_FAILURE);
1291
            goto err;
1292 1293 1294
        }

        new_points = OPENSSL_malloc((num + 1) * sizeof(EC_POINT *));
1295
        if (new_points == NULL) {
1296
            ECerr(EC_F_ECP_NISTZ256_POINTS_MUL, ERR_R_MALLOC_FAILURE);
1297
            goto err;
1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314
        }

        memcpy(new_scalars, scalars, num * sizeof(BIGNUM *));
        new_scalars[num] = scalar;
        memcpy(new_points, points, num * sizeof(EC_POINT *));
        new_points[num] = generator;

        scalars = new_scalars;
        points = new_points;
        num++;
    }

    if (num) {
        P256_POINT *out = &t.p;
        if (p_is_infinity)
            out = &p.p;

1315 1316
        if (!ecp_nistz256_windowed_mul(group, out, scalars, points, num, ctx))
            goto err;
1317 1318 1319 1320 1321

        if (!p_is_infinity)
            ecp_nistz256_point_add(&p.p, &p.p, out);
    }

1322
    /* Not constant-time, but we're only operating on the public output. */
1323 1324 1325 1326 1327
    if (!bn_set_words(r->X, p.p.X, P256_LIMBS) ||
        !bn_set_words(r->Y, p.p.Y, P256_LIMBS) ||
        !bn_set_words(r->Z, p.p.Z, P256_LIMBS)) {
        goto err;
    }
1328
    r->Z_is_one = is_one(p.p.Z) & 1;
1329 1330 1331

    ret = 1;

1332 1333 1334 1335
err:
    if (ctx)
        BN_CTX_end(ctx);
    BN_CTX_free(new_ctx);
R
Rich Salz 已提交
1336 1337
    OPENSSL_free(new_points);
    OPENSSL_free(new_scalars);
1338 1339 1340
    return ret;
}

E
Emilia Kasper 已提交
1341 1342 1343
__owur static int ecp_nistz256_get_affine(const EC_GROUP *group,
                                          const EC_POINT *point,
                                          BIGNUM *x, BIGNUM *y, BN_CTX *ctx)
1344 1345 1346 1347 1348 1349
{
    BN_ULONG z_inv2[P256_LIMBS];
    BN_ULONG z_inv3[P256_LIMBS];
    BN_ULONG x_aff[P256_LIMBS];
    BN_ULONG y_aff[P256_LIMBS];
    BN_ULONG point_x[P256_LIMBS], point_y[P256_LIMBS], point_z[P256_LIMBS];
1350
    BN_ULONG x_ret[P256_LIMBS], y_ret[P256_LIMBS];
1351 1352

    if (EC_POINT_is_at_infinity(group, point)) {
1353
        ECerr(EC_F_ECP_NISTZ256_GET_AFFINE, EC_R_POINT_AT_INFINITY);
1354 1355 1356
        return 0;
    }

1357 1358 1359
    if (!ecp_nistz256_bignum_to_field_elem(point_x, point->X) ||
        !ecp_nistz256_bignum_to_field_elem(point_y, point->Y) ||
        !ecp_nistz256_bignum_to_field_elem(point_z, point->Z)) {
1360
        ECerr(EC_F_ECP_NISTZ256_GET_AFFINE, EC_R_COORDINATES_OUT_OF_RANGE);
1361 1362 1363 1364 1365 1366 1367 1368
        return 0;
    }

    ecp_nistz256_mod_inverse(z_inv3, point_z);
    ecp_nistz256_sqr_mont(z_inv2, z_inv3);
    ecp_nistz256_mul_mont(x_aff, z_inv2, point_x);

    if (x != NULL) {
1369 1370 1371
        ecp_nistz256_from_mont(x_ret, x_aff);
        if (!bn_set_words(x, x_ret, P256_LIMBS))
            return 0;
1372 1373 1374 1375 1376
    }

    if (y != NULL) {
        ecp_nistz256_mul_mont(z_inv3, z_inv3, z_inv2);
        ecp_nistz256_mul_mont(y_aff, z_inv3, point_y);
1377 1378 1379
        ecp_nistz256_from_mont(y_ret, y_aff);
        if (!bn_set_words(y, y_ret, P256_LIMBS))
            return 0;
1380 1381 1382 1383 1384
    }

    return 1;
}

1385
static NISTZ256_PRE_COMP *ecp_nistz256_pre_comp_new(const EC_GROUP *group)
1386
{
1387
    NISTZ256_PRE_COMP *ret = NULL;
1388 1389 1390 1391

    if (!group)
        return NULL;

1392
    ret = OPENSSL_zalloc(sizeof(*ret));
1393

1394
    if (ret == NULL) {
1395
        ECerr(EC_F_ECP_NISTZ256_PRE_COMP_NEW, ERR_R_MALLOC_FAILURE);
1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406
        return ret;
    }

    ret->group = group;
    ret->w = 6;                 /* default */
    ret->precomp = NULL;
    ret->precomp_storage = NULL;
    ret->references = 1;
    return ret;
}

1407
NISTZ256_PRE_COMP *EC_nistz256_pre_comp_dup(NISTZ256_PRE_COMP *p)
1408
{
1409 1410 1411
    if (p != NULL)
        CRYPTO_add(&p->references, 1, CRYPTO_LOCK_EC_PRE_COMP);
    return p;
1412 1413
}

1414
void EC_nistz256_pre_comp_free(NISTZ256_PRE_COMP *pre)
1415
{
1416 1417
    if (pre == NULL
            || CRYPTO_add(&pre->references, -1, CRYPTO_LOCK_EC_PRE_COMP) > 0)
1418
        return;
R
Rich Salz 已提交
1419
    OPENSSL_free(pre->precomp_storage);
1420 1421 1422 1423
    OPENSSL_free(pre);
}


1424
static int ecp_nistz256_window_have_precompute_mult(const EC_GROUP *group)
1425 1426 1427
{
    /* There is a hard-coded table for the default generator. */
    const EC_POINT *generator = EC_GROUP_get0_generator(group);
1428

1429 1430 1431 1432 1433
    if (generator != NULL && ecp_nistz256_is_affine_G(generator)) {
        /* There is a hard-coded table for the default generator. */
        return 1;
    }

1434
    return HAVEPRECOMP(group, nistz256);
1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480
}

const EC_METHOD *EC_GFp_nistz256_method(void)
{
    static const EC_METHOD ret = {
        EC_FLAGS_DEFAULT_OCT,
        NID_X9_62_prime_field,
        ec_GFp_mont_group_init,
        ec_GFp_mont_group_finish,
        ec_GFp_mont_group_clear_finish,
        ec_GFp_mont_group_copy,
        ec_GFp_mont_group_set_curve,
        ec_GFp_simple_group_get_curve,
        ec_GFp_simple_group_get_degree,
        ec_GFp_simple_group_check_discriminant,
        ec_GFp_simple_point_init,
        ec_GFp_simple_point_finish,
        ec_GFp_simple_point_clear_finish,
        ec_GFp_simple_point_copy,
        ec_GFp_simple_point_set_to_infinity,
        ec_GFp_simple_set_Jprojective_coordinates_GFp,
        ec_GFp_simple_get_Jprojective_coordinates_GFp,
        ec_GFp_simple_point_set_affine_coordinates,
        ecp_nistz256_get_affine,
        0, 0, 0,
        ec_GFp_simple_add,
        ec_GFp_simple_dbl,
        ec_GFp_simple_invert,
        ec_GFp_simple_is_at_infinity,
        ec_GFp_simple_is_on_curve,
        ec_GFp_simple_cmp,
        ec_GFp_simple_make_affine,
        ec_GFp_simple_points_make_affine,
        ecp_nistz256_points_mul,                    /* mul */
        ecp_nistz256_mult_precompute,               /* precompute_mult */
        ecp_nistz256_window_have_precompute_mult,   /* have_precompute_mult */
        ec_GFp_mont_field_mul,
        ec_GFp_mont_field_sqr,
        0,                                          /* field_div */
        ec_GFp_mont_field_encode,
        ec_GFp_mont_field_decode,
        ec_GFp_mont_field_set_to_one
    };

    return &ret;
}