math: new exp2f and expf

from https://github.com/ARM-software/optimized-routines,
commit 04884bd04eac4b251da4026900010ea7d8850edc

In expf TOINT_INTRINSICS is kept, but is unused, it would require support
for __builtin_round and __builtin_lround as single instruction.

code size change: +94 bytes.
benchmark on x86_64 before, after, speedup:

-Os:
  expf rthruput:   9.19 ns/call  8.11 ns/call 1.13x
   expf latency:  34.19 ns/call 18.77 ns/call 1.82x
 exp2f rthruput:   5.59 ns/call  6.52 ns/call 0.86x
  exp2f latency:  17.93 ns/call 16.70 ns/call 1.07x
-O3:
  expf rthruput:   9.12 ns/call  4.92 ns/call 1.85x
   expf latency:  34.44 ns/call 18.99 ns/call 1.81x
 exp2f rthruput:   5.58 ns/call  4.49 ns/call 1.24x
  exp2f latency:  17.95 ns/call 16.94 ns/call 1.06x

src/internal/libm.h

@@ -64,6 +64,22 @@ union ldshape {
 /* Support signaling NaNs.  */
 #define WANT_SNAN 0
 
+#ifndef TOINT_INTRINSICS
+#define TOINT_INTRINSICS 0
+#endif
+
+#if TOINT_INTRINSICS
+/* Round x to nearest int in all rounding modes, ties have to be rounded
+   consistently with converttoint so the results match.  If the result
+   would be outside of [-2^31, 2^31-1] then the semantics is unspecified.  */
+static double_t roundtoint(double_t);
+
+/* Convert x to nearest int in all rounding modes, ties have to be rounded
+   consistently with roundtoint.  If the result is not representible in an
+   int32_t then the semantics is unspecified.  */
+static int32_t converttoint(double_t);
+#endif
+
 /* Helps static branch prediction so hot path can be better optimized.  */
 #ifdef __GNUC__
 #define predict_true(x) __builtin_expect(!!(x), 1)

src/math/exp2f.c

-/* origin: FreeBSD /usr/src/lib/msun/src/s_exp2f.c */
-/*-
- * Copyright (c) 2005 David Schultz <das@FreeBSD.ORG>
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
+/*
+ * Single-precision 2^x function.
  *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
+ * Copyright (c) 2017-2018, Arm Limited.
+ * SPDX-License-Identifier: MIT
  */
 
+#include <math.h>
+#include <stdint.h>
 #include "libm.h"
+#include "exp2f_data.h"
 
-#define TBLSIZE 16
+/*
+EXP2F_TABLE_BITS = 5
+EXP2F_POLY_ORDER = 3
 
-static const float
-redux = 0x1.8p23f / TBLSIZE,
-P1    = 0x1.62e430p-1f,
-P2    = 0x1.ebfbe0p-3f,
-P3    = 0x1.c6b348p-5f,
-P4    = 0x1.3b2c9cp-7f;
+ULP error: 0.502 (nearest rounding.)
+Relative error: 1.69 * 2^-34 in [-1/64, 1/64] (before rounding.)
+Wrong count: 168353 (all nearest rounding wrong results with fma.)
+Non-nearest ULP error: 1 (rounded ULP error)
+*/
 
-static const double exp2ft[TBLSIZE] = {
-  0x1.6a09e667f3bcdp-1,
-  0x1.7a11473eb0187p-1,
-  0x1.8ace5422aa0dbp-1,
-  0x1.9c49182a3f090p-1,
-  0x1.ae89f995ad3adp-1,
-  0x1.c199bdd85529cp-1,
-  0x1.d5818dcfba487p-1,
-  0x1.ea4afa2a490dap-1,
-  0x1.0000000000000p+0,
-  0x1.0b5586cf9890fp+0,
-  0x1.172b83c7d517bp+0,
-  0x1.2387a6e756238p+0,
-  0x1.306fe0a31b715p+0,
-  0x1.3dea64c123422p+0,
-  0x1.4bfdad5362a27p+0,
-  0x1.5ab07dd485429p+0,
-};
+#define N (1 << EXP2F_TABLE_BITS)
+#define T __exp2f_data.tab
+#define C __exp2f_data.poly
+#define SHIFT __exp2f_data.shift_scaled
+
+static inline uint32_t top12(float x)
+{
+	return asuint(x) >> 20;
+}
 
-/*
- * exp2f(x): compute the base 2 exponential of x
- *
- * Accuracy: Peak error < 0.501 ulp; location of peak: -0.030110927.
- *
- * Method: (equally-spaced tables)
- *
- *   Reduce x:
- *     x = k + y, for integer k and |y| <= 1/2.
- *     Thus we have exp2f(x) = 2**k * exp2(y).
- *
- *   Reduce y:
- *     y = i/TBLSIZE + z for integer i near y * TBLSIZE.
- *     Thus we have exp2(y) = exp2(i/TBLSIZE) * exp2(z),
- *     with |z| <= 2**-(TBLSIZE+1).
- *
- *   We compute exp2(i/TBLSIZE) via table lookup and exp2(z) via a
- *   degree-4 minimax polynomial with maximum error under 1.4 * 2**-33.
- *   Using double precision for everything except the reduction makes
- *   roundoff error insignificant and simplifies the scaling step.
- *
- *   This method is due to Tang, but I do not use his suggested parameters:
- *
- *      Tang, P.  Table-driven Implementation of the Exponential Function
- *      in IEEE Floating-Point Arithmetic.  TOMS 15(2), 144-157 (1989).
- */
 float exp2f(float x)
 {
-	double_t t, r, z;
-	union {float f; uint32_t i;} u = {x};
-	union {double f; uint64_t i;} uk;
-	uint32_t ix, i0, k;
+	uint32_t abstop;
+	uint64_t ki, t;
+	double_t kd, xd, z, r, r2, y, s;
 
-	/* Filter out exceptional cases. */
-	ix = u.i & 0x7fffffff;
-	if (ix > 0x42fc0000) {  /* |x| > 126 */
-		if (ix > 0x7f800000) /* NaN */
-			return x;
-		if (u.i >= 0x43000000 && u.i < 0x80000000) {  /* x >= 128 */
-			x *= 0x1p127f;
-			return x;
-		}
-		if (u.i >= 0x80000000) {  /* x < -126 */
-			if (u.i >= 0xc3160000 || (u.i & 0x0000ffff))
-				FORCE_EVAL(-0x1p-149f/x);
-			if (u.i >= 0xc3160000)  /* x <= -150 */
-				return 0;
-		}
-	} else if (ix <= 0x33000000) {  /* |x| <= 0x1p-25 */
-		return 1.0f + x;
+	xd = (double_t)x;
+	abstop = top12(x) & 0x7ff;
+	if (predict_false(abstop >= top12(128.0f))) {
+		/* |x| >= 128 or x is nan.  */
+		if (asuint(x) == asuint(-INFINITY))
+			return 0.0f;
+		if (abstop >= top12(INFINITY))
+			return x + x;
+		if (x > 0.0f)
+			return __math_oflowf(0);
+		if (x <= -150.0f)
+			return __math_uflowf(0);
 	}
 
-	/* Reduce x, computing z, i0, and k. */
-	u.f = x + redux;
-	i0 = u.i;
-	i0 += TBLSIZE / 2;
-	k = i0 / TBLSIZE;
-	uk.i = (uint64_t)(0x3ff + k)<<52;
-	i0 &= TBLSIZE - 1;
-	u.f -= redux;
-	z = x - u.f;
-	/* Compute r = exp2(y) = exp2ft[i0] * p(z). */
-	r = exp2ft[i0];
-	t = r * z;
-	r = r + t * (P1 + z * P2) + t * (z * z) * (P3 + z * P4);
+	/* x = k/N + r with r in [-1/(2N), 1/(2N)] and int k.  */
+	kd = eval_as_double(xd + SHIFT);
+	ki = asuint64(kd);
+	kd -= SHIFT; /* k/N for int k.  */
+	r = xd - kd;
 
-	/* Scale by 2**k */
-	return r * uk.f;
+	/* exp2(x) = 2^(k/N) * 2^r ~= s * (C0*r^3 + C1*r^2 + C2*r + 1) */
+	t = T[ki % N];
+	t += ki << (52 - EXP2F_TABLE_BITS);
+	s = asdouble(t);
+	z = C[0] * r + C[1];
+	r2 = r * r;
+	y = C[2] * r + 1;
+	y = z * r2 + y;
+	y = y * s;
+	return eval_as_float(y);
 }

src/math/exp2f_data.c

+/*
+ * Shared data between expf, exp2f and powf.
+ *
+ * Copyright (c) 2017-2018, Arm Limited.
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "exp2f_data.h"
+
+#define N (1 << EXP2F_TABLE_BITS)
+
+const struct exp2f_data __exp2f_data = {
+  /* tab[i] = uint(2^(i/N)) - (i << 52-BITS)
+     used for computing 2^(k/N) for an int |k| < 150 N as
+     double(tab[k%N] + (k << 52-BITS)) */
+  .tab = {
+0x3ff0000000000000, 0x3fefd9b0d3158574, 0x3fefb5586cf9890f, 0x3fef9301d0125b51,
+0x3fef72b83c7d517b, 0x3fef54873168b9aa, 0x3fef387a6e756238, 0x3fef1e9df51fdee1,
+0x3fef06fe0a31b715, 0x3feef1a7373aa9cb, 0x3feedea64c123422, 0x3feece086061892d,
+0x3feebfdad5362a27, 0x3feeb42b569d4f82, 0x3feeab07dd485429, 0x3feea47eb03a5585,
+0x3feea09e667f3bcd, 0x3fee9f75e8ec5f74, 0x3feea11473eb0187, 0x3feea589994cce13,
+0x3feeace5422aa0db, 0x3feeb737b0cdc5e5, 0x3feec49182a3f090, 0x3feed503b23e255d,
+0x3feee89f995ad3ad, 0x3feeff76f2fb5e47, 0x3fef199bdd85529c, 0x3fef3720dcef9069,
+0x3fef5818dcfba487, 0x3fef7c97337b9b5f, 0x3fefa4afa2a490da, 0x3fefd0765b6e4540,
+  },
+  .shift_scaled = 0x1.8p+52 / N,
+  .poly = {
+  0x1.c6af84b912394p-5, 0x1.ebfce50fac4f3p-3, 0x1.62e42ff0c52d6p-1,
+  },
+  .shift = 0x1.8p+52,
+  .invln2_scaled = 0x1.71547652b82fep+0 * N,
+  .poly_scaled = {
+  0x1.c6af84b912394p-5/N/N/N, 0x1.ebfce50fac4f3p-3/N/N, 0x1.62e42ff0c52d6p-1/N,
+  },
+};

src/math/exp2f_data.h

+/*
+ * Copyright (c) 2017-2018, Arm Limited.
+ * SPDX-License-Identifier: MIT
+ */
+#ifndef _EXP2F_DATA_H
+#define _EXP2F_DATA_H
+
+#include <features.h>
+#include <stdint.h>
+
+/* Shared between expf, exp2f and powf.  */
+#define EXP2F_TABLE_BITS 5
+#define EXP2F_POLY_ORDER 3
+extern hidden const struct exp2f_data {
+	uint64_t tab[1 << EXP2F_TABLE_BITS];
+	double shift_scaled;
+	double poly[EXP2F_POLY_ORDER];
+	double shift;
+	double invln2_scaled;
+	double poly_scaled[EXP2F_POLY_ORDER];
+} __exp2f_data;
+
+#endif

src/math/expf.c

-/* origin: FreeBSD /usr/src/lib/msun/src/e_expf.c */
 /*
- * Conversion to float by Ian Lance Taylor, Cygnus Support, ian@cygnus.com.
- */
-/*
- * ====================================================
- * Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved.
+ * Single-precision e^x function.
  *
- * Developed at SunPro, a Sun Microsystems, Inc. business.
- * Permission to use, copy, modify, and distribute this
- * software is freely granted, provided that this notice
- * is preserved.
- * ====================================================
+ * Copyright (c) 2017-2018, Arm Limited.
+ * SPDX-License-Identifier: MIT
  */
 
+#include <math.h>
+#include <stdint.h>
 #include "libm.h"
+#include "exp2f_data.h"
 
-static const float
-half[2] = {0.5,-0.5},
-ln2hi   = 6.9314575195e-1f,  /* 0x3f317200 */
-ln2lo   = 1.4286067653e-6f,  /* 0x35bfbe8e */
-invln2  = 1.4426950216e+0f,  /* 0x3fb8aa3b */
 /*
- * Domain [-0.34568, 0.34568], range ~[-4.278e-9, 4.447e-9]:
- * |x*(exp(x)+1)/(exp(x)-1) - p(x)| < 2**-27.74
- */
-P1 =  1.6666625440e-1f, /*  0xaaaa8f.0p-26 */
-P2 = -2.7667332906e-3f; /* -0xb55215.0p-32 */
+EXP2F_TABLE_BITS = 5
+EXP2F_POLY_ORDER = 3
 
-float expf(float x)
+ULP error: 0.502 (nearest rounding.)
+Relative error: 1.69 * 2^-34 in [-ln2/64, ln2/64] (before rounding.)
+Wrong count: 170635 (all nearest rounding wrong results with fma.)
+Non-nearest ULP error: 1 (rounded ULP error)
+*/
+
+#define N (1 << EXP2F_TABLE_BITS)
+#define InvLn2N __exp2f_data.invln2_scaled
+#define T __exp2f_data.tab
+#define C __exp2f_data.poly_scaled
+
+static inline uint32_t top12(float x)
 {
-	float_t hi, lo, c, xx, y;
-	int k, sign;
-	uint32_t hx;
+	return asuint(x) >> 20;
+}
 
-	GET_FLOAT_WORD(hx, x);
-	sign = hx >> 31;   /* sign bit of x */
-	hx &= 0x7fffffff;  /* high word of |x| */
+float expf(float x)
+{
+	uint32_t abstop;
+	uint64_t ki, t;
+	double_t kd, xd, z, r, r2, y, s;
 
-	/* special cases */
-	if (hx >= 0x42aeac50) {  /* if |x| >= -87.33655f or NaN */
-		if (hx > 0x7f800000) /* NaN */
-			return x;
-		if (hx >= 0x42b17218 && !sign) {  /* x >= 88.722839f */
-			/* overflow */
-			x *= 0x1p127f;
-			return x;
-		}
-		if (sign) {
-			/* underflow */
-			FORCE_EVAL(-0x1p-149f/x);
-			if (hx >= 0x42cff1b5)  /* x <= -103.972084f */
-				return 0;
-		}
+	xd = (double_t)x;
+	abstop = top12(x) & 0x7ff;
+	if (predict_false(abstop >= top12(88.0f))) {
+		/* |x| >= 88 or x is nan.  */
+		if (asuint(x) == asuint(-INFINITY))
+			return 0.0f;
+		if (abstop >= top12(INFINITY))
+			return x + x;
+		if (x > 0x1.62e42ep6f) /* x > log(0x1p128) ~= 88.72 */
+			return __math_oflowf(0);
+		if (x < -0x1.9fe368p6f) /* x < log(0x1p-150) ~= -103.97 */
+			return __math_uflowf(0);
 	}
 
-	/* argument reduction */
-	if (hx > 0x3eb17218) {  /* if |x| > 0.5 ln2 */
-		if (hx > 0x3f851592)  /* if |x| > 1.5 ln2 */
-			k = invln2*x + half[sign];
-		else
-			k = 1 - sign - sign;
-		hi = x - k*ln2hi;  /* k*ln2hi is exact here */
-		lo = k*ln2lo;
-		x = hi - lo;
-	} else if (hx > 0x39000000) {  /* |x| > 2**-14 */
-		k = 0;
-		hi = x;
-		lo = 0;
-	} else {
-		/* raise inexact */
-		FORCE_EVAL(0x1p127f + x);
-		return 1 + x;
-	}
+	/* x*N/Ln2 = k + r with r in [-1/2, 1/2] and int k.  */
+	z = InvLn2N * xd;
+
+	/* Round and convert z to int, the result is in [-150*N, 128*N] and
+	   ideally ties-to-even rule is used, otherwise the magnitude of r
+	   can be bigger which gives larger approximation error.  */
+#if TOINT_INTRINSICS
+	kd = roundtoint(z);
+	ki = converttoint(z);
+#else
+# define SHIFT __exp2f_data.shift
+	kd = eval_as_double(z + SHIFT);
+	ki = asuint64(kd);
+	kd -= SHIFT;
+#endif
+	r = z - kd;
 
-	/* x is now in primary range */
-	xx = x*x;
-	c = x - xx*(P1+xx*P2);
-	y = 1 + (x*c/(2-c) - lo + hi);
-	if (k == 0)
-		return y;
-	return scalbnf(y, k);
+	/* exp(x) = 2^(k/N) * 2^(r/N) ~= s * (C0*r^3 + C1*r^2 + C2*r + 1) */
+	t = T[ki % N];
+	t += ki << (52 - EXP2F_TABLE_BITS);
+	s = asdouble(t);
+	z = C[0] * r + C[1];
+	r2 = r * r;
+	y = C[2] * r + 1;
+	y = z * r2 + y;
+	y = y * s;
+	return eval_as_float(y);
 }