jit_kernel_test.cc 24.7 KB
Newer Older
T
tensor-tang 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15
/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */

#include "paddle/fluid/operators/math/jit_kernel.h"
T
tensor-tang 已提交
16
#include <cmath>    // for exp
T
tensor-tang 已提交
17
#include <cstring>  // for memcpy
T
tensor-tang 已提交
18
#include <random>
T
tensor-tang 已提交
19 20 21 22 23
#include <string>
#include <vector>
#include "gflags/gflags.h"
#include "glog/logging.h"
#include "gtest/gtest.h"
24
#include "paddle/fluid/operators/math/jit_kernel_refer.h"
P
peizhilin 已提交
25
#include "paddle/fluid/platform/port.h"
T
tensor-tang 已提交
26

T
tensor-tang 已提交
27 28 29 30 31 32 33 34
#ifdef PADDLE_WITH_MKLML
#include "paddle/fluid/platform/dynload/mklml.h"
#endif

#ifdef __AVX__
#include <immintrin.h>
#endif

T
tensor-tang 已提交
35 36
constexpr int repeat = 20000;

T
tensor-tang 已提交
37 38 39
// TODO(TJ): benchmark and test should be seperated,
// benchmark should verify more sizes

T
tensor-tang 已提交
40 41 42 43 44 45 46
inline double GetCurrentUS() {
  struct timeval time;
  gettimeofday(&time, NULL);
  return 1e+6 * time.tv_sec + time.tv_usec;
}

template <typename T>
T
tensor-tang 已提交
47 48
void RandomVec(const int n, T* a, const T lower = static_cast<T>(-20.f),
               const T upper = static_cast<T>(20.f)) {
T
tensor-tang 已提交
49 50 51 52 53 54 55 56
  static unsigned int seed = 100;
  std::mt19937 rng(seed++);
  std::uniform_real_distribution<double> uniform_dist(0, 1);
  for (int i = 0; i < n; ++i) {
    a[i] = static_cast<T>(uniform_dist(rng) * (upper - lower) + lower);
  }
}

T
tensor-tang 已提交
57 58 59 60 61 62 63 64 65 66
#if defined __AVX__ || defined __AVX2__
void vrelu_intri8(const int n, const float* x, float* y) {
  __m256 tmp = _mm256_loadu_ps(x);
  tmp = _mm256_max_ps(tmp, _mm256_setzero_ps());
  _mm256_storeu_ps(y, tmp);
}
#endif

TEST(JitKernel, vrelu) {
  namespace jit = paddle::operators::math::jitkernel;
67
  namespace refer = paddle::operators::math::jitkernel::refer;
T
tensor-tang 已提交
68
  for (int d : {3, 7, 8, 15, 16, 30, 256, 512}) {
T
tensor-tang 已提交
69 70 71 72 73 74 75 76 77 78
    std::vector<float> x(d);
    std::vector<float> zref(d), ztgt(d);
    RandomVec<float>(d, x.data(), -10.f, 1.f);
    const auto& ker =
        jit::KernelPool::Instance().template Get<jit::VReluKernel<float>>(d);
    const float* x_data = x.data();
    float* ztgt_data = ztgt.data();
    float* zref_data = zref.data();
    auto trefs = GetCurrentUS();
    for (int i = 0; i < repeat; ++i) {
79
      refer::VRelu<float>(x_data, zref_data, d);
T
tensor-tang 已提交
80 81 82 83 84 85 86 87 88
    }
    auto trefe = GetCurrentUS();
#if defined __AVX__ || defined __AVX2__
    if (d == 8) {
      auto si0 = GetCurrentUS();
      for (int i = 0; i < repeat; ++i) {
        vrelu_intri8(d, x_data, zref_data);
      }
      auto si1 = GetCurrentUS();
89
      VLOG(3) << "Vec size 8 intr takes: " << (si1 - si0) / repeat << " us";
T
tensor-tang 已提交
90 91 92 93
    }
#endif
    auto ttgts = GetCurrentUS();
    for (int i = 0; i < repeat; ++i) {
T
tensor-tang 已提交
94
      ker->Compute(x_data, ztgt_data, d);
T
tensor-tang 已提交
95 96
    }
    auto ttgte = GetCurrentUS();
M
minqiyang 已提交
97 98
    VLOG(3) << "Vec size " << d << ": refer takes: " << (trefe - trefs) / repeat
            << " us, tgt takes: " << (ttgte - ttgts) / repeat << " us";
T
tensor-tang 已提交
99 100 101 102 103 104
    for (int i = 0; i < d; ++i) {
      EXPECT_NEAR(ztgt_data[i], zref_data[i], 1e-3);
    }
  }
}

T
tensor-tang 已提交
105 106
TEST(JitKernel, vaddbias) {
  namespace jit = paddle::operators::math::jitkernel;
107
  namespace refer = paddle::operators::math::jitkernel::refer;
T
tensor-tang 已提交
108 109 110 111 112 113 114 115 116 117 118 119
  for (int d : {7, 8, 15, 16, 30, 64, 100, 128, 256}) {
    std::vector<float> x(d);
    std::vector<float> zref(d), ztgt(d);
    RandomVec<float>(d, x.data(), -2.f, 2.f);
    const auto& ker =
        jit::KernelPool::Instance().template Get<jit::VAddBiasKernel<float>>(d);
    const float a = 2.f;
    const float* x_data = x.data();
    float* ztgt_data = ztgt.data();
    float* zref_data = zref.data();
    auto trefs = GetCurrentUS();
    for (int i = 0; i < repeat; ++i) {
120
      refer::VAddBias<float>(&a, x_data, zref_data, d);
T
tensor-tang 已提交
121 122 123 124
    }
    auto trefe = GetCurrentUS();
    auto ttgts = GetCurrentUS();
    for (int i = 0; i < repeat; ++i) {
T
tensor-tang 已提交
125
      ker->Compute(&a, x_data, ztgt_data, d);
T
tensor-tang 已提交
126 127 128
    }
    auto ttgte = GetCurrentUS();

M
minqiyang 已提交
129 130
    VLOG(3) << "Vec size " << d << ": refer takes: " << (trefe - trefs) / repeat
            << " us, tgt takes: " << (ttgte - ttgts) / repeat << " us";
T
tensor-tang 已提交
131 132 133 134 135 136
    for (int i = 0; i < d; ++i) {
      EXPECT_NEAR(ztgt_data[i], zref_data[i], 1e-3);
    }
  }
}

T
tensor-tang 已提交
137 138 139 140 141 142 143 144
#ifdef PADDLE_WITH_MKLML
void vexp_mkl(const int n, const float* x, float* y) {
  paddle::platform::dynload::vsExp(n, x, y);
}
#endif

TEST(JitKernel, vexp) {
  namespace jit = paddle::operators::math::jitkernel;
145
  namespace refer = paddle::operators::math::jitkernel::refer;
T
tensor-tang 已提交
146
  for (int d : {1, 3, 4, 6, 7, 8, 12, 15, 16, 20, 30, 128, 256}) {
T
tensor-tang 已提交
147 148 149 150 151 152 153 154 155 156
    std::vector<float> x(d);
    std::vector<float> zref(d), ztgt(d);
    RandomVec<float>(d, x.data(), -2.f, 2.f);
    const auto& ker =
        jit::KernelPool::Instance().template Get<jit::VExpKernel<float>>(d);
    const float* x_data = x.data();
    float* ztgt_data = ztgt.data();
    float* zref_data = zref.data();
    auto trefs = GetCurrentUS();
    for (int i = 0; i < repeat; ++i) {
157
      refer::VExp<float>(x_data, zref_data, d);
T
tensor-tang 已提交
158 159 160 161 162 163 164 165 166 167 168 169 170
    }
    auto trefe = GetCurrentUS();

#ifdef PADDLE_WITH_MKLML
    auto tmkls = GetCurrentUS();
    for (int i = 0; i < repeat; ++i) {
      vexp_mkl(d, x_data, zref_data);
    }
    auto tmkle = GetCurrentUS();
#endif

    auto ttgts = GetCurrentUS();
    for (int i = 0; i < repeat; ++i) {
T
tensor-tang 已提交
171
      // ker->Compute(x_data, ztgt_data);
T
tensor-tang 已提交
172
      ker->Compute(x_data, ztgt_data, d);
T
tensor-tang 已提交
173 174 175
    }
    auto ttgte = GetCurrentUS();

M
minqiyang 已提交
176
    VLOG(3) << "Vec size " << d << ": refer takes: " << (trefe - trefs) / repeat
T
tensor-tang 已提交
177
#ifdef PADDLE_WITH_MKLML
M
minqiyang 已提交
178
            << " us, mkl takes: " << (tmkle - tmkls) / repeat << " us, "
T
tensor-tang 已提交
179
#else
M
minqiyang 已提交
180
            << " us, "
T
tensor-tang 已提交
181
#endif
182 183

            << "tgt takes: " << (ttgte - ttgts) / repeat << " us";
T
tensor-tang 已提交
184 185
    for (int i = 0; i < d; ++i) {
      EXPECT_NEAR(ztgt_data[i], zref_data[i], 1e-3);
186 187 188 189 190 191 192 193 194 195 196 197 198 199
    }
  }
}

void vsigmoid_better(
    const std::shared_ptr<
        const paddle::operators::math::jitkernel::VExpKernel<float>>& vexp,
    const int n, const float* x, float* y) {
  const float min = SIGMOID_THRESHOLD_MIN;
  const float max = SIGMOID_THRESHOLD_MAX;
  for (int i = 0; i < n; ++i) {
    y[i] = (x[i] < min) ? min : ((x[i] > max) ? max : x[i]);
    y[i] = 0.f - y[i];
  }
T
tensor-tang 已提交
200
  vexp->Compute(y, y, n);
201 202 203 204 205 206 207
  for (int i = 0; i < n; ++i) {
    y[i] = 1.f / (1.f + y[i]);
  }
}

TEST(JitKernel, vsigmoid) {
  namespace jit = paddle::operators::math::jitkernel;
208
  namespace refer = paddle::operators::math::jitkernel::refer;
T
tensor-tang 已提交
209
  for (int d : {1, 3, 4, 6, 7, 8, 15, 16, 30, 32, 64, 100, 128, 256}) {
210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226
    std::vector<float> x(d);
    std::vector<float> zref(d), ztgt(d);
    RandomVec<float>(d, x.data(), -2.f, 2.f);
    const auto& ker =
        jit::KernelPool::Instance().template Get<jit::VSigmoidKernel<float>>(d);
    const auto& vexp =
        jit::KernelPool::Instance().template Get<jit::VExpKernel<float>>(d);
    const float* x_data = x.data();
    float* ztgt_data = ztgt.data();
    float* zref_data = zref.data();
    auto tmkls = GetCurrentUS();
    for (int i = 0; i < repeat; ++i) {
      vsigmoid_better(vexp, d, x_data, zref_data);
    }
    auto tmkle = GetCurrentUS();
    auto trefs = GetCurrentUS();
    for (int i = 0; i < repeat; ++i) {
227
      refer::VSigmoid<float>(x_data, zref_data, d);
228 229 230 231
    }
    auto trefe = GetCurrentUS();
    auto ttgts = GetCurrentUS();
    for (int i = 0; i < repeat; ++i) {
T
tensor-tang 已提交
232
      ker->Compute(x_data, ztgt_data, d);
233 234 235
    }
    auto ttgte = GetCurrentUS();

M
minqiyang 已提交
236 237 238
    VLOG(3) << "Vec size " << d << ": refer takes: " << (trefe - trefs) / repeat
            << " us, better(jit exp) takes: " << (tmkle - tmkls) / repeat
            << " us, tgt takes: " << (ttgte - ttgts) / repeat << " us";
T
tensor-tang 已提交
239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254
    for (int i = 0; i < d; ++i) {
      EXPECT_NEAR(ztgt_data[i], zref_data[i], 1e-3);
    }
  }
}

void vtanh_better(
    const std::shared_ptr<
        const paddle::operators::math::jitkernel::VScalKernel<float>>& vscal,
    const std::shared_ptr<
        const paddle::operators::math::jitkernel::VSigmoidKernel<float>>&
        vsigmoid,
    const std::shared_ptr<
        const paddle::operators::math::jitkernel::VAddBiasKernel<float>>&
        vaddbias,
    const int n, const float* x, float* y) {
T
tensor-tang 已提交
255 256
  const float a = 2.f, b = -1.f;
  vscal->Compute(&a, x, y, n);
T
tensor-tang 已提交
257
  vsigmoid->Compute(y, y, n);
T
tensor-tang 已提交
258 259
  vscal->Compute(&a, y, y, n);
  vaddbias->Compute(&b, y, y, n);
T
tensor-tang 已提交
260 261 262 263
}

TEST(JitKernel, vtanh) {
  namespace jit = paddle::operators::math::jitkernel;
264
  namespace refer = paddle::operators::math::jitkernel::refer;
T
tensor-tang 已提交
265
  for (int d : {1, 2, 3, 4, 5, 6, 7, 8, 15, 16, 30, 32, 64, 100, 128, 256}) {
T
tensor-tang 已提交
266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286
    std::vector<float> x(d);
    std::vector<float> zref(d), ztgt(d);
    RandomVec<float>(d, x.data(), -2.f, 2.f);
    const auto& ker =
        jit::KernelPool::Instance().template Get<jit::VTanhKernel<float>>(d);
    const auto& vscal =
        jit::KernelPool::Instance().template Get<jit::VScalKernel<float>>(d);
    const auto& vsigmoid =
        jit::KernelPool::Instance().template Get<jit::VSigmoidKernel<float>>(d);
    const auto& vaddbias =
        jit::KernelPool::Instance().template Get<jit::VAddBiasKernel<float>>(d);
    const float* x_data = x.data();
    float* ztgt_data = ztgt.data();
    float* zref_data = zref.data();
    auto tmkls = GetCurrentUS();
    for (int i = 0; i < repeat; ++i) {
      vtanh_better(vscal, vsigmoid, vaddbias, d, x_data, zref_data);
    }
    auto tmkle = GetCurrentUS();
    auto trefs = GetCurrentUS();
    for (int i = 0; i < repeat; ++i) {
287
      refer::VTanh<float>(x_data, zref_data, d);
T
tensor-tang 已提交
288 289 290 291
    }
    auto trefe = GetCurrentUS();
    auto ttgts = GetCurrentUS();
    for (int i = 0; i < repeat; ++i) {
T
tensor-tang 已提交
292
      ker->Compute(x_data, ztgt_data, d);
T
tensor-tang 已提交
293 294 295
    }
    auto ttgte = GetCurrentUS();

M
minqiyang 已提交
296 297 298
    VLOG(3) << "Vec size " << d << ": refer takes: " << (trefe - trefs) / repeat
            << " us, better(jit exp) takes: " << (tmkle - tmkls) / repeat
            << " us, tgt takes: " << (ttgte - ttgts) / repeat << " us";
299 300
    for (int i = 0; i < d; ++i) {
      EXPECT_NEAR(ztgt_data[i], zref_data[i], 1e-3);
T
tensor-tang 已提交
301 302 303 304
    }
  }
}

T
tensor-tang 已提交
305 306 307 308 309 310 311 312 313 314 315 316
void lstm_ctht_better(
    const std::shared_ptr<
        const paddle::operators::math::jitkernel::VSigmoidKernel<float>>&
        vsigmoid_3d,
    const std::shared_ptr<
        const paddle::operators::math::jitkernel::VTanhKernel<float>>& vtanh_d,
    const std::shared_ptr<
        const paddle::operators::math::jitkernel::VMulKernel<float>>& vmul_d,
    const std::shared_ptr<
        const paddle::operators::math::jitkernel::VAddKernel<float>>& vadd_d,
    const int d, float* gates, const float* ct_1, float* ct, float* ht) {
  int d2 = d * 2;
T
tensor-tang 已提交
317 318
  vsigmoid_3d->Compute(gates + d, gates + d, 3 * d);
  vtanh_d->Compute(gates, gates, d);
T
tensor-tang 已提交
319 320
  vmul_d->Compute(gates, gates + d, gates + d, d);
  vmul_d->Compute(ct_1, gates + d2, gates + d2, d);
T
tensor-tang 已提交
321
  vadd_d->Compute(gates + d, gates + d2, ct, d);
T
tensor-tang 已提交
322
  /* H_t = act_cell(C_t) * ogated */
T
tensor-tang 已提交
323
  vtanh_d->Compute(ct, gates + d2, d);
T
tensor-tang 已提交
324
  vmul_d->Compute(gates + d2, gates + d * 3, ht, d);
T
tensor-tang 已提交
325 326 327 328
}

TEST(JitKernel, lstm) {
  namespace jit = paddle::operators::math::jitkernel;
329
  namespace refer = paddle::operators::math::jitkernel::refer;
T
tensor-tang 已提交
330
  for (int d : {1, 2, 3, 4, 5, 6, 7, 8, 15, 16, 30, 32, 64, 100}) {
T
tensor-tang 已提交
331 332 333 334 335 336 337 338 339
    int d4 = d * 4;
    int d3 = d * 3;
    std::vector<float> x(d4), xref(d4);
    std::vector<float> ct_1(d), ct_tgt(d), ht_tgt(d);
    std::vector<float> ct_ref(d), ht_ref(d);
    RandomVec<float>(d4, x.data(), -2.f, 2.f);
    RandomVec<float>(d, ct_1.data(), -2.f, 2.f);
    memcpy(xref.data(), x.data(), sizeof(float) * d4);
    std::string act_gate = "sigmoid", act_cand = "tanh", act_cell = "tanh";
340
    const jit::lstm_attr_t attr(d, act_gate, act_cand, act_cell, false);
T
tensor-tang 已提交
341 342
    const auto& ker =
        jit::KernelPool::Instance()
343 344
            .template Get<jit::LSTMKernel<float>, const jit::lstm_attr_t&>(
                attr);
T
tensor-tang 已提交
345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363
    // below kernels are used to compute refer
    const auto& vsigmoid_3d =
        jit::KernelPool::Instance().template Get<jit::VSigmoidKernel<float>>(
            d3);
    const auto& vtanh_d =
        jit::KernelPool::Instance().template Get<jit::VTanhKernel<float>>(d);
    const auto& vmul_d =
        jit::KernelPool::Instance().template Get<jit::VMulKernel<float>>(d);
    const auto& vadd_d =
        jit::KernelPool::Instance().template Get<jit::VAddKernel<float>>(d);

    float* x_data = x.data();
    float* xref_data = xref.data();
    const float* ct_1_data = ct_1.data();
    float* ct_tgt_data = ct_tgt.data();
    float* ht_tgt_data = ht_tgt.data();
    float* ct_ref_data = ct_ref.data();
    float* ht_ref_data = ht_ref.data();
    // compute once to check correctness
364 365 366 367 368 369 370
    jit::lstm_t step;
    step.gates = xref_data;
    step.ct_1 = ct_1_data;
    step.ct = ct_ref_data;
    step.ht = ht_ref_data;
    refer::LSTMCtHt<float>(&step, &attr);

371 372 373 374
    step.gates = x_data;
    step.ct = ct_tgt_data;
    step.ht = ht_tgt_data;
    ker->ComputeCtHt(&step, &attr);
T
tensor-tang 已提交
375 376 377 378 379 380 381 382 383 384 385 386 387
    for (int i = 0; i < d; ++i) {
      EXPECT_NEAR(ct_tgt_data[i], ct_ref_data[i], 1e-3);
      EXPECT_NEAR(ht_tgt_data[i], ht_ref_data[i], 1e-3);
    }

    auto tmkls = GetCurrentUS();
    for (int i = 0; i < repeat; ++i) {
      lstm_ctht_better(vsigmoid_3d, vtanh_d, vmul_d, vadd_d, d, xref_data,
                       ct_1_data, ct_ref_data, ht_ref_data);
    }
    auto tmkle = GetCurrentUS();
    auto trefs = GetCurrentUS();
    for (int i = 0; i < repeat; ++i) {
388
      refer::LSTMCtHt<float>(&step, &attr);
T
tensor-tang 已提交
389 390 391 392
    }
    auto trefe = GetCurrentUS();
    auto ttgts = GetCurrentUS();
    for (int i = 0; i < repeat; ++i) {
393
      ker->ComputeCtHt(&step, &attr);
T
tensor-tang 已提交
394 395
    }
    auto ttgte = GetCurrentUS();
M
minqiyang 已提交
396 397 398
    VLOG(3) << "Vec size " << d << ": refer takes: " << (trefe - trefs) / repeat
            << " us, better(jit) takes: " << (tmkle - tmkls) / repeat
            << " us, tgt takes: " << (ttgte - ttgts) / repeat << " us";
T
tensor-tang 已提交
399 400 401
  }
}

T
tensor-tang 已提交
402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426
#if defined __AVX__ || defined __AVX2__
void vscal_intri8(const int n, const float a, const float* x, float* y) {
  __m256 tmp;
  __m256 scalar = _mm256_set1_ps(a);
  tmp = _mm256_loadu_ps(x);
  tmp = _mm256_mul_ps(tmp, scalar);
  _mm256_storeu_ps(y, tmp);
}
void vscal_inp_intri8(const int n, const float a, float* x) {
  __m256 tmp;
  __m256 scalar = _mm256_set1_ps(a);
  tmp = _mm256_loadu_ps(x);
  tmp = _mm256_mul_ps(tmp, scalar);
  _mm256_storeu_ps(x, tmp);
}
#endif

#ifdef PADDLE_WITH_MKLML
void vscal_inp_mkl(const int n, const float a, float* x) {
  paddle::platform::dynload::cblas_sscal(n, a, x, 1);
}
#endif

TEST(JitKernel, vscal) {
  namespace jit = paddle::operators::math::jitkernel;
427
  namespace refer = paddle::operators::math::jitkernel::refer;
T
tensor-tang 已提交
428 429 430 431 432 433 434 435 436 437 438 439 440 441
  for (int d : {7, 8, 15, 16, 30, 256, 512}) {
    std::vector<float> x(d), y(d);
    std::vector<float> zref(d), ztgt(d);
    RandomVec<float>(d, x.data());
    std::memcpy(y.data(), x.data(), sizeof(float) * d);
    float a = 2.f;
    const auto& ker =
        jit::KernelPool::Instance().template Get<jit::VScalKernel<float>>(d);
    const float* x_data = x.data();
    float* y_data = y.data();
    float* ztgt_data = ztgt.data();
    float* zref_data = zref.data();
    auto trefs = GetCurrentUS();
    for (int i = 0; i < repeat; ++i) {
442
      refer::VScal<float>(&a, x_data, zref_data, d);
T
tensor-tang 已提交
443 444 445 446
    }
    auto trefe = GetCurrentUS();
    auto trefs1 = GetCurrentUS();
    for (int i = 0; i < repeat; ++i) {
447
      refer::VScal<float>(&a, y_data, y_data, d);
T
tensor-tang 已提交
448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470
    }
    auto trefe1 = GetCurrentUS();

#ifdef PADDLE_WITH_MKLML
    auto tmkls = GetCurrentUS();
    for (int i = 0; i < repeat; ++i) {
      vscal_inp_mkl(d, a, y_data);
    }
    auto tmkle = GetCurrentUS();
#endif

#if defined __AVX__ || defined __AVX2__
    if (d == 8) {
      auto si0 = GetCurrentUS();
      for (int i = 0; i < repeat; ++i) {
        vscal_intri8(d, a, x_data, zref_data);
      }
      auto si1 = GetCurrentUS();
      auto si2 = GetCurrentUS();
      for (int i = 0; i < repeat; ++i) {
        vscal_inp_intri8(d, a, y_data);
      }
      auto si3 = GetCurrentUS();
M
minqiyang 已提交
471
      VLOG(3) << "Vec size 8 intr takes: " << (si1 - si0) / repeat
M
minqiyang 已提交
472
              << " us, inplace: " << (si3 - si2) / repeat << " us";
T
tensor-tang 已提交
473 474 475 476 477
    }
#endif

    auto ttgts = GetCurrentUS();
    for (int i = 0; i < repeat; ++i) {
T
tensor-tang 已提交
478
      ker->Compute(&a, x_data, ztgt_data, d);
T
tensor-tang 已提交
479 480 481 482
    }
    auto ttgte = GetCurrentUS();
    auto ttgts1 = GetCurrentUS();
    for (int i = 0; i < repeat; ++i) {
T
tensor-tang 已提交
483
      ker->Compute(&a, y_data, y_data, d);
T
tensor-tang 已提交
484 485
    }
    auto ttgte1 = GetCurrentUS();
M
minqiyang 已提交
486 487
    VLOG(3) << "Vec size " << d << ": refer takes: " << (trefe - trefs) / repeat
            << " us, inplace takes: " << (trefe1 - trefs1) / repeat
T
tensor-tang 已提交
488
#ifdef PADDLE_WITH_MKLML
M
minqiyang 已提交
489
            << " us, mkl inplace takes: " << (tmkle - tmkls) / repeat << " us, "
T
tensor-tang 已提交
490
#else
M
minqiyang 已提交
491
            << " us, "
T
tensor-tang 已提交
492
#endif
M
minqiyang 已提交
493 494
            << "tgt takes: " << (ttgte - ttgts) / repeat
            << "us, tgt inplace takes: " << (ttgte1 - ttgts1) / repeat << " us";
T
tensor-tang 已提交
495 496 497 498 499
    for (int i = 0; i < d; ++i) {
      EXPECT_NEAR(ztgt_data[i], zref_data[i], 1e-3);
    }
  }
}
T
tensor-tang 已提交
500

T
tensor-tang 已提交
501
#if defined __AVX__ || defined __AVX2__
T
tensor-tang 已提交
502
void vmul_intri8(const int n, const float* x, const float* y, float* z) {
T
tensor-tang 已提交
503 504 505 506 507 508 509
  __m256 tmpx, tmpy;
  tmpx = _mm256_loadu_ps(x);
  tmpy = _mm256_loadu_ps(y);
  tmpx = _mm256_mul_ps(tmpx, tmpy);
  _mm256_storeu_ps(z, tmpx);
}
#endif
T
tensor-tang 已提交
510

T
tensor-tang 已提交
511 512 513
#ifdef PADDLE_WITH_MKLML
void vmul_mkl(const int n, const float* x, const float* y, float* z) {
  paddle::platform::dynload::vsMul(n, x, y, z);
T
tensor-tang 已提交
514
}
T
tensor-tang 已提交
515
#endif
T
tensor-tang 已提交
516

T
tensor-tang 已提交
517 518
TEST(JitKernel, vmul) {
  namespace jit = paddle::operators::math::jitkernel;
519
  namespace refer = paddle::operators::math::jitkernel::refer;
520
  for (int d : {7, 8, 15, 16, 20, 30, 256, 512, 1000, 1024}) {
T
tensor-tang 已提交
521 522 523 524 525 526 527 528 529 530
    std::vector<float> x(d), y(d);
    std::vector<float> zref(d), ztgt(d);
    RandomVec<float>(d, x.data());
    RandomVec<float>(d, y.data());
    const auto& ker =
        jit::KernelPool::Instance().template Get<jit::VMulKernel<float>>(d);
    const float* x_data = x.data();
    const float* y_data = y.data();
    float* ztgt_data = ztgt.data();
    float* zref_data = zref.data();
T
tensor-tang 已提交
531
    auto trefs = GetCurrentUS();
T
tensor-tang 已提交
532
    for (int i = 0; i < repeat; ++i) {
533
      refer::VMul<float>(x_data, y_data, zref_data, d);
T
tensor-tang 已提交
534
    }
T
tensor-tang 已提交
535
    auto trefe = GetCurrentUS();
T
tensor-tang 已提交
536

T
tensor-tang 已提交
537 538
#ifdef PADDLE_WITH_MKLML
    auto tmkls = GetCurrentUS();
T
tensor-tang 已提交
539
    for (int i = 0; i < repeat; ++i) {
T
tensor-tang 已提交
540
      vmul_mkl(d, x_data, y_data, zref_data);
T
tensor-tang 已提交
541
    }
T
tensor-tang 已提交
542 543
    auto tmkle = GetCurrentUS();
#endif
T
tensor-tang 已提交
544

T
tensor-tang 已提交
545 546 547 548
#if defined __AVX__ || defined __AVX2__
    if (d == 8) {
      auto si0 = GetCurrentUS();
      for (int i = 0; i < repeat; ++i) {
T
tensor-tang 已提交
549
        vmul_intri8(d, x_data, y_data, zref_data);
T
tensor-tang 已提交
550 551
      }
      auto si1 = GetCurrentUS();
M
minqiyang 已提交
552
      VLOG(3) << "Vec size 8 intr takes: " << (si1 - si0) / repeat;
T
tensor-tang 已提交
553 554 555 556 557
    }
#endif

    auto ttgts = GetCurrentUS();
    for (int i = 0; i < repeat; ++i) {
T
tensor-tang 已提交
558
      ker->Compute(x_data, y_data, ztgt_data, d);
T
tensor-tang 已提交
559 560 561
    }
    auto ttgte = GetCurrentUS();

M
minqiyang 已提交
562
    VLOG(3) << "Vec size " << d << ": refer takes: " << (trefe - trefs) / repeat
T
tensor-tang 已提交
563
#ifdef PADDLE_WITH_MKLML
M
minqiyang 已提交
564
            << " us, mkl takes: " << (tmkle - tmkls) / repeat << " us, "
T
tensor-tang 已提交
565
#else
M
minqiyang 已提交
566
            << " us, "
T
tensor-tang 已提交
567
#endif
M
minqiyang 已提交
568
            << "tgt takes: " << (ttgte - ttgts) / repeat << " us";
T
tensor-tang 已提交
569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592
    for (int i = 0; i < d; ++i) {
      EXPECT_NEAR(ztgt_data[i], zref_data[i], 1e-3);
    }
  }
}

#if defined __AVX__ || defined __AVX2__
void vadd_intri8(const int n, const float* x, const float* y, float* z) {
  __m256 tmpx, tmpy;
  tmpx = _mm256_loadu_ps(x);
  tmpy = _mm256_loadu_ps(y);
  tmpx = _mm256_add_ps(tmpx, tmpy);
  _mm256_storeu_ps(z, tmpx);
}
#endif

#ifdef PADDLE_WITH_MKLML
void vadd_mkl(const int n, const float* x, const float* y, float* z) {
  paddle::platform::dynload::vsAdd(n, x, y, z);
}
#endif

TEST(JitKernel, vadd) {
  namespace jit = paddle::operators::math::jitkernel;
593
  namespace refer = paddle::operators::math::jitkernel::refer;
T
tensor-tang 已提交
594 595 596 597 598 599 600 601 602 603 604 605 606
  for (int d : {7, 8, 15, 16, 30, 256, 512}) {
    std::vector<float> x(d), y(d);
    std::vector<float> zref(d), ztgt(d);
    RandomVec<float>(d, x.data());
    RandomVec<float>(d, y.data());
    const auto& ker =
        jit::KernelPool::Instance().template Get<jit::VAddKernel<float>>(d);
    const float* x_data = x.data();
    const float* y_data = y.data();
    float* ztgt_data = ztgt.data();
    float* zref_data = zref.data();
    auto trefs = GetCurrentUS();
    for (int i = 0; i < repeat; ++i) {
607
      refer::VAdd<float>(x_data, y_data, zref_data, d);
T
tensor-tang 已提交
608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625
    }
    auto trefe = GetCurrentUS();

#ifdef PADDLE_WITH_MKLML
    auto tmkls = GetCurrentUS();
    for (int i = 0; i < repeat; ++i) {
      vadd_mkl(d, x_data, y_data, zref_data);
    }
    auto tmkle = GetCurrentUS();
#endif

#if defined __AVX__ || defined __AVX2__
    if (d == 8) {
      auto si0 = GetCurrentUS();
      for (int i = 0; i < repeat; ++i) {
        vadd_intri8(d, x_data, y_data, zref_data);
      }
      auto si1 = GetCurrentUS();
M
minqiyang 已提交
626
      VLOG(3) << "Vec size 8 intr takes: " << (si1 - si0) / repeat;
T
tensor-tang 已提交
627 628 629
    }
#endif

T
tensor-tang 已提交
630 631
    auto ttgts = GetCurrentUS();
    for (int i = 0; i < repeat; ++i) {
T
tensor-tang 已提交
632
      ker->Compute(x_data, y_data, ztgt_data, d);
T
tensor-tang 已提交
633 634 635
    }
    auto ttgte = GetCurrentUS();

M
minqiyang 已提交
636
    VLOG(3) << "Vec size " << d << ": refer takes: " << (trefe - trefs) / repeat
T
tensor-tang 已提交
637
#ifdef PADDLE_WITH_MKLML
M
minqiyang 已提交
638
            << " us, mkl takes: " << (tmkle - tmkls) / repeat << " us, "
T
tensor-tang 已提交
639
#else
M
minqiyang 已提交
640
            << " us, "
T
tensor-tang 已提交
641
#endif
M
minqiyang 已提交
642
            << "tgt takes: " << (ttgte - ttgts) / repeat << " us";
T
tensor-tang 已提交
643 644 645 646 647 648
    for (int i = 0; i < d; ++i) {
      EXPECT_NEAR(ztgt_data[i], zref_data[i], 1e-3);
    }
  }
}

T
tensor-tang 已提交
649 650 651 652 653
void vaddrelu_better(
    const std::shared_ptr<
        const paddle::operators::math::jitkernel::VAddKernel<float>>& vadd,
    const std::shared_ptr<
        const paddle::operators::math::jitkernel::VReluKernel<float>>& vrelu,
T
tensor-tang 已提交
654 655
    const float* x, const float* y, float* z, int d) {
  vadd->Compute(x, y, z, d);
T
tensor-tang 已提交
656
  vrelu->Compute(z, z, d);
T
tensor-tang 已提交
657 658 659 660
}

TEST(JitKernel, vaddrelu) {
  namespace jit = paddle::operators::math::jitkernel;
661
  namespace refer = paddle::operators::math::jitkernel::refer;
T
tensor-tang 已提交
662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678
  for (int d : {7, 8, 15, 16, 30, 256, 512}) {
    std::vector<float> x(d), y(d);
    std::vector<float> zref(d), ztgt(d);
    RandomVec<float>(d, x.data());
    RandomVec<float>(d, y.data());
    const auto& ker =
        jit::KernelPool::Instance().template Get<jit::VAddReluKernel<float>>(d);
    const auto& vadd =
        jit::KernelPool::Instance().template Get<jit::VAddKernel<float>>(d);
    const auto& vrelu =
        jit::KernelPool::Instance().template Get<jit::VReluKernel<float>>(d);
    const float* x_data = x.data();
    const float* y_data = y.data();
    float* ztgt_data = ztgt.data();
    float* zref_data = zref.data();
    auto trefs = GetCurrentUS();
    for (int i = 0; i < repeat; ++i) {
679
      refer::VAddRelu<float>(x_data, y_data, zref_data, d);
T
tensor-tang 已提交
680 681 682 683
    }
    auto trefe = GetCurrentUS();
    auto tmkls = GetCurrentUS();
    for (int i = 0; i < repeat; ++i) {
T
tensor-tang 已提交
684
      vaddrelu_better(vadd, vrelu, x_data, y_data, zref_data, d);
T
tensor-tang 已提交
685 686 687 688
    }
    auto tmkle = GetCurrentUS();
    auto ttgts = GetCurrentUS();
    for (int i = 0; i < repeat; ++i) {
T
tensor-tang 已提交
689
      ker->Compute(x_data, y_data, ztgt_data, d);
T
tensor-tang 已提交
690 691
    }
    auto ttgte = GetCurrentUS();
M
minqiyang 已提交
692 693 694
    VLOG(3) << "Vec size " << d << ": refer takes: " << (trefe - trefs) / repeat
            << " us, better takes: " << (tmkle - tmkls) / repeat << " us, "
            << "tgt takes: " << (ttgte - ttgts) / repeat << " us";
T
tensor-tang 已提交
695 696 697 698 699 700
    for (int i = 0; i < d; ++i) {
      EXPECT_NEAR(ztgt_data[i], zref_data[i], 1e-3);
    }
  }
}

T
tensor-tang 已提交
701 702 703 704
TEST(JitKernel, pool) {
  namespace jit = paddle::operators::math::jitkernel;
  const int frame_size = 4;
  std::string act_gate = "sigmoid", act_cand = "tanh", act_cell = "tanh";
705 706
  jit::lstm_attr_t attr(frame_size, act_gate, act_cand, act_cell, false);

707 708
  // empty call it to avoid unknown flag 'use_pinned_memory' on Mac
  paddle::platform::jit::MayIUse(paddle::platform::jit::avx);
T
tensor-tang 已提交
709
  const auto& plstm1 =
T
tensor-tang 已提交
710
      jit::KernelPool::Instance()
711 712
          .template Get<jit::LSTMKernel<float>, const jit::lstm_attr_t&>(attr);

T
tensor-tang 已提交
713
  const auto& plstm2 =
T
tensor-tang 已提交
714
      jit::KernelPool::Instance()
715 716 717
          .template Get<jit::LSTMKernel<float>, const jit::lstm_attr_t&>(attr);
  EXPECT_EQ(plstm1, plstm2);

T
tensor-tang 已提交
718 719
  const auto& peephole =
      jit::KernelPool::Instance()
720 721
          .template Get<jit::LSTMKernel<float>, const jit::lstm_attr_t&>(
              jit::lstm_attr_t(frame_size, act_gate, act_cand, act_cell, true));
T
tensor-tang 已提交
722
  EXPECT_TRUE(plstm1 != peephole);
T
tensor-tang 已提交
723

T
tensor-tang 已提交
724
  const auto& pvmul_f =
T
tensor-tang 已提交
725
      jit::KernelPool::Instance().template Get<jit::VMulKernel<float>>(4);
T
tensor-tang 已提交
726 727
  EXPECT_TRUE(std::dynamic_pointer_cast<const jit::Kernel>(plstm2) !=
              std::dynamic_pointer_cast<const jit::Kernel>(pvmul_f));
T
tensor-tang 已提交
728

T
tensor-tang 已提交
729
  const auto& pvmul_d =
T
tensor-tang 已提交
730
      jit::KernelPool::Instance().template Get<jit::VMulKernel<double>>(4);
T
tensor-tang 已提交
731 732
  EXPECT_TRUE(std::dynamic_pointer_cast<const jit::Kernel>(pvmul_f) !=
              std::dynamic_pointer_cast<const jit::Kernel>(pvmul_d));
T
tensor-tang 已提交
733

T
tensor-tang 已提交
734
  const auto& pvmul_from_key = jit::KernelPool::Instance().Get("vmulfjit4");
T
tensor-tang 已提交
735 736 737 738 739
#if defined(__APPLE__) || defined(__OSX__) || defined(_WIN32)
  EXPECT_EQ(pvmul_from_key, nullptr);
#else
  EXPECT_EQ(pvmul_from_key, pvmul_f);
#endif
T
tensor-tang 已提交
740
  const auto& pvmul_from_key2 = jit::KernelPool::Instance().Get("vmulfjit");
T
tensor-tang 已提交
741
  EXPECT_TRUE(pvmul_from_key2 == nullptr);
T
tensor-tang 已提交
742
}