BaseMatrix.cu 57.0 KB
Newer Older
1
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
Z
zhangjinchao01 已提交
2 3 4 5 6 7 8 9 10 11 12 13 14 15

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */

#include <paddle/utils/Logging.h>
L
liaogang 已提交
16 17
#include <string.h>
#include <cmath>
Z
zhangjinchao01 已提交
18 19
#include "BaseMatrix.h"
#include "MathFunctions.h"
L
liaogang 已提交
20 21 22 23
#include "SIMDFunctions.h"
#include "hl_matrix_apply.cuh"
#include "hl_matrix_base.cuh"
#include "hl_matrix_ops.cuh"
Z
zhangjinchao01 已提交
24 25 26 27 28

namespace paddle {

const char* SPARSE_SUPPORT_ERROR = "Sparse Matrix/Vector is not supported.";

L
liaogang 已提交
29
template <class T>
Z
zhangjinchao01 已提交
30 31 32 33 34 35 36
template <class Op>
int BaseMatrixT<T>::applyUnary(Op op) {
  MatrixOffset offset(0, 0);
  applyUnary(op, height_, width_, offset);
  return 0;
}

L
liaogang 已提交
37
template <class T>
Z
zhangjinchao01 已提交
38
template <class Op>
L
liaogang 已提交
39 40 41
int BaseMatrixT<T>::applyUnary(Op op,
                               int numRows,
                               int numCols,
Z
zhangjinchao01 已提交
42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60
                               MatrixOffset& offset) {
  CHECK(!this->isSparse()) << SPARSE_SUPPORT_ERROR;
  int dimM = numRows;
  int dimN = numCols;
  int lda = stride_;

  T* A = data_;
  CAL_MATRIX_START_ADDRESS(A, height_, width_, lda, offset.aCol_, offset.aRow_);

  CHECK_LE(dimM + offset.aRow_, this->height_);
  CHECK_LE(dimN + offset.aCol_, this->width_);
  if (true == useGpu_) {
    hl_gpu_apply_unary_op(op, A, dimM, dimN, lda);
  } else {
    hl_cpu_apply_unary_op(op, A, dimM, dimN, lda);
  }
  return 0;
}

L
liaogang 已提交
61
template <class T>
Z
zhangjinchao01 已提交
62 63 64 65 66 67 68 69 70 71
template <class Op>
int BaseMatrixT<T>::applyBinary(Op op, BaseMatrixT& b) {
  CHECK(height_ == b.height_ && width_ == b.width_)
      << "Matrix dimensions are not equal";

  MatrixOffset offset(0, 0, 0, 0);
  applyBinary(op, b, height_, width_, offset);
  return 0;
}

L
liaogang 已提交
72
template <class T>
Z
zhangjinchao01 已提交
73
template <class Op>
L
liaogang 已提交
74 75
int BaseMatrixT<T>::applyBinary(
    Op op, BaseMatrixT& b, int numRows, int numCols, MatrixOffset& offset) {
Z
zhangjinchao01 已提交
76 77 78 79
  applyBinary(op, b, numRows, numCols, offset, false_type(), false_type());
  return 0;
}

L
liaogang 已提交
80
template <class T>
Z
zhangjinchao01 已提交
81
template <class Op, class bAsRowVector, class bAsColVector>
L
liaogang 已提交
82 83 84 85 86 87 88
int BaseMatrixT<T>::applyBinary(Op op,
                                BaseMatrixT& b,
                                int numRows,
                                int numCols,
                                MatrixOffset& offset,
                                bAsRowVector,
                                bAsColVector) {
Z
zhangjinchao01 已提交
89 90 91 92 93 94 95 96 97 98 99 100
  CHECK(!this->isSparse()) << SPARSE_SUPPORT_ERROR;
  CHECK(!b.isSparse()) << SPARSE_SUPPORT_ERROR;
  CHECK(useGpu_ == b.useGpu_) << "Matrix type mismatch";

  int dimM = numRows;
  int dimN = numCols;
  int lda = stride_;
  int ldb = b.stride_;

  T* A = data_;
  T* B = b.data_;
  CAL_MATRIX_START_ADDRESS(A, height_, width_, lda, offset.aCol_, offset.aRow_);
L
liaogang 已提交
101 102
  CAL_MATRIX_START_ADDRESS(
      B, b.height_, b.width_, ldb, offset.bCol_, offset.bRow_);
Z
zhangjinchao01 已提交
103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124
  CHECK_LE(dimM + offset.aRow_, this->height_);
  CHECK_LE(dimN + offset.aCol_, this->width_);
  if (!bAsRowVector::value && !bAsColVector::value) {
    CHECK_LE(dimM + offset.bRow_, b.height_);
    CHECK_LE(dimN + offset.bCol_, b.width_);
  } else if (bAsRowVector::value && !bAsColVector::value) {
    CHECK_LE(dimN + offset.bCol_, b.width_);
  } else if (!bAsRowVector::value && bAsColVector::value) {
    CHECK_LE(dimM + offset.bRow_, b.height_);
  } else {
  }
  if (true == useGpu_) {
    hl_gpu_apply_binary_op<T, Op, bAsRowVector::value, bAsColVector::value>(
        op, A, B, dimM, dimN, lda, ldb);
  } else {
    hl_cpu_apply_binary_op<T, Op, bAsRowVector::value, bAsColVector::value>(
        op, A, B, dimM, dimN, lda, ldb);
  }

  return 0;
}

L
liaogang 已提交
125
template <class T>
Z
zhangjinchao01 已提交
126 127 128 129 130 131 132 133 134 135 136 137 138
template <class Op>
int BaseMatrixT<T>::applyTernary(Op op, BaseMatrixT& b, BaseMatrixT& c) {
  CHECK_EQ(height_, b.height_);
  CHECK_EQ(width_, b.width_);
  CHECK_EQ(height_, c.height_);
  CHECK_EQ(width_, c.width_);

  MatrixOffset offset(0, 0, 0, 0, 0, 0);
  applyTernary(op, b, c, height_, width_, offset);

  return 0;
}

L
liaogang 已提交
139
template <class T>
Z
zhangjinchao01 已提交
140
template <class Op>
L
liaogang 已提交
141 142 143 144 145
int BaseMatrixT<T>::applyTernary(Op op,
                                 BaseMatrixT& b,
                                 BaseMatrixT& c,
                                 int numRows,
                                 int numCols,
Z
zhangjinchao01 已提交
146 147 148 149 150 151
                                 MatrixOffset& offset) {
  applyTernary(op, b, c, numRows, numCols, offset, false_type(), false_type());

  return 0;
}

L
liaogang 已提交
152
template <class T>
Z
zhangjinchao01 已提交
153
template <class Op, class cAsRowVector, class cAsColVector>
L
liaogang 已提交
154 155 156 157 158 159 160 161
int BaseMatrixT<T>::applyTernary(Op op,
                                 BaseMatrixT& b,
                                 BaseMatrixT& c,
                                 int numRows,
                                 int numCols,
                                 MatrixOffset& offset,
                                 cAsRowVector,
                                 cAsColVector) {
Z
zhangjinchao01 已提交
162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177
  CHECK(!this->isSparse()) << SPARSE_SUPPORT_ERROR;
  CHECK(!b.isSparse()) << SPARSE_SUPPORT_ERROR;
  CHECK(!c.isSparse()) << SPARSE_SUPPORT_ERROR;
  CHECK_EQ(useGpu_, b.useGpu_);
  CHECK_EQ(useGpu_, c.useGpu_);

  int dimM = numRows;
  int dimN = numCols;
  int lda = stride_;
  int ldb = b.stride_;
  int ldc = c.stride_;

  T* A = data_;
  T* B = b.data_;
  T* C = c.data_;
  CAL_MATRIX_START_ADDRESS(A, height_, width_, lda, offset.aCol_, offset.aRow_);
L
liaogang 已提交
178 179 180 181
  CAL_MATRIX_START_ADDRESS(
      B, b.height_, b.width_, ldb, offset.bCol_, offset.bRow_);
  CAL_MATRIX_START_ADDRESS(
      C, c.height_, c.width_, ldc, offset.cCol_, offset.cRow_);
Z
zhangjinchao01 已提交
182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197

  CHECK_LE(dimM + offset.aRow_, this->height_);
  CHECK_LE(dimN + offset.aCol_, this->width_);
  CHECK_LE(dimM + offset.bRow_, b.height_);
  CHECK_LE(dimN + offset.bCol_, b.width_);
  if (!cAsRowVector::value && !cAsColVector::value) {
    CHECK_LE(dimM + offset.cRow_, c.height_);
    CHECK_LE(dimN + offset.cCol_, c.width_);
  } else if (cAsRowVector::value && !cAsColVector::value) {
    CHECK_LE(dimN + offset.cCol_, c.width_);
  } else if (!cAsRowVector::value && cAsColVector::value) {
    CHECK_LE(dimM + offset.cRow_, c.height_);
  } else {
  }

  if (true == useGpu_) {
L
liaogang 已提交
198
    hl_gpu_apply_ternary_op<T, Op, cAsRowVector::value, cAsColVector::value>(
Z
zhangjinchao01 已提交
199 200
        op, A, B, C, dimM, dimN, lda, ldb, ldc);
  } else {
L
liaogang 已提交
201
    hl_cpu_apply_ternary_op<T, Op, cAsRowVector::value, cAsColVector::value>(
Z
zhangjinchao01 已提交
202 203 204 205 206 207
        op, A, B, C, dimM, dimN, lda, ldb, ldc);
  }

  return 0;
}

L
liaogang 已提交
208
template <class T>
Z
zhangjinchao01 已提交
209
template <class Op>
L
liaogang 已提交
210 211 212
int BaseMatrixT<T>::applyQuaternary(Op op,
                                    BaseMatrixT& b,
                                    BaseMatrixT& c,
Z
zhangjinchao01 已提交
213 214 215 216 217 218 219 220 221 222 223 224 225 226
                                    BaseMatrixT& d) {
  CHECK_EQ(height_, b.height_);
  CHECK_EQ(width_, b.width_);
  CHECK_EQ(height_, c.height_);
  CHECK_EQ(width_, c.width_);
  CHECK_EQ(height_, d.height_);
  CHECK_EQ(width_, d.width_);

  MatrixOffset offset(0, 0, 0, 0, 0, 0, 0, 0);
  applyQuaternary(op, b, c, d, height_, width_, offset);

  return 0;
}

L
liaogang 已提交
227
template <class T>
Z
zhangjinchao01 已提交
228
template <class Op>
L
liaogang 已提交
229 230 231 232 233 234
int BaseMatrixT<T>::applyQuaternary(Op op,
                                    BaseMatrixT& b,
                                    BaseMatrixT& c,
                                    BaseMatrixT& d,
                                    int numRows,
                                    int numCols,
Z
zhangjinchao01 已提交
235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255
                                    MatrixOffset& offset) {
  CHECK(!this->isSparse()) << SPARSE_SUPPORT_ERROR;
  CHECK(!b.isSparse()) << SPARSE_SUPPORT_ERROR;
  CHECK(!c.isSparse()) << SPARSE_SUPPORT_ERROR;
  CHECK(!d.isSparse()) << SPARSE_SUPPORT_ERROR;
  CHECK_EQ(useGpu_, b.useGpu_);
  CHECK_EQ(useGpu_, c.useGpu_);
  CHECK_EQ(useGpu_, d.useGpu_);

  int dimM = numRows;
  int dimN = numCols;
  int lda = stride_;
  int ldb = b.stride_;
  int ldc = c.stride_;
  int ldd = d.stride_;

  T* A = data_;
  T* B = b.data_;
  T* C = c.data_;
  T* D = d.data_;
  CAL_MATRIX_START_ADDRESS(A, height_, width_, lda, offset.aCol_, offset.aRow_);
L
liaogang 已提交
256 257 258 259 260 261
  CAL_MATRIX_START_ADDRESS(
      B, b.height_, b.width_, ldb, offset.bCol_, offset.bRow_);
  CAL_MATRIX_START_ADDRESS(
      C, c.height_, c.width_, ldc, offset.cCol_, offset.cRow_);
  CAL_MATRIX_START_ADDRESS(
      D, d.height_, d.width_, ldd, offset.dCol_, offset.dRow_);
Z
zhangjinchao01 已提交
262 263 264 265 266 267 268 269 270 271

  CHECK_LE(dimM + offset.aRow_, this->height_);
  CHECK_LE(dimN + offset.aCol_, this->width_);
  CHECK_LE(dimM + offset.bRow_, b.height_);
  CHECK_LE(dimN + offset.bCol_, b.width_);
  CHECK_LE(dimM + offset.cRow_, c.height_);
  CHECK_LE(dimN + offset.cCol_, c.width_);
  CHECK_LE(dimM + offset.dRow_, d.height_);
  CHECK_LE(dimN + offset.dCol_, d.width_);
  if (true == useGpu_) {
L
liaogang 已提交
272
    hl_gpu_apply_quaternary_op(op, A, B, C, D, dimM, dimN, lda, ldb, ldc, ldd);
Z
zhangjinchao01 已提交
273
  } else {
L
liaogang 已提交
274
    hl_cpu_apply_quaternary_op(op, A, B, C, D, dimM, dimN, lda, ldb, ldc, ldd);
Z
zhangjinchao01 已提交
275 276 277 278 279
  }

  return 0;
}

L
liaogang 已提交
280 281 282 283 284
template <class T>
template <class Agg,
          class Op,
          class Saver,
          class aAsRowVector,
Z
zhangjinchao01 已提交
285
          class aAsColVector>
L
liaogang 已提交
286 287 288 289 290 291 292 293 294
int BaseMatrixT<T>::aggregate(Agg agg,
                              Op op,
                              Saver sv,
                              BaseMatrixT& b,
                              int numRows,
                              int numCols,
                              MatrixOffset& offset,
                              aAsRowVector,
                              aAsColVector) {
Z
zhangjinchao01 已提交
295 296 297 298 299 300 301
  CHECK_EQ(useGpu_, b.useGpu_);

  int ld = stride_;
  int ldb = b.stride_;

  T* dst = data_;
  T* B = b.data_;
L
liaogang 已提交
302 303 304 305
  CAL_MATRIX_START_ADDRESS(
      dst, height_, width_, ld, offset.aCol_, offset.aRow_);
  CAL_MATRIX_START_ADDRESS(
      B, b.height_, b.width_, ldb, offset.bCol_, offset.bRow_);
Z
zhangjinchao01 已提交
306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325

  if (aAsRowVector::value && !aAsColVector::value) {
    if (useGpu_) {
      hl_gpu_matrix_column_op(agg, op, sv, numRows, numCols, dst, B, ldb);
    } else {
      hl_cpu_matrix_column_op(agg, op, sv, numRows, numCols, dst, B, ldb);
    }
  } else if (!aAsRowVector::value && aAsColVector::value) {
    if (useGpu_) {
      hl_gpu_matrix_row_op(agg, op, sv, numRows, numCols, dst, ld, B, ldb);
    } else {
      hl_cpu_matrix_row_op(agg, op, sv, numRows, numCols, dst, ld, B, ldb);
    }
  } else {
    LOG(FATAL) << "not supported";
  }

  return 0;
}

L
liaogang 已提交
326 327 328 329 330
template <class T>
template <class Agg,
          class Op,
          class Saver,
          class aAsRowVector,
Z
zhangjinchao01 已提交
331
          class aAsColVector>
L
liaogang 已提交
332 333 334 335 336 337 338 339 340
int BaseMatrixT<T>::aggregate(Agg agg,
                              Op op,
                              Saver sv,
                              BaseMatrixT& b,
                              BaseMatrixT& c,
                              int numRows,
                              int numCols,
                              MatrixOffset& offset,
                              aAsRowVector,
Z
zhangjinchao01 已提交
341 342 343 344 345 346 347 348 349 350 351
                              aAsColVector) {
  CHECK_EQ(useGpu_, b.useGpu_);
  CHECK_EQ(useGpu_, c.useGpu_);

  int ld = stride_;
  int ldb = b.stride_;
  int ldc = c.stride_;

  T* dst = data_;
  T* B = b.data_;
  T* C = c.data_;
L
liaogang 已提交
352 353 354 355 356 357
  CAL_MATRIX_START_ADDRESS(
      dst, height_, width_, ld, offset.aCol_, offset.aRow_);
  CAL_MATRIX_START_ADDRESS(
      B, b.height_, b.width_, ldb, offset.bCol_, offset.bRow_);
  CAL_MATRIX_START_ADDRESS(
      C, c.height_, c.width_, ldc, offset.cCol_, offset.cRow_);
Z
zhangjinchao01 已提交
358 359 360

  if (aAsRowVector::value && !aAsColVector::value) {
    if (useGpu_) {
L
liaogang 已提交
361 362
      hl_gpu_matrix_column_op(
          agg, op, sv, numRows, numCols, dst, B, ldb, C, ldc);
Z
zhangjinchao01 已提交
363
    } else {
L
liaogang 已提交
364 365
      hl_cpu_matrix_column_op(
          agg, op, sv, numRows, numCols, dst, B, ldb, C, ldc);
Z
zhangjinchao01 已提交
366 367 368
    }
  } else if (!aAsRowVector::value && aAsColVector::value) {
    if (useGpu_) {
L
liaogang 已提交
369 370
      hl_gpu_matrix_row_op(
          agg, op, sv, numRows, numCols, dst, ld, B, ldb, C, ldc);
Z
zhangjinchao01 已提交
371
    } else {
L
liaogang 已提交
372 373
      hl_cpu_matrix_row_op(
          agg, op, sv, numRows, numCols, dst, ld, B, ldb, C, ldc);
Z
zhangjinchao01 已提交
374 375 376 377 378 379 380 381 382 383 384 385 386 387
    }
  } else {
    LOG(FATAL) << "not supported";
  }

  return 0;
}

/**
 * @brief   unary operator.
 *
 */

DEFINE_MATRIX_UNARY_OP(Neg, a = -a);
L
liaogang 已提交
388 389 390 391
template <class T>
void BaseMatrixT<T>::neg() {
  applyUnary(unary::Neg<T>());
}
Z
zhangjinchao01 已提交
392 393

DEFINE_MATRIX_UNARY_OP(Exp, a = exp(a));
L
liaogang 已提交
394 395 396 397
template <>
void BaseMatrixT<real>::exp2() {
  applyUnary(unary::Exp<real>());
}
Z
zhangjinchao01 已提交
398 399

DEFINE_MATRIX_UNARY_OP(Log, a = log(a));
L
liaogang 已提交
400
template <>
H
hedaoyuan 已提交
401
void BaseMatrixT<real>::log2() {
Z
zhangjinchao01 已提交
402 403 404 405 406 407 408 409
  if (useGpu_) {
    applyUnary(unary::Log<real>());
  } else {
    vLog(height_ * width_, data_, data_);
  }
}

DEFINE_MATRIX_UNARY_OP(Sqrt, a = sqrt(a));
L
liaogang 已提交
410 411 412 413
template <>
void BaseMatrixT<real>::sqrt2() {
  applyUnary(unary::Sqrt<real>());
}
Z
zhangjinchao01 已提交
414 415

DEFINE_MATRIX_UNARY_OP(Square, a = a * a);
L
liaogang 已提交
416 417 418 419
template <class T>
void BaseMatrixT<T>::square2() {
  applyUnary(unary::Square<T>());
}
Z
zhangjinchao01 已提交
420 421

DEFINE_MATRIX_UNARY_OP(Reciprocal, a = 1.0f / a);
L
liaogang 已提交
422 423 424 425
template <class T>
void BaseMatrixT<T>::reciprocal2() {
  applyUnary(unary::Reciprocal<T>());
}
Z
zhangjinchao01 已提交
426 427

DEFINE_MATRIX_UNARY_OP(Abs, a = a > 0 ? a : -a);
L
liaogang 已提交
428 429 430 431
template <class T>
void BaseMatrixT<T>::abs2() {
  applyUnary(unary::Abs<T>());
}
Z
zhangjinchao01 已提交
432 433

DEFINE_MATRIX_UNARY_OP(Sign, a = (a > 0) - (a < 0));
L
liaogang 已提交
434 435 436 437
template <class T>
void BaseMatrixT<T>::sign2() {
  applyUnary(unary::Sign<T>());
}
Z
zhangjinchao01 已提交
438 439

DEFINE_MATRIX_UNARY_OP(Zero, a = 0);
L
liaogang 已提交
440 441 442 443
template <class T>
void BaseMatrixT<T>::zero() {
  applyUnary(unary::Zero<T>());
}
Z
zhangjinchao01 已提交
444

L
liaogang 已提交
445
template <class T>
Z
zhangjinchao01 已提交
446 447 448 449 450 451 452 453
void BaseMatrixT<T>::zeroAtOffset(int64_t columnOffset, int64_t numColumns) {
  int numRows = height_;
  int numCols = numColumns;
  MatrixOffset offset(columnOffset, 0);
  applyUnary(unary::Zero<T>(), numRows, numCols, offset);
}

DEFINE_MATRIX_UNARY_OP(One, a = 1);
L
liaogang 已提交
454 455 456 457
template <class T>
void BaseMatrixT<T>::one() {
  applyUnary(unary::One<T>());
}
Z
zhangjinchao01 已提交
458 459

DEFINE_MATRIX_UNARY_PARAMETER_OP(Pow, ONE_PARAMETER, a = pow(a, p));
L
liaogang 已提交
460
template <>
H
hedaoyuan 已提交
461
void BaseMatrixT<real>::pow2(real p) {
Z
zhangjinchao01 已提交
462 463 464 465 466 467 468 469
  if (useGpu_) {
    applyUnary(unary::Pow<real>(p));
  } else {
    vPow(height_ * width_, data_, p, data_);
  }
}

DEFINE_MATRIX_UNARY_PARAMETER_OP(SubScalar, ONE_PARAMETER, a -= p);
L
liaogang 已提交
470 471 472 473
template <class T>
void BaseMatrixT<T>::subScalar(T p) {
  applyUnary(unary::SubScalar<T>(p));
}
Z
zhangjinchao01 已提交
474 475

DEFINE_MATRIX_UNARY_PARAMETER_OP(MulScalar, ONE_PARAMETER, a *= p);
L
liaogang 已提交
476 477 478 479
template <class T>
void BaseMatrixT<T>::mulScalar(T p) {
  applyUnary(unary::MulScalar<T>(p));
}
Z
zhangjinchao01 已提交
480 481

DEFINE_MATRIX_UNARY_PARAMETER_OP(DivScalar, ONE_PARAMETER, a /= p);
L
liaogang 已提交
482 483 484 485
template <class T>
void BaseMatrixT<T>::divScalar(T p) {
  applyUnary(unary::DivScalar<T>(p));
}
Z
zhangjinchao01 已提交
486 487

DEFINE_MATRIX_UNARY_PARAMETER_OP(Assign, ONE_PARAMETER, a = p);
L
liaogang 已提交
488 489 490 491
template <class T>
void BaseMatrixT<T>::assign(T p) {
  applyUnary(unary::Assign<T>(p));
}
Z
zhangjinchao01 已提交
492 493

DEFINE_MATRIX_UNARY_PARAMETER_OP(Add, ONE_PARAMETER, a += p);
L
liaogang 已提交
494 495 496 497
template <class T>
void BaseMatrixT<T>::add(T p) {
  applyUnary(unary::Add<T>(p));
}
Z
zhangjinchao01 已提交
498 499

DEFINE_MATRIX_UNARY_PARAMETER_OP(Add2, TWO_PARAMETER, a = a * p1 + p2);
L
liaogang 已提交
500 501 502 503
template <class T>
void BaseMatrixT<T>::add(T p1, T p2) {
  applyUnary(unary::Add2<T>(p1, p2));
}
Z
zhangjinchao01 已提交
504

L
liaogang 已提交
505 506
DEFINE_MATRIX_UNARY_PARAMETER_OP(Clip,
                                 TWO_PARAMETER,
Z
zhangjinchao01 已提交
507
                                 a = a < p1 ? p1 : (a > p2 ? p2 : a));
L
liaogang 已提交
508 509 510 511
template <class T>
void BaseMatrixT<T>::clip(T p1, T p2) {
  applyUnary(unary::Clip<T>(p1, p2));
}
Z
zhangjinchao01 已提交
512

L
liaogang 已提交
513 514 515 516
DEFINE_MATRIX_BINARY_PARAMETER_OP(ClipDerivative,
                                  TWO_PARAMETER,
                                  a = b < p1 ? 0 : (b > p2 ? 0 : 1));
template <class T>
G
guosheng 已提交
517 518 519 520
void BaseMatrixT<T>::clipDerivative(BaseMatrixT& b, T p1, T p2) {
  applyBinary(binary::ClipDerivative<T>(p1, p2), b);
}

L
liaogang 已提交
521 522
DEFINE_MATRIX_UNARY_PARAMETER_OP(BiggerThanScalar,
                                 ONE_PARAMETER,
Z
zhangjinchao01 已提交
523
                                 a = a > p ? 1.0f : 0.0f);
L
liaogang 已提交
524
template <class T>
Z
zhangjinchao01 已提交
525 526 527 528
void BaseMatrixT<T>::biggerThanScalar(T p) {
  applyUnary(unary::BiggerThanScalar<T>(p));
}

L
liaogang 已提交
529 530
DEFINE_MATRIX_UNARY_PARAMETER_OP(DownClip, ONE_PARAMETER, a = a > p ? a : p);
template <class T>
Z
zhangjinchao01 已提交
531 532 533 534 535 536 537 538 539 540
void BaseMatrixT<T>::downClip(T p) {
  applyUnary(unary::DownClip<T>(p));
}

/**
 * @brief   binary operator.
 *
 */

DEFINE_MATRIX_BINARY_OP(Add, a += b);
L
liaogang 已提交
541
template <class T>
Z
zhangjinchao01 已提交
542 543 544 545
void BaseMatrixT<T>::add(BaseMatrixT& b) {
  applyBinary(binary::Add<T>(), b);
}

L
liaogang 已提交
546
template <>
Z
zhangjinchao01 已提交
547 548 549 550 551 552 553 554 555 556
void BaseMatrixT<real>::add(BaseMatrixT& b) {
  if (useGpu_) {
    applyBinary(binary::Add<real>(), b);
  } else {  // cpu branch
    CHECK_EQ(height_, b.height_);
    CHECK_EQ(width_, b.width_);
    vAdd(height_ * width_, data_, b.data_, data_);
  }
}

L
liaogang 已提交
557
template <class T>
Z
zhangjinchao01 已提交
558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575
void BaseMatrixT<T>::addAtOffset(BaseMatrixT& b, int64_t columnOffset) {
  if (columnOffset + b.width_ <= width_) {
    int numRows = height_;
    int numCols = b.width_;
    MatrixOffset offset(columnOffset, 0, 0, 0);
    applyBinary(binary::Add<T>(), b, numRows, numCols, offset);
  } else if (columnOffset + width_ <= b.width_) {
    int numRows = height_;
    int numCols = width_;
    MatrixOffset offset(0, 0, columnOffset, 0);
    applyBinary(binary::Add<T>(), b, numRows, numCols, offset);
  } else {
    LOG(FATAL) << "Wrong argument "
               << " a.width=" << width_ << " b.width=" << b.width_
               << " columnOffset=" << columnOffset;
  }
}

L
liaogang 已提交
576
template <class T>
Z
zhangjinchao01 已提交
577 578 579 580 581 582
void BaseMatrixT<T>::addP2P(BaseMatrixT& b) {
  T* A = data_;
  T* B = b.data_;
  int dimM = height_;
  int dimN = width_;

L
liaogang 已提交
583 584
  hl_gpu_apply_binary_op<T, binary::Add<T>, 0, 0>(
      binary::Add<T>(), A, B, dimM, dimN, dimN, dimN);
Z
zhangjinchao01 已提交
585 586
}

L
liaogang 已提交
587
template <class T>
Z
zhangjinchao01 已提交
588 589 590 591
void BaseMatrixT<T>::addColVector(BaseMatrixT& b) {
  MatrixOffset offset(0, 0, 0, 0);
  int numRows = height_;
  int numCols = width_;
L
liaogang 已提交
592 593 594 595 596 597
  applyBinary(binary::Add<T>(),
              b,
              numRows,
              numCols,
              offset,
              false_type(),
Z
zhangjinchao01 已提交
598 599 600
              true_type() /* bAsColVector */);
}

L
liaogang 已提交
601
template <class T>
Z
zhangjinchao01 已提交
602 603 604 605
void BaseMatrixT<T>::addRowVector(BaseMatrixT& b) {
  MatrixOffset offset(0, 0, 0, 0);
  int numRows = height_;
  int numCols = width_;
L
liaogang 已提交
606 607 608 609 610 611 612
  applyBinary(binary::Add<T>(),
              b,
              numRows,
              numCols,
              offset,
              true_type() /* bAsRowVector */,
              false_type());
Z
zhangjinchao01 已提交
613 614 615
}

DEFINE_MATRIX_BINARY_PARAMETER_OP(Add1, ONE_PARAMETER, a += b * p);
L
liaogang 已提交
616
template <class T>
Z
zhangjinchao01 已提交
617 618 619 620 621
void BaseMatrixT<T>::add(BaseMatrixT& b, T p) {
  applyBinary(binary::Add1<T>(p), b);
}

DEFINE_MATRIX_BINARY_PARAMETER_OP(Pow, ONE_PARAMETER, a = pow(b, p));
L
liaogang 已提交
622
template <>
H
hedaoyuan 已提交
623
void BaseMatrixT<real>::pow2(BaseMatrixT& b, real p) {
Z
zhangjinchao01 已提交
624 625 626 627 628 629 630 631
  if (useGpu_) {
    applyBinary(binary::Pow<real>(p), b);
  } else {
    vPow(height_ * width_, b.data_, p, data_);
  }
}

DEFINE_MATRIX_BINARY_PARAMETER_OP(Add2, TWO_PARAMETER, a = p1 * a + p2 * b);
L
liaogang 已提交
632
template <class T>
Z
zhangjinchao01 已提交
633 634 635 636
void BaseMatrixT<T>::add(BaseMatrixT& b, T p1, T p2) {
  applyBinary(binary::Add2<T>(p1, p2), b);
}

L
liaogang 已提交
637
template <class T>
Z
zhangjinchao01 已提交
638 639 640 641
void BaseMatrixT<T>::addBias(BaseMatrixT& b, T scale) {
  MatrixOffset offset(0, 0, 0, 0);
  int numRows = height_;
  int numCols = width_;
L
liaogang 已提交
642 643 644 645 646 647 648
  applyBinary(binary::Add1<T>(scale),
              b,
              numRows,
              numCols,
              offset,
              true_type() /* bAsRowVector */,
              false_type());
Z
zhangjinchao01 已提交
649 650 651
}

DEFINE_MATRIX_BINARY_OP(Sub, a -= b);
L
liaogang 已提交
652 653 654 655
template <class T>
void BaseMatrixT<T>::sub(BaseMatrixT& b) {
  applyBinary(binary::Sub<T>(), b);
}
Z
zhangjinchao01 已提交
656 657

DEFINE_MATRIX_BINARY_PARAMETER_OP(Sub1, ONE_PARAMETER, a -= b * p);
L
liaogang 已提交
658
template <class T>
Z
zhangjinchao01 已提交
659 660 661 662 663
void BaseMatrixT<T>::sub(BaseMatrixT& b, T p) {
  applyBinary(binary::Sub1<T>(p), b);
}

DEFINE_MATRIX_BINARY_OP(Relu, b = a > 0.0f ? a : 0.0f);
L
liaogang 已提交
664 665 666 667
template <class T>
void BaseMatrixT<T>::relu(BaseMatrixT& b) {
  applyBinary(binary::Relu<T>(), b);
}
Z
zhangjinchao01 已提交
668 669

DEFINE_MATRIX_BINARY_OP(ReluDerivative, a *= (b > 0.0f ? 1.0f : 0.0f));
L
liaogang 已提交
670
template <class T>
Z
zhangjinchao01 已提交
671 672 673 674 675 676 677 678 679
void BaseMatrixT<T>::reluDerivative(BaseMatrixT& b) {
  applyBinary(binary::ReluDerivative<T>(), b);
}

DEFINE_MATRIX_BINARY_OP(Softrelu, const T THRESHOLD = 40.0;
                        b = log(1.0 + exp((a > THRESHOLD)
                                              ? THRESHOLD
                                              : ((a < -THRESHOLD) ? (-THRESHOLD)
                                                                  : a))));
L
liaogang 已提交
680
template <>
Z
zhangjinchao01 已提交
681 682 683 684 685 686 687 688 689
void BaseMatrixT<real>::softrelu(BaseMatrixT& b) {
  applyBinary(binary::Softrelu<real>(), b);
}

DEFINE_MATRIX_BINARY_OP(
    SoftreluDerivative, const T THRESHOLD = 40.0;
    a *= (1.0 - exp(-1.0 * ((b > THRESHOLD)
                                ? THRESHOLD
                                : ((b < -THRESHOLD) ? (-THRESHOLD) : b)))));
L
liaogang 已提交
690
template <>
Z
zhangjinchao01 已提交
691 692 693 694 695 696
void BaseMatrixT<real>::softreluDerivative(BaseMatrixT& b) {
  applyBinary(binary::SoftreluDerivative<real>(), b);
}

DEFINE_MATRIX_BINARY_PARAMETER_OP(Brelu, TWO_PARAMETER, b = a > p1 ? a : p1;
                                  b = b < p2 ? b : p2);
L
liaogang 已提交
697
template <class T>
Z
zhangjinchao01 已提交
698
void BaseMatrixT<T>::brelu(BaseMatrixT& b) {
L
liaogang 已提交
699
  int p1 = 0, p2 = 24;  //! TODO(yuyang18): Make p1,p2 configuable.
Z
zhangjinchao01 已提交
700 701 702
  applyBinary(binary::Brelu<T>(p1, p2), b);
}

L
liaogang 已提交
703 704
DEFINE_MATRIX_BINARY_PARAMETER_OP(BreluDerivative,
                                  TWO_PARAMETER,
Z
zhangjinchao01 已提交
705
                                  a *= (b > p1 && b < p2) ? 1.0 : 0.0);
L
liaogang 已提交
706
template <class T>
Z
zhangjinchao01 已提交
707 708 709 710 711 712
void BaseMatrixT<T>::breluDerivative(BaseMatrixT& b) {
  int p1 = 0, p2 = 24;
  applyBinary(binary::BreluDerivative<T>(p1, p2), b);
}

DEFINE_MATRIX_BINARY_OP(Square, b = a * a);
L
liaogang 已提交
713
template <class T>
H
hedaoyuan 已提交
714
void BaseMatrixT<T>::square2(BaseMatrixT& b) {
Z
zhangjinchao01 已提交
715 716 717 718
  applyBinary(binary::Square<T>(), b);
}

DEFINE_MATRIX_BINARY_OP(SquareDerivative, a *= 2.0 * b);
L
liaogang 已提交
719
template <class T>
Z
zhangjinchao01 已提交
720 721 722 723
void BaseMatrixT<T>::squareDerivative(BaseMatrixT& b) {
  applyBinary(binary::SquareDerivative<T>(), b);
}

L
liaogang 已提交
724 725 726 727
DEFINE_MATRIX_BINARY_OP(Tanh, T tmp = -2.0 * a;
                        tmp = (tmp > EXP_MAX_INPUT) ? EXP_MAX_INPUT : tmp;
                        b = 2.0 / (1.0 + std::exp(tmp)) - 1.0);
template <>
Z
zhangjinchao01 已提交
728 729 730 731 732
void BaseMatrixT<real>::tanh(BaseMatrixT& b) {
  applyBinary(binary::Tanh<real>(), b);
}

DEFINE_MATRIX_BINARY_OP(TanhDerivative, a *= 1 - b * b);
L
liaogang 已提交
733
template <class T>
Z
zhangjinchao01 已提交
734 735 736 737
void BaseMatrixT<T>::tanhDerivative(BaseMatrixT& b) {
  applyBinary(binary::TanhDerivative<T>(), b);
}

L
liaogang 已提交
738 739 740
DEFINE_MATRIX_BINARY_PARAMETER_OP(
    ScaledTanh, TWO_PARAMETER, b = p1 * (2.0 / (1.0 + exp(-2 * p2 * a)) - 1.0));
template <>
Z
zhangjinchao01 已提交
741 742 743 744
void BaseMatrixT<real>::scaledTanh(BaseMatrixT& b, real p1, real p2) {
  applyBinary(binary::ScaledTanh<real>(p1, p2), b);
}

L
liaogang 已提交
745 746
DEFINE_MATRIX_BINARY_PARAMETER_OP(ScaledTanhDerivative,
                                  TWO_PARAMETER,
Z
zhangjinchao01 已提交
747
                                  a *= p2 * (p1 - b * b));
L
liaogang 已提交
748
template <class T>
Z
zhangjinchao01 已提交
749 750 751 752 753
void BaseMatrixT<T>::scaledTanhDerivative(BaseMatrixT& b, T p1, T p2) {
  applyBinary(binary::ScaledTanhDerivative<T>(p1 * p1, p2 / p1), b);
}

DEFINE_MATRIX_BINARY_OP(Reciprocal, b = 1.0f / a);
L
liaogang 已提交
754
template <class T>
H
hedaoyuan 已提交
755
void BaseMatrixT<T>::reciprocal2(BaseMatrixT& b) {
Z
zhangjinchao01 已提交
756 757 758 759
  applyBinary(binary::Reciprocal<T>(), b);
}

DEFINE_MATRIX_BINARY_OP(ReciprocalDerivative, a *= -b * b);
L
liaogang 已提交
760
template <class T>
Z
zhangjinchao01 已提交
761 762 763 764 765
void BaseMatrixT<T>::reciprocalDerivative(BaseMatrixT& b) {
  applyBinary(binary::ReciprocalDerivative<T>(), b);
}

DEFINE_MATRIX_BINARY_OP(Abs, b = a > 0.0f ? a : -a);
L
liaogang 已提交
766 767 768 769
template <class T>
void BaseMatrixT<T>::abs2(BaseMatrixT& b) {
  applyBinary(binary::Abs<T>(), b);
}
Z
zhangjinchao01 已提交
770 771

DEFINE_MATRIX_BINARY_OP(AbsDerivative, a = (b > 0) ? a : (b < 0) ? -a : 0);
L
liaogang 已提交
772
template <class T>
Z
zhangjinchao01 已提交
773 774 775 776
void BaseMatrixT<T>::absDerivative(BaseMatrixT& b) {
  applyBinary(binary::AbsDerivative<T>(), b);
}

L
liaogang 已提交
777 778 779 780 781 782 783
DEFINE_MATRIX_BINARY_OP(Sigmoid, const T THRESHOLD_MIN = -40.0;
                        const T THRESHOLD_MAX = 13.0;
                        T tmp = (a < THRESHOLD_MIN)
                                    ? THRESHOLD_MIN
                                    : ((a > THRESHOLD_MAX) ? THRESHOLD_MAX : a);
                        b = 1.0f / (1.0f + exp(-tmp)));
template <>
Z
zhangjinchao01 已提交
784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816
void BaseMatrixT<real>::sigmoid(BaseMatrixT& b) {
  if (useGpu_) {
    applyBinary(binary::Sigmoid<real>(), b);
  } else {  // cpu versioni
    size_t numSamples = this->height_;
    size_t dim = this->width_;
    CHECK_EQ(b.height_, numSamples);
    CHECK_EQ(b.width_, dim);
    const real* in = this->data_;
    real* out = b.data_;

    // out = - in
    const float THRESHOLD_MIN = -40.0;  // make sure sigmoid(x) > 0
    const float THRESHOLD_MAX = 13.0;   // make sure sigmoid(x) < 1
    for (size_t i = 0; i < numSamples * dim; ++i) {
      real tmp = in[i];
      tmp = (tmp < THRESHOLD_MIN)
                ? THRESHOLD_MIN
                : ((tmp > THRESHOLD_MAX) ? THRESHOLD_MAX : tmp);
      out[i] = -tmp;
    }

    // out = exp(out)
    vExp(numSamples * dim, out, out);

    // out = 1 / (1 + out)
    for (size_t i = 0; i < numSamples * dim; ++i) {
      out[i] = 1 / (1 + out[i]);
    }
  }
}

DEFINE_MATRIX_BINARY_OP(SigmoidDerivative, a *= b * (1 - b));
L
liaogang 已提交
817
template <class T>
Z
zhangjinchao01 已提交
818 819 820 821 822
void BaseMatrixT<T>::sigmoidDerivative(BaseMatrixT& b) {
  applyBinary(binary::SigmoidDerivative<T>(), b);
}

DEFINE_MATRIX_BINARY_OP(ExpDerivative, a *= b);
L
liaogang 已提交
823
template <class T>
Z
zhangjinchao01 已提交
824 825 826 827 828
void BaseMatrixT<T>::expDerivative(BaseMatrixT& b) {
  applyBinary(binary::ExpDerivative<T>(), b);
}

DEFINE_MATRIX_BINARY_OP(Sign, b = a > 0.0f ? 1.0f : -1.0f);
L
liaogang 已提交
829
template <class T>
H
hedaoyuan 已提交
830 831 832
void BaseMatrixT<T>::sign2(BaseMatrixT& b) {
  applyBinary(binary::Sign<T>(), b);
}
Z
zhangjinchao01 已提交
833 834

DEFINE_MATRIX_BINARY_OP(Exp, a = exp(b));
L
liaogang 已提交
835
template <>
H
hedaoyuan 已提交
836
void BaseMatrixT<real>::exp2(BaseMatrixT& b) {
Z
zhangjinchao01 已提交
837 838 839 840
  applyBinary(binary::Exp<real>(), b);
}

DEFINE_MATRIX_BINARY_OP(Log, a = log(b));
L
liaogang 已提交
841
template <>
H
hedaoyuan 已提交
842
void BaseMatrixT<real>::log2(BaseMatrixT& b) {
Z
zhangjinchao01 已提交
843 844 845 846 847 848 849 850
  if (useGpu_) {
    applyBinary(binary::Log<real>(), b);
  } else {
    vLog(height_ * width_, b.data_, data_);
  }
}

DEFINE_MATRIX_BINARY_OP(Sqrt, a = sqrt(b));
L
liaogang 已提交
851
template <>
H
hedaoyuan 已提交
852
void BaseMatrixT<real>::sqrt2(BaseMatrixT& b) {
Z
zhangjinchao01 已提交
853 854 855 856
  applyBinary(binary::Sqrt<real>(), b);
}

DEFINE_MATRIX_BINARY_OP(InvSqrt, a = 1.0f / sqrt(b));
L
liaogang 已提交
857
template <>
Z
zhangjinchao01 已提交
858 859 860 861 862 863 864 865 866 867 868
void BaseMatrixT<real>::invSqrt(BaseMatrixT& b) {
  if (useGpu_) {
    applyBinary(binary::InvSqrt<real>(), b);
  } else {  // cpu branch
    CHECK_EQ(height_, b.height_);
    CHECK_EQ(width_, b.width_);
    vInvSqrt(height_ * width_, b.data_, data_);
  }
}

DEFINE_MATRIX_BINARY_PARAMETER_OP(IsEqual, ONE_PARAMETER, a = (b == p));
L
liaogang 已提交
869
template <class T>
Z
zhangjinchao01 已提交
870 871 872 873 874
void BaseMatrixT<T>::isEqualTo(BaseMatrixT& b, T value) {
  applyBinary(binary::IsEqual<T>(value), b);
}

DEFINE_MATRIX_BINARY_PARAMETER_OP(AddScalar, ONE_PARAMETER, a = b + p);
L
liaogang 已提交
875
template <class T>
Z
zhangjinchao01 已提交
876 877 878 879 880
void BaseMatrixT<T>::addScalar(BaseMatrixT& b, T p) {
  applyBinary(binary::AddScalar<T>(p), b);
}

DEFINE_MATRIX_BINARY_PARAMETER_OP(SubScalar, ONE_PARAMETER, a = b - p);
L
liaogang 已提交
881
template <class T>
Z
zhangjinchao01 已提交
882 883 884 885 886
void BaseMatrixT<T>::subScalar(BaseMatrixT& b, T p) {
  applyBinary(binary::SubScalar<T>(p), b);
}

DEFINE_MATRIX_BINARY_PARAMETER_OP(MulScalar, ONE_PARAMETER, a = b * p);
L
liaogang 已提交
887
template <class T>
Z
zhangjinchao01 已提交
888 889 890 891 892
void BaseMatrixT<T>::mulScalar(BaseMatrixT& b, T p) {
  applyBinary(binary::MulScalar<T>(p), b);
}

DEFINE_MATRIX_BINARY_PARAMETER_OP(DivScalar, ONE_PARAMETER, a = b / p);
L
liaogang 已提交
893
template <class T>
Z
zhangjinchao01 已提交
894 895 896 897 898
void BaseMatrixT<T>::divScalar(BaseMatrixT& b, T p) {
  applyBinary(binary::DivScalar<T>(p), b);
}

DEFINE_MATRIX_BINARY_PARAMETER_OP(ScalarDiv, ONE_PARAMETER, a = p / b);
L
liaogang 已提交
899
template <class T>
Z
zhangjinchao01 已提交
900 901 902 903 904 905 906 907 908 909 910
void BaseMatrixT<T>::scalarDiv(BaseMatrixT& b, T p) {
  applyBinary(binary::ScalarDiv<T>(p), b);
}

/**
 * @brief   ternary operator.
 *
 */

DEFINE_MATRIX_TERNARY_OP(SoftCrossEntropy,
                         a = -c * log(b) - (1 - c) * log(1 - b));
L
liaogang 已提交
911
template <>
Z
zhangjinchao01 已提交
912 913 914 915 916
void BaseMatrixT<real>::softCrossEntropy(BaseMatrixT& b, BaseMatrixT& c) {
  applyTernary(ternary::SoftCrossEntropy<real>(), b, c);
}

DEFINE_MATRIX_TERNARY_OP(SoftCrossEntropyBp, a += (b - c) / (b * (1 - b)));
L
liaogang 已提交
917
template <class T>
Z
zhangjinchao01 已提交
918 919 920 921 922 923
void BaseMatrixT<T>::softCrossEntropyBp(BaseMatrixT& b, BaseMatrixT& c) {
  applyTernary(ternary::SoftCrossEntropyBp<T>(), b, c);
}

DEFINE_MATRIX_TERNARY_OP(BinaryCrossEntropy,
                         a = c > 0.5 ? -log(b) : -log(1.0 - b));
L
liaogang 已提交
924
template <>
Z
zhangjinchao01 已提交
925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951
void BaseMatrixT<real>::binaryLabelCrossEntropy(BaseMatrixT& b,
                                                BaseMatrixT& c) {
  if (useGpu_) {
    applyTernary(ternary::BinaryCrossEntropy<real>(), b, c);
  } else {
    CHECK_EQ(height_, b.height_);
    CHECK_EQ(height_, c.height_);
    CHECK_EQ(width_, b.width_);
    CHECK_EQ(width_, c.width_);

    size_t size = height_ * width_;
    real* out = b.data_;
    real* label = c.data_;
    real* cost = data_;

    for (size_t i = 0; i < size; ++i) {
      cost[i] = label[i] > 0.5 ? out[i] : 1.0 - out[i];
    }
    vLog(size, cost, cost);
    for (size_t i = 0; i < size; ++i) {
      cost[i] *= -1.0;
    }
  }
}

DEFINE_MATRIX_TERNARY_OP(BinaryCrossEntropyBp,
                         a += c > 0.5 ? -1.0 / b : 1.0 / (1.0 - b));
L
liaogang 已提交
952
template <class T>
Z
zhangjinchao01 已提交
953 954 955 956 957
void BaseMatrixT<T>::binaryLabelCrossEntropyBp(BaseMatrixT& b, BaseMatrixT& c) {
  applyTernary(ternary::BinaryCrossEntropyBp<T>(), b, c);
}

DEFINE_MATRIX_TERNARY_OP(Add, a = b + c);
L
liaogang 已提交
958
template <class T>
Z
zhangjinchao01 已提交
959 960 961 962 963
void BaseMatrixT<T>::add(BaseMatrixT& b, BaseMatrixT& c) {
  applyTernary(ternary::Add<T>(), b, c);
}

DEFINE_MATRIX_TERNARY_PARAMETER_OP(Add1, TWO_PARAMETER, a = p1 * b + p2 * c);
L
liaogang 已提交
964
template <class T>
Z
zhangjinchao01 已提交
965 966 967 968 969
void BaseMatrixT<T>::add(BaseMatrixT& b, T p1, BaseMatrixT& c, T p2) {
  applyTernary(ternary::Add1<T>(p1, p2), b, c);
}

DEFINE_MATRIX_TERNARY_OP(Sub, a = b - c);
L
liaogang 已提交
970
template <class T>
Z
zhangjinchao01 已提交
971 972 973 974 975
void BaseMatrixT<T>::sub(BaseMatrixT& b, BaseMatrixT& c) {
  applyTernary(ternary::Sub<T>(), b, c);
}

DEFINE_MATRIX_TERNARY_PARAMETER_OP(Sub1, TWO_PARAMETER, a = p1 * b - p2 * c);
L
liaogang 已提交
976
template <class T>
Z
zhangjinchao01 已提交
977 978 979 980 981
void BaseMatrixT<T>::sub(BaseMatrixT& b, T p1, BaseMatrixT& c, T p2) {
  applyTernary(ternary::Sub1<T>(p1, p2), b, c);
}

DEFINE_MATRIX_TERNARY_OP(Add2, a = a + b + c);
L
liaogang 已提交
982
template <class T>
Z
zhangjinchao01 已提交
983 984 985 986
void BaseMatrixT<T>::add2(BaseMatrixT& b, BaseMatrixT& c) {
  applyTernary(ternary::Add2<T>(), b, c);
}

L
liaogang 已提交
987 988
DEFINE_MATRIX_TERNARY_PARAMETER_OP(Add3,
                                   THREE_PARAMETER,
Z
zhangjinchao01 已提交
989
                                   a = p1 * a + p2 * b + p3 * c);
L
liaogang 已提交
990
template <class T>
Z
zhangjinchao01 已提交
991 992 993 994
void BaseMatrixT<T>::add2(BaseMatrixT& b, BaseMatrixT& c, T p1, T p2, T p3) {
  applyTernary(ternary::Add3<T>(p1, p2, p3), b, c);
}

L
liaogang 已提交
995 996
DEFINE_MATRIX_TERNARY_PARAMETER_OP(SgdUpdate,
                                   THREE_PARAMETER,
Z
zhangjinchao01 已提交
997 998
                                   c = p2 * c - p1 * (b + p3 * a);
                                   a = a + c);
L
liaogang 已提交
999
template <class T>
Z
zhangjinchao01 已提交
1000 1001
void BaseMatrixT<T>::sgdUpdate(BaseMatrixT& b,  // grad
                               BaseMatrixT& c,  // mom
L
liaogang 已提交
1002 1003 1004
                               T p1,            // learningRate,
                               T p2,            // momentum,
                               T p3) {          // decayRate
Z
zhangjinchao01 已提交
1005 1006 1007
  applyTernary(ternary::SgdUpdate<T>(p1, p2, p3), b, c);
}

L
liaogang 已提交
1008 1009
DEFINE_MATRIX_QUATERNARY_PARAMETER_OP(SgdUpdate,
                                      THREE_PARAMETER,
Z
zhangjinchao01 已提交
1010 1011
                                      c = p2 * c - p1 * d * (b + p3 * a);
                                      a += c);
L
liaogang 已提交
1012
template <class T>
Z
zhangjinchao01 已提交
1013 1014 1015
void BaseMatrixT<T>::sgdUpdate(BaseMatrixT& b,  // grad,
                               BaseMatrixT& c,  // mom,
                               BaseMatrixT& d,  // lr,
L
liaogang 已提交
1016 1017 1018
                               T p1,            // learningRate,
                               T p2,            // momentum,
                               T p3) {          // decayRate
Z
zhangjinchao01 已提交
1019 1020 1021 1022 1023 1024 1025
  applyQuaternary(quaternary::SgdUpdate<T>(p1, p2, p3), b, c, d);
}

DEFINE_MATRIX_BINARY_PARAMETER_OP(ApplyL1, ONE_PARAMETER, T lambda = p * b;
                                  a = (a > lambda)
                                          ? (a - lambda)
                                          : (a < -lambda) ? (a + lambda) : 0);
L
liaogang 已提交
1026
template <class T>
Z
zhangjinchao01 已提交
1027 1028 1029 1030
void BaseMatrixT<T>::applyL1(BaseMatrixT& lr, T learningRate, T decayRate) {
  applyBinary(binary::ApplyL1<T>(learningRate * decayRate), lr);
}

L
liaogang 已提交
1031
template <>
Z
zhangjinchao01 已提交
1032 1033 1034 1035 1036 1037
void BaseMatrixT<real>::applyL1(BaseMatrixT& lr,
                                real learningRate,
                                real decayRate) {
  if (useGpu_) {
    applyBinary(binary::ApplyL1<real>(learningRate * decayRate), lr);
  } else {
L
liaogang 已提交
1038 1039 1040 1041
    simd::decayL1(this->data_,
                  this->data_,
                  lr.data_,
                  learningRate * decayRate,
Z
zhangjinchao01 已提交
1042 1043 1044 1045 1046 1047 1048 1049
                  height_ * width_);
  }
}

DEFINE_MATRIX_UNARY_PARAMETER_OP(ApplyL1, ONE_PARAMETER, T lambda = p;
                                 a = (a > lambda)
                                         ? (a - lambda)
                                         : (a < -lambda) ? (a + lambda) : 0);
L
liaogang 已提交
1050
template <class T>
Z
zhangjinchao01 已提交
1051 1052 1053 1054
void BaseMatrixT<T>::applyL1(T learningRate, T decayRate) {
  applyUnary(unary::ApplyL1<T>(learningRate * decayRate));
}

L
liaogang 已提交
1055
template <>
Z
zhangjinchao01 已提交
1056 1057 1058 1059
void BaseMatrixT<real>::applyL1(real learningRate, real decayRate) {
  if (useGpu_) {
    applyUnary(unary::ApplyL1<real>(learningRate * decayRate));
  } else {
L
liaogang 已提交
1060 1061
    simd::decayL1(
        this->data_, this->data_, learningRate * decayRate, height_ * width_);
Z
zhangjinchao01 已提交
1062 1063 1064
  }
}

L
liaogang 已提交
1065 1066
DEFINE_MATRIX_BINARY_PARAMETER_OP(ApplyL2,
                                  ONE_PARAMETER,
Z
zhangjinchao01 已提交
1067
                                  a *= (1.0f / (1.0f + p * b)));
L
liaogang 已提交
1068
template <class T>
Z
zhangjinchao01 已提交
1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080
void BaseMatrixT<T>::applyL2(BaseMatrixT& lr, T learningRate, T decayRate) {
  if (useGpu_) {
    applyBinary(binary::ApplyL2<T>(learningRate * decayRate), lr);
  } else {
    size_t size = this->height_ * this->width_;
    T decay = learningRate * decayRate;
    for (size_t j = 0; j < size; ++j) {
      this->data_[j] *= 1.0f / (1.0f + decay * lr.data_[j]);
    }
  }
}

L
liaogang 已提交
1081
template <class T>
Z
zhangjinchao01 已提交
1082 1083 1084 1085 1086
void BaseMatrixT<T>::applyL2(T learningRate, T decayRate) {
  BaseMatrixT<T>::mulScalar(1.0f / (1.0f + learningRate * decayRate));
}

DEFINE_MATRIX_BINARY_OP(DotMul, a *= b);
L
liaogang 已提交
1087
template <class T>
Z
zhangjinchao01 已提交
1088 1089 1090 1091 1092
void BaseMatrixT<T>::dotMul(BaseMatrixT& b) {
  applyBinary(binary::DotMul<T>(), b);
}

DEFINE_MATRIX_TERNARY_OP(DotMul, a = b * c);
L
liaogang 已提交
1093
template <class T>
Z
zhangjinchao01 已提交
1094 1095 1096 1097 1098
void BaseMatrixT<T>::dotMul(BaseMatrixT& b, BaseMatrixT& c) {
  applyTernary(ternary::DotMul<T>(), b, c);
}

DEFINE_MATRIX_TERNARY_OP(DotDiv, a = (b == 0.0) ? 0.0 : b / c);
L
liaogang 已提交
1099
template <class T>
Z
zhangjinchao01 已提交
1100 1101 1102 1103
void BaseMatrixT<T>::dotDiv(BaseMatrixT& b, BaseMatrixT& c) {
  applyTernary(ternary::DotDiv<T>(), b, c);
}

L
liaogang 已提交
1104 1105
DEFINE_MATRIX_TERNARY_PARAMETER_OP(DotDiv2P,
                                   TWO_PARAMETER,
Z
zhangjinchao01 已提交
1106
                                   a = (b + p1) / (c + p2));
L
liaogang 已提交
1107
template <class T>
Z
zhangjinchao01 已提交
1108 1109 1110 1111 1112 1113 1114 1115 1116
void BaseMatrixT<T>::dotDiv(BaseMatrixT& b, BaseMatrixT& c, T p1, T p2) {
  applyTernary(ternary::DotDiv2P<T>(p1, p2), b, c);
}

DEFINE_MATRIX_QUATERNARY_OP(RankLoss, const T THRESHOLD = 40.0; a = b - c;
                            a = (a > THRESHOLD)
                                    ? THRESHOLD
                                    : ((a < -THRESHOLD) ? (-THRESHOLD) : a);
                            a = log(1 + exp(a)) - a * d);
L
liaogang 已提交
1117
template <>
Z
zhangjinchao01 已提交
1118 1119 1120 1121 1122 1123 1124 1125 1126 1127
void BaseMatrixT<real>::rankLoss(BaseMatrixT& b,
                                 BaseMatrixT& c,
                                 BaseMatrixT& d) {
  applyQuaternary(quaternary::RankLoss<real>(), b, c, d);
}

DEFINE_MATRIX_QUATERNARY_OP(RankLossBp, const T THRESHOLD = 40.0; a = b - c;
                            a = (a > THRESHOLD)
                                    ? THRESHOLD
                                    : ((a < -THRESHOLD) ? (-THRESHOLD) : a);
L
liaogang 已提交
1128 1129 1130
                            a = exp(a);
                            a = (a / (1 + a) - d));
template <>
Z
zhangjinchao01 已提交
1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142
void BaseMatrixT<real>::rankLossBp(BaseMatrixT& b,
                                   BaseMatrixT& c,
                                   BaseMatrixT& d) {
  applyQuaternary(quaternary::RankLossBp<real>(), b, c, d);
}

/* this = log(1 + exp(b)) - c * b */
DEFINE_MATRIX_TERNARY_OP(LogisticRegressionLoss, const T THRESHOLD = 40.0;
                         T x = (b > THRESHOLD) ? THRESHOLD : (b < -THRESHOLD)
                                                                 ? -THRESHOLD
                                                                 : b;
                         a = log(1 + exp(x)) - c * x);
L
liaogang 已提交
1143
template <>
Z
zhangjinchao01 已提交
1144 1145 1146 1147 1148 1149 1150 1151 1152
void BaseMatrixT<real>::logisticRegressionLoss(BaseMatrixT& b, BaseMatrixT& c) {
  applyTernary(ternary::LogisticRegressionLoss<real>(), b, c);
}

/* this = exp(b)/(1+exp(b)) - c */
DEFINE_MATRIX_TERNARY_OP(LogisticRegressionLossBp, const T THRESHOLD = 40.0;
                         T x = (b > THRESHOLD) ? THRESHOLD : (b < -THRESHOLD)
                                                                 ? -THRESHOLD
                                                                 : b;
L
liaogang 已提交
1153 1154 1155
                         x = exp(x);
                         a = x / (1 + x) - c);
template <>
Z
zhangjinchao01 已提交
1156 1157 1158 1159 1160 1161
void BaseMatrixT<real>::logisticRegressionLossBp(BaseMatrixT& b,
                                                 BaseMatrixT& c) {
  applyTernary(ternary::LogisticRegressionLossBp<real>(), b, c);
}

DEFINE_MATRIX_TERNARY_OP(BiggerThan, a = (b > c) ? 1.0f : 0.0f);
L
liaogang 已提交
1162
template <class T>
Z
zhangjinchao01 已提交
1163 1164 1165 1166 1167 1168
void BaseMatrixT<T>::biggerThan(BaseMatrixT& b, BaseMatrixT& c) {
  applyTernary(ternary::BiggerThan<T>(), b, c);
}

DEFINE_MATRIX_QUATERNARY_OP(
    BiggerThan, a = ((b > c && d > 0.5f) || (b < c && d < 0.5f)) ? 1.0f : 0.0f);
L
liaogang 已提交
1169
template <class T>
Z
zhangjinchao01 已提交
1170 1171 1172 1173 1174 1175 1176
void BaseMatrixT<T>::biggerThan(BaseMatrixT& b,
                                BaseMatrixT& c,
                                BaseMatrixT& d) {
  applyQuaternary(quaternary::BiggerThan<T>(), b, c, d);
}

DEFINE_MATRIX_TERNARY_OP(Max, a = (b > c) ? b : c);
L
liaogang 已提交
1177
template <class T>
H
hedaoyuan 已提交
1178
void BaseMatrixT<T>::max2(BaseMatrixT& b, BaseMatrixT& c) {
Z
zhangjinchao01 已提交
1179 1180 1181
  applyTernary(ternary::Max<T>(), b, c);
}

L
liaogang 已提交
1182 1183
DEFINE_MATRIX_TERNARY_PARAMETER_OP(BinaryClassificationError,
                                   ONE_PARAMETER,
Z
zhangjinchao01 已提交
1184
                                   c += ((a > p) == (b > p)) ? 0.0f : 1.0f);
L
liaogang 已提交
1185 1186 1187 1188 1189
template <class T>
void BaseMatrixT<T>::binaryClassificationError2(size_t destCol,
                                                BaseMatrixT& b,
                                                BaseMatrixT& c,
                                                T p) {
Z
zhangjinchao01 已提交
1190 1191 1192 1193
  CHECK(!useGpu_) << "do not support gpu";
  MatrixOffset offset(0, 0, 0, 0, destCol, 0);
  int numRows = b.height_;
  int numCols = b.width_;
L
liaogang 已提交
1194 1195 1196 1197 1198 1199 1200 1201
  b.applyTernary(ternary::BinaryClassificationError<T>(p),
                 c,
                 *this,
                 numRows,
                 numCols,
                 offset,
                 false_type(),
                 true_type() /*cAsColVector*/);
Z
zhangjinchao01 已提交
1202 1203
}

L
liaogang 已提交
1204
template <>
Z
zhangjinchao01 已提交
1205 1206 1207 1208 1209 1210 1211
void BaseMatrixT<real>::binaryClassificationError(size_t destCol,
                                                  BaseMatrixT& b,
                                                  BaseMatrixT& c,
                                                  real p) {
  MatrixOffset offset(destCol, 0, 0, 0, 0, 0);
  int numRows = b.height_;
  int numCols = b.width_;
L
liaogang 已提交
1212 1213 1214 1215 1216 1217 1218 1219 1220
  aggregate(aggregate::sum(),
            base::binary::classificationError(p),
            base::binary::add(),
            b,
            c,
            numRows,
            numCols,
            offset,
            false_type(),
Z
zhangjinchao01 已提交
1221 1222 1223
            true_type() /*aAsColVector*/);
}

L
liaogang 已提交
1224 1225
DEFINE_MATRIX_QUATERNARY_PARAMETER_OP(Add3,
                                      THREE_PARAMETER,
Z
zhangjinchao01 已提交
1226
                                      a = p1 * b + p2 * c + p3 * d);
L
liaogang 已提交
1227 1228 1229
template <class T>
void BaseMatrixT<T>::add3(
    BaseMatrixT& b, BaseMatrixT& c, BaseMatrixT& d, T p1, T p2, T p3) {
Z
zhangjinchao01 已提交
1230 1231 1232 1233
  applyQuaternary(quaternary::Add3<T>(p1, p2, p3), b, c, d);
}

DEFINE_MATRIX_TERNARY_OP(DotMulSquare, a = b * c * c);
L
liaogang 已提交
1234
template <class T>
Z
zhangjinchao01 已提交
1235 1236 1237 1238 1239
void BaseMatrixT<T>::dotMulSquare(BaseMatrixT& b, BaseMatrixT& c) {
  applyTernary(ternary::DotMulSquare<T>(), b, c);
}

DEFINE_MATRIX_TERNARY_OP(DotSquareSquare, a = b * b * c * c);
L
liaogang 已提交
1240
template <class T>
Z
zhangjinchao01 已提交
1241 1242 1243 1244 1245
void BaseMatrixT<T>::dotSquareSquare(BaseMatrixT& b, BaseMatrixT& c) {
  applyTernary(ternary::DotSquareSquare<T>(), b, c);
}

DEFINE_MATRIX_BINARY_OP(DotMulSquare, a *= b * b);
L
liaogang 已提交
1246
template <class T>
Z
zhangjinchao01 已提交
1247 1248 1249 1250 1251
void BaseMatrixT<T>::dotMulSquare(BaseMatrixT& b) {
  applyBinary(binary::DotMulSquare<T>(), b);
}

DEFINE_MATRIX_BINARY_OP(DotSquareMul, a = a * a * b);
L
liaogang 已提交
1252
template <class T>
Z
zhangjinchao01 已提交
1253 1254 1255 1256
void BaseMatrixT<T>::dotSquareMul(BaseMatrixT& b) {
  applyBinary(binary::DotSquareMul<T>(), b);
}

L
liaogang 已提交
1257 1258
DEFINE_MATRIX_QUATERNARY_PARAMETER_OP(AddSquareSum,
                                      THREE_PARAMETER,
Z
zhangjinchao01 已提交
1259 1260
                                      T tmp = p1 * b + p2 * c + p3 * d;
                                      a += tmp * tmp);
L
liaogang 已提交
1261 1262 1263
template <class T>
void BaseMatrixT<T>::addSquareSum(
    BaseMatrixT& b, BaseMatrixT& c, BaseMatrixT d, T p1, T p2, T p3) {
Z
zhangjinchao01 已提交
1264 1265 1266 1267
  applyQuaternary(quaternary::AddSquareSum<T>(p1, p2, p3), b, c, d);
}

DEFINE_MATRIX_BINARY_PARAMETER_OP(AddSquare, ONE_PARAMETER, a += p * b * b);
L
liaogang 已提交
1268
template <class T>
Z
zhangjinchao01 已提交
1269 1270 1271 1272
void BaseMatrixT<T>::addSquare(BaseMatrixT& b, T p) {
  applyBinary(binary::AddSquare<T>(p), b);
}

L
liaogang 已提交
1273 1274
DEFINE_MATRIX_BINARY_PARAMETER_OP(DecayAddSquare,
                                  TWO_PARAMETER,
Z
zhangjinchao01 已提交
1275
                                  a = p1 * a + p2 * b * b);
L
liaogang 已提交
1276
template <class T>
Z
zhangjinchao01 已提交
1277 1278 1279 1280
void BaseMatrixT<T>::decayAddSquare(BaseMatrixT& b, T p1, T p2) {
  applyBinary(binary::DecayAddSquare<T>(p1, p2), b);
}

L
liaogang 已提交
1281 1282
DEFINE_MATRIX_TERNARY_PARAMETER_OP(DecayAddSquareMul,
                                   TWO_PARAMETER,
Z
zhangjinchao01 已提交
1283
                                   a = p1 * a + p2 * b * b * c * c);
L
liaogang 已提交
1284 1285 1286 1287
template <class T>
void BaseMatrixT<T>::decayAddSquareMul(BaseMatrixT& b,
                                       BaseMatrixT& c,
                                       T p1,
Z
zhangjinchao01 已提交
1288 1289 1290 1291
                                       T p2) {
  applyTernary(ternary::DecayAddSquareMul<T>(p1, p2), b, c);
}

L
liaogang 已提交
1292 1293
DEFINE_MATRIX_TERNARY_PARAMETER_OP(ReciprocalSum,
                                   THREE_PARAMETER,
Z
zhangjinchao01 已提交
1294
                                   a = 1 / (p1 * b + p2 * c + p3));
L
liaogang 已提交
1295 1296 1297
template <class T>
void BaseMatrixT<T>::reciprocalSum(
    BaseMatrixT& b, BaseMatrixT& c, T p1, T p2, T p3) {
Z
zhangjinchao01 已提交
1298 1299 1300
  applyTernary(ternary::ReciprocalSum<T>(p1, p2, p3), b, c);
}

L
liaogang 已提交
1301 1302
DEFINE_MATRIX_BINARY_PARAMETER_OP(Reciprocal2,
                                  TWO_PARAMETER,
Z
zhangjinchao01 已提交
1303
                                  a = 1 / (p1 * b + p2));
L
liaogang 已提交
1304
template <class T>
H
hedaoyuan 已提交
1305
void BaseMatrixT<T>::reciprocal2(BaseMatrixT& b, T p1, T p2) {
Z
zhangjinchao01 已提交
1306 1307 1308
  applyBinary(binary::Reciprocal2<T>(p1, p2), b);
}

L
liaogang 已提交
1309 1310
DEFINE_MATRIX_TERNARY_PARAMETER_OP(DotMulSquareSum,
                                   TWO_PARAMETER,
Z
zhangjinchao01 已提交
1311 1312
                                   T tmp = p1 * b + p2 * c;
                                   a *= tmp * tmp);
L
liaogang 已提交
1313 1314 1315 1316
template <class T>
void BaseMatrixT<T>::dotMulSquareSum(BaseMatrixT& b,
                                     BaseMatrixT& c,
                                     T p1,
Z
zhangjinchao01 已提交
1317 1318 1319 1320
                                     T p2) {
  applyTernary(ternary::DotMulSquareSum<T>(p1, p2), b, c);
}

L
liaogang 已提交
1321 1322
DEFINE_MATRIX_TERNARY_PARAMETER_OP(DotSquareSum,
                                   TWO_PARAMETER,
Z
zhangjinchao01 已提交
1323 1324
                                   T tmp = p1 * b + p2 * c;
                                   a = tmp * tmp);
L
liaogang 已提交
1325
template <class T>
Z
zhangjinchao01 已提交
1326 1327 1328 1329
void BaseMatrixT<T>::dotSquareSum(BaseMatrixT& b, BaseMatrixT& c, T p1, T p2) {
  applyTernary(ternary::DotSquareSum<T>(p1, p2), b, c);
}

L
liaogang 已提交
1330 1331
DEFINE_MATRIX_TERNARY_PARAMETER_OP(DotMulSum,
                                   TWO_PARAMETER,
Z
zhangjinchao01 已提交
1332
                                   a *= p1 * b + p2 * c);
L
liaogang 已提交
1333
template <class T>
Z
zhangjinchao01 已提交
1334 1335 1336 1337 1338
void BaseMatrixT<T>::dotMulSum(BaseMatrixT& b, BaseMatrixT& c, T p1, T p2) {
  applyTernary(ternary::DotMulSum<T>(p1, p2), b, c);
}

DEFINE_MATRIX_BINARY_OP(CopyAndClear, b = a; a = 0);
L
liaogang 已提交
1339
template <class T>
Z
zhangjinchao01 已提交
1340 1341 1342 1343
void BaseMatrixT<T>::copyAndClear(BaseMatrixT& b) {
  applyBinary(binary::CopyAndClear<T>(), b);
}

L
liaogang 已提交
1344 1345
DEFINE_MATRIX_TERNARY_PARAMETER_OP(AddDotMul,
                                   TWO_PARAMETER,
Z
zhangjinchao01 已提交
1346
                                   a = p1 * a + p2 * b * c);
L
liaogang 已提交
1347
template <class T>
Z
zhangjinchao01 已提交
1348 1349 1350 1351 1352
void BaseMatrixT<T>::addDotMul(BaseMatrixT& b, BaseMatrixT& c, T p1, T p2) {
  applyTernary(ternary::AddDotMul<T>(p1, p2), b, c);
}

DEFINE_MATRIX_BINARY_OP(Assign, a = b;);
L
liaogang 已提交
1353
template <class T>
Z
zhangjinchao01 已提交
1354 1355 1356 1357 1358 1359 1360 1361 1362 1363
void BaseMatrixT<T>::assign(BaseMatrixT& b) {
  if (useGpu_) {
    applyBinary(binary::Assign<T>(), b);
  } else {  // cpu version
    CHECK_EQ(this->height_, b.height_);
    CHECK_EQ(this->width_, b.width_);
    memcpy(data_, b.data_, sizeof(T) * height_ * width_);
  }
}

L
liaogang 已提交
1364
template <class T>
Z
zhangjinchao01 已提交
1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382
void BaseMatrixT<T>::assignAtOffset(BaseMatrixT& b, int64_t columnOffset) {
  if (columnOffset + b.width_ <= width_) {
    int numRows = height_;
    int numCols = b.width_;
    MatrixOffset offset(columnOffset, 0, 0, 0);
    applyBinary(binary::Assign<T>(), b, numRows, numCols, offset);
  } else if (columnOffset + width_ <= b.width_) {
    int numRows = height_;
    int numCols = width_;
    MatrixOffset offset(0, 0, columnOffset, 0);
    applyBinary(binary::Assign<T>(), b, numRows, numCols, offset);
  } else {
    LOG(FATAL) << "Wrong argument "
               << " a.width=" << width_ << " b.width=" << b.width_
               << " columnOffset=" << columnOffset;
  }
}

X
xutianbing 已提交
1383
DEFINE_MATRIX_BINARY_OP(DeepSwap, T tmp = a; a = b; b = tmp);
L
liaogang 已提交
1384
template <class T>
X
xutianbing 已提交
1385
void BaseMatrixT<T>::deepSwap(BaseMatrixT& b) {
L
liaogang 已提交
1386
  applyBinary(binary::DeepSwap<T>(), b);
X
xutianbing 已提交
1387 1388
}

L
liaogang 已提交
1389
template <>
Z
zhangjinchao01 已提交
1390 1391 1392 1393 1394 1395
void BaseMatrixT<real>::rowDotMul(size_t destCol,
                                  BaseMatrixT& b,
                                  BaseMatrixT& c) {
  int numRows = b.height_;
  int numCols = b.width_;
  MatrixOffset offset(destCol, 0, 0, 0, 0, 0);
L
liaogang 已提交
1396 1397 1398 1399 1400 1401 1402 1403 1404
  aggregate(aggregate::sum(),
            base::binary::mul(),
            base::binary::add(),
            b,
            c,
            numRows,
            numCols,
            offset,
            false_type(),
Z
zhangjinchao01 已提交
1405 1406 1407
            true_type() /*aAsColVector*/);
}

L
liaogang 已提交
1408
template <class T>
Z
zhangjinchao01 已提交
1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430
void BaseMatrixT<T>::rowDotMul2(size_t destCol,
                                BaseMatrixT& b,
                                BaseMatrixT& c) {
  CHECK(!useGpu_) << "do not support gpu";

  size_t height = this->height_;
  CHECK_LT(destCol, this->width_);
  CHECK_EQ(height, b.height_);
  CHECK_EQ(height, c.height_);
  CHECK_EQ(b.width_, c.width_);
  size_t width = b.width_;
  T* A = this->data_;
  const T* B = b.data_;
  const T* C = c.data_;
  for (size_t i = 0; i < height;
       ++i, A += this->width_, B += width, C += width) {
    for (size_t j = 0; j < width; ++j) {
      A[destCol] += B[j] * C[j];
    }
  }
}

L
liaogang 已提交
1431
template <>
Z
zhangjinchao01 已提交
1432 1433 1434 1435
void BaseMatrixT<real>::addDotMulVMM(BaseMatrixT& b, BaseMatrixT& c) {
  MatrixOffset offset(0, 0, 0, 0, 0, 0);
  int numRows = b.height_;
  int numCols = b.width_;
L
liaogang 已提交
1436 1437 1438 1439 1440 1441 1442 1443 1444
  aggregate(aggregate::sum(),
            base::binary::mul(),
            base::binary::add(),
            b,
            c,
            numRows,
            numCols,
            offset,
            true_type() /*aAsRowVector*/,
Z
zhangjinchao01 已提交
1445 1446 1447
            false_type());
}

L
liaogang 已提交
1448
template <class T>
Z
zhangjinchao01 已提交
1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468
void BaseMatrixT<T>::addDotMulVMM2(BaseMatrixT& b, BaseMatrixT& c) {
  CHECK(!useGpu_) << "do not support gpu";

  CHECK_EQ(height_, 1LU);
  CHECK_EQ(b.height_, c.height_);
  CHECK_EQ(width_, b.width_);
  CHECK_EQ(width_, c.width_);
  size_t height = b.height_;
  size_t width = b.width_;
  T* A = this->data_;
  const T* B = b.data_;
  const T* C = c.data_;
  for (size_t i = 0; i < height; ++i, B += width, C += width) {
    for (size_t j = 0; j < width; ++j) {
      A[j] += B[j] * C[j];
    }
  }
}

DEFINE_MATRIX_TERNARY_OP(addDotMulMMV, a += b * c);
L
liaogang 已提交
1469
template <class T>
Z
zhangjinchao01 已提交
1470 1471 1472 1473
void BaseMatrixT<T>::addDotMulMMV(BaseMatrixT& b, BaseMatrixT& c) {
  MatrixOffset offset(0, 0, 0, 0, 0, 0);
  int numRows = height_;
  int numCols = width_;
L
liaogang 已提交
1474 1475 1476 1477 1478 1479 1480 1481
  applyTernary(ternary::addDotMulMMV<T>(),
               b,
               c,
               numRows,
               numCols,
               offset,
               true_type() /*cAsRowVector*/,
               false_type());
Z
zhangjinchao01 已提交
1482 1483
}

L
liaogang 已提交
1484
template <class T>
Z
zhangjinchao01 已提交
1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503
void BaseMatrixT<T>::addDotMulMMV2(BaseMatrixT& b, BaseMatrixT& c) {
  CHECK(!useGpu_) << "do not support gpu";

  CHECK_EQ(c.height_, 1LU);
  CHECK_EQ(height_, b.height_);
  CHECK_EQ(width_, b.width_);
  CHECK_EQ(width_, c.width_);
  size_t height = height_;
  size_t width = width_;
  T* A = this->data_;
  const T* B = b.data_;
  const T* C = c.data_;
  for (size_t i = 0; i < height; ++i, A += width, B += width) {
    for (size_t j = 0; j < width; ++j) {
      A[j] += B[j] * C[j];
    }
  }
}

L
liaogang 已提交
1504
template <class T>
Z
zhangjinchao01 已提交
1505 1506 1507 1508
void BaseMatrixT<T>::rowScale(size_t cCol, BaseMatrixT& b, BaseMatrixT& c) {
  MatrixOffset offset(0, 0, 0, 0, cCol, 0);
  int numRows = height_;
  int numCols = width_;
L
liaogang 已提交
1509 1510 1511 1512 1513 1514 1515 1516
  applyTernary(ternary::DotMul<T>(),
               b,
               c,
               numRows,
               numCols,
               offset,
               false_type(),
               true_type() /*cAsColVector*/);
Z
zhangjinchao01 已提交
1517 1518
}

L
liaogang 已提交
1519
template <class T>
Z
zhangjinchao01 已提交
1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538
void BaseMatrixT<T>::rowScale2(size_t cCol, BaseMatrixT& b, BaseMatrixT& c) {
  CHECK(!useGpu_) << "do not support gpu";

  size_t height = this->height_;
  size_t width = this->width_;
  CHECK_EQ(height, b.height_);
  CHECK_EQ(width, b.width_);
  CHECK_LT(cCol, c.width_);
  CHECK_EQ(height, c.height_);
  T* A = this->data_;
  const T* B = b.data_;
  const T* C = c.data_;
  for (size_t i = 0; i < height; ++i, A += width, B += width, C += c.width_) {
    for (size_t j = 0; j < width; ++j) {
      A[j] = B[j] * C[cCol];
    }
  }
}

L
liaogang 已提交
1539
template <class T>
Z
zhangjinchao01 已提交
1540 1541 1542 1543
void BaseMatrixT<T>::colScale(size_t cRow, BaseMatrixT& b, BaseMatrixT& c) {
  MatrixOffset offset(0, 0, 0, 0, 0, cRow);
  int numRows = height_;
  int numCols = width_;
L
liaogang 已提交
1544 1545 1546 1547 1548 1549 1550 1551
  applyTernary(ternary::DotMul<T>(),
               b,
               c,
               numRows,
               numCols,
               offset,
               true_type() /* cAsRowVector */,
               false_type() /* cAsColVector */);
Z
zhangjinchao01 已提交
1552 1553
}

L
liaogang 已提交
1554
template <class T>
Z
zhangjinchao01 已提交
1555 1556 1557 1558
void BaseMatrixT<T>::addColScale(size_t cRow, BaseMatrixT& b, BaseMatrixT& c) {
  MatrixOffset offset(0, 0, 0, 0, 0, cRow);
  int numRows = height_;
  int numCols = width_;
L
liaogang 已提交
1559 1560 1561 1562 1563 1564 1565 1566
  applyTernary(ternary::addDotMulMMV<T>(),
               b,
               c,
               numRows,
               numCols,
               offset,
               true_type() /* cAsRowVector */,
               false_type() /* cAsColVector */);
Z
zhangjinchao01 已提交
1567 1568
}

L
liaogang 已提交
1569
template <class T>
Z
zhangjinchao01 已提交
1570 1571 1572 1573
void BaseMatrixT<T>::addRowScale(size_t cCol, BaseMatrixT& b, BaseMatrixT& c) {
  MatrixOffset offset(0, 0, 0, 0, cCol, 0);
  int numRows = height_;
  int numCols = width_;
L
liaogang 已提交
1574 1575 1576 1577 1578 1579 1580 1581
  applyTernary(ternary::addDotMulMMV<T>(),
               b,
               c,
               numRows,
               numCols,
               offset,
               false_type(),
               true_type() /*cAsColVector*/);
Z
zhangjinchao01 已提交
1582 1583 1584
}

DEFINE_MATRIX_TERNARY_PARAMETER_OP(RowAdd, ONE_PARAMETER, a = b + p * c);
L
liaogang 已提交
1585
template <class T>
Z
zhangjinchao01 已提交
1586 1587 1588 1589
void BaseMatrixT<T>::rowAdd(size_t cCol, BaseMatrixT& b, BaseMatrixT& c, T p) {
  MatrixOffset offset(0, 0, 0, 0, cCol, 0);
  int numRows = height_;
  int numCols = width_;
L
liaogang 已提交
1590 1591 1592 1593 1594 1595 1596 1597
  applyTernary(ternary::RowAdd<T>(p),
               b,
               c,
               numRows,
               numCols,
               offset,
               false_type(),
               true_type() /*cAsColVector*/);
Z
zhangjinchao01 已提交
1598 1599 1600
}

DEFINE_MATRIX_TERNARY_OP(RowPow, a = pow(b, c));
L
liaogang 已提交
1601
template <>
Z
zhangjinchao01 已提交
1602 1603 1604 1605 1606
void BaseMatrixT<real>::rowPow(size_t cCol, BaseMatrixT& b, BaseMatrixT& c) {
  if (useGpu_) {
    MatrixOffset offset(0, 0, 0, 0, cCol, 0);
    int numRows = height_;
    int numCols = width_;
L
liaogang 已提交
1607 1608 1609 1610 1611 1612 1613 1614
    applyTernary(ternary::RowPow<real>(),
                 b,
                 c,
                 numRows,
                 numCols,
                 offset,
                 false_type(),
                 true_type() /*cAsColVector*/);
Z
zhangjinchao01 已提交
1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630
  } else {
    size_t height = this->height_;
    size_t width = this->width_;
    CHECK_EQ(height, b.height_);
    CHECK_EQ(width, b.width_);
    CHECK_LT(cCol, c.width_);
    CHECK_EQ(height, c.height_);
    real* A = this->data_;
    const real* B = b.data_;
    const real* C = c.data_;
    for (size_t i = 0; i < height; ++i, A += width, B += width, C += c.width_) {
      vPow(width, B, C[cCol], A);
    }
  }
}

L
liaogang 已提交
1631
template <class T>
Z
zhangjinchao01 已提交
1632 1633 1634 1635
void BaseMatrixT<T>::mulRowVector(BaseMatrixT& b) {
  MatrixOffset offset(0, 0, 0, 0);
  int numRows = height_;
  int numCols = width_;
L
liaogang 已提交
1636 1637 1638 1639 1640 1641 1642
  applyBinary(binary::DotMul<T>(),
              b,
              numRows,
              numCols,
              offset,
              true_type() /* bAsRowVector */,
              false_type());
Z
zhangjinchao01 已提交
1643 1644 1645
}

DEFINE_MATRIX_BINARY_OP(DotDiv, a /= b);
L
liaogang 已提交
1646
template <class T>
Z
zhangjinchao01 已提交
1647 1648 1649 1650
void BaseMatrixT<T>::divRowVector(BaseMatrixT& b) {
  MatrixOffset offset(0, 0, 0, 0);
  int numRows = height_;
  int numCols = width_;
L
liaogang 已提交
1651 1652 1653 1654 1655 1656 1657
  applyBinary(binary::DotDiv<T>(),
              b,
              numRows,
              numCols,
              offset,
              true_type() /* bAsRowVector */,
              false_type());
Z
zhangjinchao01 已提交
1658 1659
}

L
liaogang 已提交
1660
template <class T>
1661 1662 1663 1664
void BaseMatrixT<T>::mulColVector(BaseMatrixT& b) {
  MatrixOffset offset(0, 0, 0, 0);
  int numRows = height_;
  int numCols = width_;
L
liaogang 已提交
1665 1666 1667 1668 1669 1670 1671
  applyBinary(binary::DotMul<T>(),
              b,
              numRows,
              numCols,
              offset,
              false_type(),
              true_type() /* bAsColVector */);
1672 1673
}

L
liaogang 已提交
1674
template <class T>
1675 1676 1677 1678
void BaseMatrixT<T>::divColVector(BaseMatrixT& b) {
  MatrixOffset offset(0, 0, 0, 0);
  int numRows = height_;
  int numCols = width_;
L
liaogang 已提交
1679 1680 1681 1682 1683 1684 1685
  applyBinary(binary::DotDiv<T>(),
              b,
              numRows,
              numCols,
              offset,
              false_type(),
              true_type() /* bAsColVector */);
1686 1687
}

L
liaogang 已提交
1688
template <>
Z
zhangjinchao01 已提交
1689 1690 1691
template <class Agg>
int BaseMatrixT<real>::applyRow(Agg agg, BaseMatrixT& b) {
  MatrixOffset offset(0, 0, 0, 0, 0, 0);
Y
Yu Yang 已提交
1692 1693
  size_t numRows = b.height_;
  size_t numCols = b.width_;
X
xuwei06 已提交
1694 1695
  CHECK_EQ(height_, numRows);
  CHECK_EQ(width_, 1UL);
L
liaogang 已提交
1696 1697 1698 1699 1700 1701 1702 1703 1704
  aggregate(agg,
            base::unary::identity(),
            base::binary::second(),
            b,
            numRows,
            numCols,
            offset,
            false_type(),
            true_type() /*aAsColVector*/);
Z
zhangjinchao01 已提交
1705 1706 1707 1708

  return 0;
}

L
liaogang 已提交
1709
template <>
Z
zhangjinchao01 已提交
1710 1711 1712
template <class Agg, class Saver>
int BaseMatrixT<real>::applyRow(Agg agg, Saver sv, BaseMatrixT& b) {
  MatrixOffset offset(0, 0, 0, 0, 0, 0);
Y
Yu Yang 已提交
1713 1714
  size_t numRows = b.height_;
  size_t numCols = b.width_;
X
xuwei06 已提交
1715 1716
  CHECK_EQ(height_, numRows);
  CHECK_EQ(width_, 1UL);
L
liaogang 已提交
1717 1718 1719 1720 1721 1722 1723 1724 1725
  aggregate(agg,
            base::unary::identity(),
            sv,
            b,
            numRows,
            numCols,
            offset,
            false_type(),
            true_type() /*aAsColVector*/);
Z
zhangjinchao01 已提交
1726 1727 1728 1729

  return 0;
}

L
liaogang 已提交
1730
template <>
X
xuwei06 已提交
1731
template <class Agg>
L
liaogang 已提交
1732 1733 1734 1735
int BaseMatrixT<real>::applyRow(Agg agg,
                                real scaleDest,
                                real scaleAgg,
                                BaseMatrixT& b) {
X
xuwei06 已提交
1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746
  if (scaleDest != 0) {
    applyRow(agg, base::binary::add2(scaleDest, scaleAgg), b);
  } else {
    applyRow(agg, base::binary::second(), b);
    if (scaleAgg != 1) {
      mulScalar(scaleAgg);
    }
  }
  return 0;
}

L
liaogang 已提交
1747
template <>
X
xuwei06 已提交
1748
template <class Agg, class Op, class Saver>
L
liaogang 已提交
1749 1750
int BaseMatrixT<real>::applyRow(
    Agg agg, Op op, Saver sv, BaseMatrixT& b, BaseMatrixT& c) {
X
xuwei06 已提交
1751
  MatrixOffset offset(0, 0, 0, 0, 0, 0);
Y
Yu Yang 已提交
1752 1753
  size_t numRows = b.height_;
  size_t numCols = b.width_;
X
xuwei06 已提交
1754 1755 1756 1757
  CHECK_EQ(height_, numRows);
  CHECK_EQ(width_, 1UL);
  CHECK_EQ(c.height_, numRows);
  CHECK_EQ(c.width_, numCols);
L
liaogang 已提交
1758 1759 1760 1761 1762 1763 1764 1765 1766 1767
  aggregate(agg,
            op,
            sv,
            b,
            c,
            numRows,
            numCols,
            offset,
            false_type(),
            true_type() /*aAsColVector*/);
X
xuwei06 已提交
1768 1769 1770
  return 0;
}

L
liaogang 已提交
1771
template <>
X
xuwei06 已提交
1772
template <class Agg, class Op>
L
liaogang 已提交
1773 1774 1775 1776 1777 1778
int BaseMatrixT<real>::applyRow(Agg agg,
                                Op op,
                                real scaleDest,
                                real scaleAgg,
                                BaseMatrixT& b,
                                BaseMatrixT& c) {
X
xuwei06 已提交
1779 1780 1781 1782 1783 1784 1785 1786 1787 1788 1789
  if (scaleDest != 0) {
    applyRow(agg, op, base::binary::add2(scaleDest, scaleAgg), b, c);
  } else {
    applyRow(agg, op, base::binary::second(), b, c);
    if (scaleAgg != 1) {
      mulScalar(scaleAgg);
    }
  }
  return 0;
}

L
liaogang 已提交
1790
template <>
Z
zhangjinchao01 已提交
1791 1792 1793
template <class Agg>
int BaseMatrixT<real>::applyCol(Agg agg, BaseMatrixT& b) {
  MatrixOffset offset(0, 0, 0, 0, 0, 0);
Y
Yu Yang 已提交
1794 1795
  size_t numRows = b.height_;
  size_t numCols = b.width_;
X
xuwei06 已提交
1796 1797
  CHECK_EQ(width_, numCols);
  CHECK_EQ(height_, 1UL);
L
liaogang 已提交
1798 1799 1800 1801 1802 1803 1804 1805 1806
  aggregate(agg,
            base::unary::identity(),
            base::binary::second(),
            b,
            numRows,
            numCols,
            offset,
            true_type() /*aAsRowVector*/,
            false_type());
Z
zhangjinchao01 已提交
1807 1808 1809 1810

  return 0;
}

L
liaogang 已提交
1811
template <>
Z
zhangjinchao01 已提交
1812 1813 1814
template <class Agg, class Saver>
int BaseMatrixT<real>::applyCol(Agg agg, Saver sv, BaseMatrixT& b) {
  MatrixOffset offset(0, 0, 0, 0, 0, 0);
Y
Yu Yang 已提交
1815 1816
  size_t numRows = b.height_;
  size_t numCols = b.width_;
X
xuwei06 已提交
1817 1818
  CHECK_EQ(width_, numCols);
  CHECK_EQ(height_, 1UL);
L
liaogang 已提交
1819 1820 1821 1822 1823 1824 1825 1826 1827
  aggregate(agg,
            base::unary::identity(),
            sv,
            b,
            numRows,
            numCols,
            offset,
            true_type() /*aAsRowVector*/,
            false_type());
Z
zhangjinchao01 已提交
1828 1829 1830 1831

  return 0;
}

L
liaogang 已提交
1832
template <>
X
xuwei06 已提交
1833
template <class Agg>
L
liaogang 已提交
1834 1835 1836 1837
int BaseMatrixT<real>::applyCol(Agg agg,
                                real scaleDest,
                                real scaleAgg,
                                BaseMatrixT& b) {
X
xuwei06 已提交
1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848
  if (scaleDest != 0) {
    applyCol(agg, base::binary::add2(scaleDest, scaleAgg), b);
  } else {
    applyCol(agg, base::binary::second(), b);
    if (scaleAgg != 1) {
      mulScalar(scaleAgg);
    }
  }
  return 0;
}

L
liaogang 已提交
1849
template <>
X
xuwei06 已提交
1850
void BaseMatrixT<real>::sumRows(BaseMatrixT& b, real scaleSum, real scaleDest) {
X
xuwei06 已提交
1851
  applyRow(aggregate::sum(), scaleDest, scaleSum, b);
Z
zhangjinchao01 已提交
1852 1853
}

L
liaogang 已提交
1854
template <>
Z
zhangjinchao01 已提交
1855 1856 1857 1858
void BaseMatrixT<real>::maxRows(BaseMatrixT& b) {
  applyRow(aggregate::max(), b);
}

L
liaogang 已提交
1859
template <>
Z
zhangjinchao01 已提交
1860 1861 1862 1863
void BaseMatrixT<real>::minRows(BaseMatrixT& b) {
  applyRow(aggregate::min(), b);
}

L
liaogang 已提交
1864
template <>
Z
zhangjinchao01 已提交
1865 1866 1867 1868
void BaseMatrixT<real>::maxCols(BaseMatrixT& b) {
  applyCol(aggregate::max(), b);
}

L
liaogang 已提交
1869
template <>
Z
zhangjinchao01 已提交
1870 1871 1872 1873
void BaseMatrixT<real>::minCols(BaseMatrixT& b) {
  applyCol(aggregate::min(), b);
}

L
liaogang 已提交
1874
template <>
X
xuwei06 已提交
1875
void BaseMatrixT<real>::sumCols(BaseMatrixT& b, real scaleSum, real scaleDest) {
X
xuwei06 已提交
1876
  applyCol(aggregate::sum(), scaleDest, scaleSum, b);
Z
zhangjinchao01 已提交
1877 1878
}

L
liaogang 已提交
1879 1880 1881 1882 1883 1884 1885
template <>
void BaseMatrixT<real>::sumOfSquaredDiffs(BaseMatrixT& b,
                                          BaseMatrixT& c,
                                          real scaleSum,
                                          real scaleDest) {
  applyRow(
      aggregate::sum(), base::binary::squaredDiff(), scaleDest, scaleSum, b, c);
X
xuwei06 已提交
1886 1887
}

L
liaogang 已提交
1888 1889 1890 1891 1892 1893
template <>
void BaseMatrixT<real>::sumOfProducts(BaseMatrixT& b,
                                      BaseMatrixT& c,
                                      real scaleSum,
                                      real scaleDest) {
  applyRow(aggregate::sum(), base::binary::mul(), scaleDest, scaleSum, b, c);
Z
zhangjinchao01 已提交
1894 1895 1896 1897 1898
}

template class BaseMatrixT<real>;
template class BaseMatrixT<int>;
}  // namespace paddle