elemwise.cpp 43.9 KB
Newer Older
1 2 3
#include "./erfinv.h"
#include "megbrain/opr/basic_arith.h"
#include "megbrain/opr/io.h"
M
Megvii Engine Team 已提交
4 5 6
#include "megbrain/opr/tensor_manip.h"
#include "megbrain/test/autocheck.h"
#include "megbrain/test/helper.h"
7 8

#include <algorithm>
M
Megvii Engine Team 已提交
9
#include <cmath>
10 11 12 13

using namespace mgb;

namespace {
M
Megvii Engine Team 已提交
14
using Mode = opr::Elemwise::Mode;
15

M
Megvii Engine Team 已提交
16 17 18
using InputGenerator = Maybe<thin_function<void(HostTensorND&)>>;
// msvc would check for callable of None, so we use this to replace None
const InputGenerator NONE_INPUT_GEN;
19

M
Megvii Engine Team 已提交
20
std::unordered_set<Mode, enumhash> tested_mode;
21

M
Megvii Engine Team 已提交
22 23 24 25
/* ======================= opr special impls ======================= */
float do_mod(float a, float b) {
    return std::fmod(a, b);
}
26

M
Megvii Engine Team 已提交
27
int do_mod(int a, int b) {
28
    return (a % b + b) % b;
M
Megvii Engine Team 已提交
29
}
30

31 32 33 34 35 36 37 38 39 40 41 42 43
float do_floor_div(float a, float b) {
    return std::floor(a / b);
}

int do_floor_div(int a, int b) {
    if ((a ^ b) < 0) {
        const auto quot = a / b;
        const auto rem = a % b;
        return rem ? quot - 1 : quot;
    }
    return a / b;
}

M
Megvii Engine Team 已提交
44 45 46
float do_erfinv(float x) {
    return erfinvf(x);
}
47

M
Megvii Engine Team 已提交
48 49 50
float do_erfcinv(float x) {
    return erfcinvf(x);
}
51

M
Megvii Engine Team 已提交
52 53 54
float do_h_swish(float x) {
    return x * fmaxf(fminf(x + 3.f, 6.f), 0.f) / 6.f;
}
55

M
Megvii Engine Team 已提交
56 57 58
float do_h_swish_grad(float x, float y) {
    return x < -3.f ? 0.f : (x > 3.f ? y : (2.f * x + 3.f) / 6.f * y);
}
59

M
Megvii Engine Team 已提交
60 61 62 63
template <typename T>
T do_log_sum_exp(T a, T b) {
    return std::log(std::exp(a) + std::exp(b));
}
64

M
Megvii Engine Team 已提交
65 66 67
float do_fast_tanh(float x) {
    return x * (27.f + x * x) / (27.f + 9.f * x * x);
}
68

M
Megvii Engine Team 已提交
69 70 71 72 73
float do_fast_tanh_grad(float x, float y) {
    float x_pow2 = x * x;
    float deno = 3.f + x_pow2;
    return ((-48.f * x_pow2) / deno + 27.f + x_pow2) / (deno * 9.f) * y;
}
74

M
Megvii Engine Team 已提交
75 76 77 78
float do_fuse_add_h_swish(float x, float y) {
    float z = x + y;
    return z * fmaxf(fminf(z + 3.f, 6.f), 0.f) / 6.f;
}
79

80 81 82 83 84 85 86 87
float do_softplus_grad(float x, float y) {
    float logg = -y * expf(-fabs(x)) / (1.f + expf(-fabs(x)));
    float grad0 = x > 0.f ? logg : -logg;
    float relux = x < 0.f ? 0.f : x;
    float grad1 = relux > 0.f ? y : 0.f;
    return grad0 + grad1;
}

M
Megvii Engine Team 已提交
88 89 90 91 92 93 94 95 96 97
template <typename T>
T do_shl(T, T);  // undefined
template <typename T>
T do_shr(T, T);  // undefined
int do_shl(int x, int y) {
    return x << y;
}
int do_shr(int x, int y) {
    return x >> y;
}
98

M
Megvii Engine Team 已提交
99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123
template <typename T>
struct MulType {};
template <>
struct MulType<int8_t> {
    typedef int16_t type;
};
template <>
struct MulType<int16_t> {
    typedef int32_t type;
};
template <>
struct MulType<int32_t> {
    typedef int64_t type;
};
template <>
struct MulType<uint8_t> {
    typedef uint16_t type;
};

template <typename T>
T rounding_shift_right_upward(T x, int k) {
    T mask = (T(1) << k) - 1;
    T threshold = mask >> 1;
    return (x >> k) + ((x & mask) > threshold);
}
124

M
Megvii Engine Team 已提交
125 126 127 128 129
template <typename T>
T do_round_mulh_saturate(T a, T b) {
    MEGDNN_STATIC_ASSERT(
            std::numeric_limits<T>::digits <= 32,
            "Portable RMULH is not supported for integer "
130
            "types larger than 32 bits.")
M
Megvii Engine Team 已提交
131 132
    MEGDNN_STATIC_ASSERT(
            std::numeric_limits<T>::is_integer,
133
            "Input types should be integer for RMULH")
M
Megvii Engine Team 已提交
134 135 136 137 138 139 140 141 142 143 144
    bool overflow = a == b && a == DTypeTrait<T>::min();
    // TODO: This really should be
    // rounding_shift_right_away_from_zero, but we haven't yet found a fast
    // way to implement it on ARM NEON. For now, we just try to align with
    // NEON's VQRDMULH and hope that it does not harm our NN badly.
    return overflow
                 ? DTypeTrait<T>::max()
                 : static_cast<T>(rounding_shift_right_upward(
                           typename MulType<T>::type(a) * typename MulType<T>::type(b),
                           std::numeric_limits<T>::digits));
}
145

M
Megvii Engine Team 已提交
146 147 148 149 150
float do_gelu_grad(float x, float y) {
    float phi = 1.f / sqrtf(2.0 * M_PI) * expf(-0.5f * x * x);
    float normcdf_v = 0.5f * (1.f + erff(x / sqrtf(2.f)));
    return y * (normcdf_v + x * phi);
}
151

M
Megvii Engine Team 已提交
152
/* ======================= basic framework ======================= */
153

M
Megvii Engine Team 已提交
154 155 156 157
template <typename ctype, bool stable_sign = false>
void gen_nozero(HostTensorND& dest) {
    static RNGxorshf rng{next_rand_seed()};
    auto ptr = dest.template ptr<ctype>();
158

M
Megvii Engine Team 已提交
159 160 161 162 163 164
    if (DTypeTrait<ctype>::category == DTypeCategory::FLOAT) {
        for (size_t i = 0, it = dest.shape().total_nr_elems(); i < it; ++i) {
            auto v = rng() / (rng.max() + 1.0) * 3 - 1.5;
            bool vsign = v > 0;
            if (stable_sign) {
                vsign = i % 2;
165
            }
M
Megvii Engine Team 已提交
166 167
            v = std::abs(v) + 0.1;
            ptr[i] = vsign ? v : -v;
168
        }
M
Megvii Engine Team 已提交
169 170 171 172
    } else {
        for (size_t i = 0, it = dest.shape().total_nr_elems(); i < it; ++i) {
            ctype v = rng() / (rng.max() + 1.0) * 65536 - 32767, vsat = i % 2 * 2 - 1;
            ptr[i] = v == 0 ? vsat : v;
173
        }
M
Megvii Engine Team 已提交
174 175
    }
}
176

M
Megvii Engine Team 已提交
177 178 179
template <class Trait>
struct CheckerConfig {
    static constexpr bool enable_binary_inp_swap() { return true; }
180

M
Megvii Engine Team 已提交
181 182 183 184
    static constexpr bool allow_inp_grad(size_t idx) {
        MGB_MARK_USED_VAR(idx);
        return true;
    }
185

M
Megvii Engine Team 已提交
186 187 188 189 190
    template <typename ctype>
    static InputGenerator get_inp_gen(size_t idx) {
        MGB_MARK_USED_VAR(idx);
        return NONE_INPUT_GEN;
    }
191

M
Megvii Engine Team 已提交
192 193 194 195
    template <class Opt>
    static void update_opt(Opt& opt) {
        opt.numdiff_eps = 1e-2;
    }
196

M
Megvii Engine Team 已提交
197 198 199
    template <class Checker>
    static void update_checker(Checker& checker) {
        MGB_MARK_USED_VAR(checker);
200
    }
M
Megvii Engine Team 已提交
201 202 203 204 205 206 207 208 209 210 211
};

template <typename ctype>
InputGenerator get_inp_gen_f32_range(float low, float high) {
    mgb_assert(std::is_same<ctype MGB_COMMA dt_float32>::value && high - low >= 0.1);
    auto gen = [low, high](HostTensorND& dest) {
        HostTensorGenerator<dtype::Float32, RandomDistribution::UNIFORM> gen{low, high};
        dest = *gen(dest.shape());
    };
    return gen;
}
212

M
Megvii Engine Team 已提交
213 214 215 216 217 218 219 220 221 222 223 224 225
#define DEF_TRAIT(_mode, _expr)                                                      \
    struct _mode {                                                                   \
        static constexpr size_t ARITY = _CUR_ARITY;                                  \
        static constexpr Mode MODE = Mode::_mode;                                    \
        static constexpr bool ALLOW_INT = _ALLOW_INT;                                \
        static constexpr bool ALLOW_FLOAT = _ALLOW_FLOAT;                            \
        static constexpr bool ALLOW_BOOL = _ALLOW_BOOL;                              \
        static constexpr const char* NAME = #_mode;                                  \
        template <typename ctype>                                                    \
        static inline ctype apply(std::array<const ctype*, ARITY> inp, size_t idx) { \
            _EXPAND_PARAMS;                                                          \
            return _expr;                                                            \
        }                                                                            \
226 227 228 229
    };

#include "./elemwise_binary_trait_def.inl"
#include "./elemwise_ternary_trait_def.inl"
M
Megvii Engine Team 已提交
230
#include "./elemwise_unary_trait_def.inl"
231 232 233

#undef DEF_TRAIT

M
Megvii Engine Team 已提交
234 235 236 237
//! ensure nonzero value on some specific input
template <size_t nozero_idx, bool large_eps = true>
struct NoZeroCheckerConfig : public CheckerConfig<void> {
    static constexpr bool enable_binary_inp_swap() { return false; }
238

M
Megvii Engine Team 已提交
239 240 241 242 243 244
    template <typename ctype>
    static InputGenerator get_inp_gen(size_t idx) {
        if (idx != nozero_idx)
            return NONE_INPUT_GEN;
        return gen_nozero<ctype>;
    }
245

M
Megvii Engine Team 已提交
246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359
    template <class Opt>
    static void update_opt(Opt& opt) {
        if (large_eps)
            opt.numdiff_eps_single_inp[nozero_idx] = 0.05;
    }
};
struct NoGradCheckerConfig : public CheckerConfig<void> {
    static constexpr bool allow_inp_grad(size_t) { return false; }
};

/* ======================= unary config ======================= */
template <>
struct CheckerConfig<RELU> : public NoZeroCheckerConfig<0> {};
template <>
struct CheckerConfig<ABS> : public NoZeroCheckerConfig<0> {};
template <>
struct CheckerConfig<CEIL> : public NoGradCheckerConfig {};
template <>
struct CheckerConfig<FLOOR> : public NoGradCheckerConfig {};
template <>
struct CheckerConfig<ROUND> : public NoGradCheckerConfig {};
template <>
struct CheckerConfig<LOG> : public CheckerConfig<void> {
    template <typename ctype>
    static InputGenerator get_inp_gen(size_t) {
        return get_inp_gen_f32_range<ctype>(0.1, 4);
    }
    template <class Opt>
    static void update_opt(Opt& opt) {
        opt.numdiff_eps = 1e-2;
        opt.numdiff_max_err = 0.1;
    }
};
template <>
struct CheckerConfig<LOG1P> : public CheckerConfig<void> {
    template <typename ctype>
    static InputGenerator get_inp_gen(size_t) {
        return get_inp_gen_f32_range<ctype>(-0.2, 0.2);
    }
};
template <>
struct CheckerConfig<ACOS> : public CheckerConfig<void> {
    template <typename ctype>
    static InputGenerator get_inp_gen(size_t) {
        return get_inp_gen_f32_range<ctype>(-0.95, 0.95);
    }
    template <class Opt>
    static void update_opt(Opt& opt) {
        opt.numdiff_eps = 2e-3;
        opt.numdiff_max_err = 4e-3;
    }
};
template <>
struct CheckerConfig<ASIN> : public CheckerConfig<ACOS> {};
template <>
struct CheckerConfig<TANH> : public CheckerConfig<void> {
    template <typename ctype>
    static InputGenerator get_inp_gen(size_t) {
        return get_inp_gen_f32_range<ctype>(-5, 5);
    }
    template <class Opt>
    static void update_opt(Opt& opt) {
        opt.numdiff_eps = 2e-2;
    }
};
template <>
struct CheckerConfig<SIGMOID_GRAD> : public CheckerConfig<void> {
    template <class Opt>
    static void update_opt(Opt& opt) {
        opt.numdiff_eps = 2e-2;
    }
};
template <>
struct CheckerConfig<ERF> : public CheckerConfig<void> {
    template <class Opt>
    static void update_opt(Opt& opt) {
        opt.numdiff_eps = 2e-2;
    }
};
template <>
struct CheckerConfig<ERFINV> : public NoGradCheckerConfig {
    template <typename ctype>
    static InputGenerator get_inp_gen(size_t) {
        return get_inp_gen_f32_range<ctype>(-1, 1);
    }
    template <class Opt>
    static void update_opt(Opt& opt) {
        opt.numdiff_eps = 2e-2;
    }
};
template <>
struct CheckerConfig<ERFC> : public CheckerConfig<void> {
    template <class Opt>
    static void update_opt(Opt& opt) {
        opt.numdiff_eps = 2e-2;
    }
};
template <>
struct CheckerConfig<ERFCINV> : public NoGradCheckerConfig {
    template <typename ctype>
    static InputGenerator get_inp_gen(size_t) {
        return get_inp_gen_f32_range<ctype>(0, 2);
    }
    template <class Opt>
    static void update_opt(Opt& opt) {
        opt.numdiff_eps = 2e-2;
    }
};

template <>
struct CheckerConfig<H_SWISH> : public CheckerConfig<void> {};
template <>
struct CheckerConfig<H_SWISH_GRAD> : public NoGradCheckerConfig {};

M
Megvii Engine Team 已提交
360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452
template <>
struct CheckerConfig<TAN> : public NoGradCheckerConfig {
    template <typename ctype>
    static InputGenerator get_inp_gen(size_t) {
        return get_inp_gen_f32_range<ctype>(-1.2, 1.2);
    }
};
template <>
struct CheckerConfig<SINH> : public CheckerConfig<void> {
    template <typename ctype>
    static InputGenerator get_inp_gen(size_t) {
        return get_inp_gen_f32_range<ctype>(-5, 5);
    }
    template <class Opt>
    static void update_opt(Opt& opt) {
        opt.numdiff_eps = 1e-2;
        opt.numdiff_max_err = 0.1;
    }
};
template <>
struct CheckerConfig<COSH> : public CheckerConfig<SINH> {};
template <>
struct CheckerConfig<ASINH> : public CheckerConfig<void> {
    template <class Opt>
    static void update_opt(Opt& opt) {
        opt.numdiff_eps = 1e-2;
        opt.numdiff_max_err = 0.1;
    }
};
template <>
struct CheckerConfig<ACOSH> : public CheckerConfig<ASINH> {
    template <typename ctype>
    static InputGenerator get_inp_gen(size_t) {
        return get_inp_gen_f32_range<ctype>(1.05, 5);
    }
};
template <>
struct CheckerConfig<ATANH> : public CheckerConfig<ASINH> {
    template <typename ctype>
    static InputGenerator get_inp_gen(size_t) {
        return get_inp_gen_f32_range<ctype>(-0.95, 0.95);
    }
};
template <>
struct CheckerConfig<SOFTPLUS> : public CheckerConfig<void> {};
template <>
struct CheckerConfig<LOGSIGMOID> : public CheckerConfig<void> {};
template <>
struct CheckerConfig<SQUARE> : public CheckerConfig<void> {};
template <>
struct CheckerConfig<SQRT> : public CheckerConfig<void> {
    template <typename ctype>
    static InputGenerator get_inp_gen(size_t) {
        return get_inp_gen_f32_range<ctype>(0.05, 5);
    }
    template <class Opt>
    static void update_opt(Opt& opt) {
        opt.numdiff_eps = 1e-2;
        opt.numdiff_max_err = 0.1;
    }
};
template <>
struct CheckerConfig<RELU6> : public CheckerConfig<void> {
    template <typename ctype, class Checker>
    static void do_update_checker(Checker& checker) {
        auto icoord = [](const typename Checker::NumInpArray& inp) {
            auto p0 = inp[0]->template ptr<ctype>();
            for (size_t i = 0, it = inp[0]->shape().total_nr_elems(); i < it; ++i) {
                if (std::abs(p0[i]) < 1) {
                    p0[i] += 2;
                } else if (std::abs(p0[i] - 6) < 1) {
                    p0[i] += 2;
                }
            }
        };
        checker.set_input_coordinator(icoord);
    }
    template <class Checker>
    static void update_checker(Checker& checker) {
        using ctype = typename Checker::ctype;
        return do_update_checker<ctype>(checker);
    }
};
template <>
struct CheckerConfig<HSIGMOID> : public CheckerConfig<void> {
    template <typename ctype>
    static InputGenerator get_inp_gen(size_t) {
        return get_inp_gen_f32_range<ctype>(-2.95, 2.95);
    }
};
template <>
struct CheckerConfig<SIGN> : public NoZeroCheckerConfig<0> {};

M
Megvii Engine Team 已提交
453 454 455 456 457 458 459 460
/* ======================= binary config ======================= */
template <bool for_mod>
struct BinaryInputMinGap : public CheckerConfig<void> {
    template <typename ctype, class Checker>
    static void do_update_checker(Checker& checker) {
        auto icoord = [](const typename Checker::NumInpArray& inp) {
            static const ctype GAP{for_mod ? 0.01f : 0.1f};
            if (DTypeTrait<ctype>::category != DTypeCategory::FLOAT)
461
                return;
M
Megvii Engine Team 已提交
462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479
            auto p0 = inp[0]->template ptr<ctype>(), p1 = inp[1]->template ptr<ctype>();
            for (size_t i = 0, it = inp[0]->shape().total_nr_elems(); i < it; ++i) {
                if (for_mod) {
                    auto p1v = std::abs(p1[i]), mod = std::fmod(p0[i], p1v);
                    mod += mod < 0 ? p1v : 0;
                    if (mod < GAP || mod > p1v - GAP) {
                        mgb_assert(p1v > GAP * 4);
                        ctype m0, m1;
                        do {
                            p0[i] += GAP;
                            m0 = std::fmod(p0[i] - GAP, p1[i]);
                            m1 = std::fmod(p0[i] + GAP, p1[i]);
                        } while (std::abs(m1 - m0) > GAP * 2 + 1e-3);
                    }
                } else {
                    if (std::abs(p0[i] - p1[i]) < GAP) {
                        p1[i] += p0[i] < p1[i] ? GAP : -GAP;
                    }
480 481 482
                }
            }
        };
M
Megvii Engine Team 已提交
483
        checker.set_input_coordinator(icoord);
484 485
    }

M
Megvii Engine Team 已提交
486 487 488 489 490 491 492 493 494 495 496 497
    template <class Checker>
    static void update_checker(Checker& checker) {
        using ctype = typename Checker::ctype;
        if (std::is_integral<ctype>::value)
            return;
        if (std::is_same<ctype, dt_float16>::value)
            return do_update_checker<dt_float16>(checker);
        if (std::is_same<ctype, dt_float32>::value)
            return do_update_checker<dt_float32>(checker);
        mgb_assert(0);
    }
};
498

M
Megvii Engine Team 已提交
499 500
struct BinaryEQInput : public CheckerConfig<void> {
    static constexpr bool allow_inp_grad(size_t idx) { return idx >= 2; }
501

M
Megvii Engine Team 已提交
502 503 504 505 506 507 508 509 510 511 512
    template <class Checker>
    static void update_checker(Checker& checker) {
        using ctype = typename Checker::ctype;
        auto icoord = [](const typename Checker::NumInpArray& inp) {
            if (DTypeTrait<ctype>::category != DTypeCategory::FLOAT)
                return;
            auto p0 = inp[0]->template ptr<ctype>(), p1 = inp[1]->template ptr<ctype>();
            RNGxorshf rng{next_rand_seed()};
            for (size_t i = 0, it = inp[0]->shape().total_nr_elems(); i < it; ++i) {
                p0[i] = rng() % 3 == 0 ? p1[i] : p0[i];
            }
513
        };
M
Megvii Engine Team 已提交
514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539
        checker.set_input_coordinator(icoord);
    }
};

struct BinaryPlaneNoPiInput : public CheckerConfig<void> {
    template <class Checker>
    static void update_checker(Checker& checker) {
        using ctype = typename Checker::ctype;
        auto icoord = [](const typename Checker::NumInpArray& inp) {
            if (DTypeTrait<ctype>::category != DTypeCategory::FLOAT)
                return;
            auto p0 = inp[0]->template ptr<ctype>(), p1 = inp[1]->template ptr<ctype>();
            RNGxorshf rng{next_rand_seed()};
            auto maxv = rng.max() + 1.0;
            for (size_t i = 0, it = inp[0]->shape().total_nr_elems(); i < it; ++i) {
                //! To be numerical stable, r cannot be too small
                auto r = rng() / maxv * 2 + 0.5;  //! radious
                //! Avoid pi value due to periodicity
                //! Numerical diff will be wrong there
                //! Range [-pi+eps, pi-eps]
                auto t = rng() / maxv * 3.1 * 2 - 3.1;  //! angle
                //! First input is y in space
                p0[i] = r * std::sin(t);
                //! Second input is x in space
                p1[i] = r * std::cos(t);
            }
540
        };
M
Megvii Engine Team 已提交
541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594
        checker.set_input_coordinator(icoord);
    }
    static constexpr bool enable_binary_inp_swap() { return false; }
};
template <>
struct CheckerConfig<ATAN2> : public BinaryPlaneNoPiInput {
    template <class Opt>
    static void update_opt(Opt& opt) {
        opt.numdiff_eps = 1e-3;
        opt.numdiff_max_err = 0.02;
    }
};

template <>
struct CheckerConfig<ABS_GRAD> : public NoZeroCheckerConfig<0> {};
template <>
struct CheckerConfig<FLOOR_DIV> : public NoZeroCheckerConfig<1, false> {
    static constexpr bool allow_inp_grad(size_t) { return false; }
};
template <>
struct CheckerConfig<TRUE_DIV> : public NoZeroCheckerConfig<1, false> {
    template <class Opt>
    static void update_opt(Opt& opt) {
        opt.numdiff_eps = 1e-2;
        opt.numdiff_max_err = 0.1;
    }
};
template <>
struct CheckerConfig<EQ> : public BinaryEQInput {};
template <>
struct CheckerConfig<LEQ> : public NoGradCheckerConfig {};
template <>
struct CheckerConfig<LT> : public NoGradCheckerConfig {};
template <>
struct CheckerConfig<FUSE_ADD_H_SWISH> : public CheckerConfig<void> {};
template <>
struct CheckerConfig<SWITCH_GT0> : public NoZeroCheckerConfig<0> {};
template <>
struct CheckerConfig<POW> : public CheckerConfig<void> {
    static constexpr bool enable_binary_inp_swap() { return false; }
    template <class Opt>
    static void update_opt(Opt& opt) {
        opt.numdiff_eps = 1e-2;
        opt.numdiff_max_err = 0.06;
    }
    template <typename ctype>
    static InputGenerator get_inp_gen(size_t idx) {
        auto func = [](HostTensorND& dest) {
            dest = *HostTensorGenerator<typename DTypeTrait<ctype>::dtype>{}(
                    dest.shape());
            auto ptr = dest.ptr<ctype>();
            for (size_t i = 0, t = dest.shape().total_nr_elems(); i < t; ++i) {
                ptr[i] = std::abs(ptr[i]) + 0.1;
            }
M
Megvii Engine Team 已提交
595
        };
M
Megvii Engine Team 已提交
596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614
        if (idx == 0)
            return func;
        return NONE_INPUT_GEN;
    }
};
template <>
struct CheckerConfig<MAX> : public BinaryInputMinGap<false> {};
template <>
struct CheckerConfig<MIN> : public BinaryInputMinGap<false> {};
template <>
struct CheckerConfig<MOD> : public NoZeroCheckerConfig<1, false>,
                            public BinaryInputMinGap<true> {
    using NoZeroCheckerConfig<1, false>::get_inp_gen;
    using NoZeroCheckerConfig<1, false>::enable_binary_inp_swap;
    using BinaryInputMinGap<true>::update_checker;

    template <class Opt>
    static void update_opt(Opt& opt) {
        opt.numdiff_eps = 0.003;
615 616
    }

M
Megvii Engine Team 已提交
617 618
    static constexpr bool allow_inp_grad(size_t idx) { return idx == 0; }
};
619

M
Megvii Engine Team 已提交
620 621 622
template <>
struct CheckerConfig<SHL> : public CheckerConfig<void> {
    static constexpr bool enable_binary_inp_swap() { return false; }
623

M
Megvii Engine Team 已提交
624
    static constexpr bool allow_inp_grad(size_t idx) { return false; }
625

M
Megvii Engine Team 已提交
626 627 628 629 630 631 632 633 634 635 636 637 638
    template <typename ctype>
    static InputGenerator get_inp_gen(size_t);
};
template <>
struct CheckerConfig<SHR> : public CheckerConfig<SHL> {};

template <>
InputGenerator CheckerConfig<SHL>::get_inp_gen<int>(size_t idx) {
    if (!idx)
        return NONE_INPUT_GEN;
    auto gen = [](HostTensorND& dest) {
        HostTensorGenerator<dtype::Int32, RandomDistribution::UNIFORM> gen{0, 32};
        dest = *gen(dest.shape());
639
    };
M
Megvii Engine Team 已提交
640 641
    return gen;
}
642

M
Megvii Engine Team 已提交
643 644 645 646 647 648 649 650 651 652 653 654 655 656 657
template <>
struct CheckerConfig<FUSE_ADD_RELU> : public CheckerConfig<void> {
    template <typename ctype>
    static InputGenerator get_inp_gen(size_t) {
        return gen_nozero<ctype, true>;
    }
};

template <>
struct CheckerConfig<FAST_TANH> : public CheckerConfig<void> {
    template <typename ctype>
    static InputGenerator get_inp_gen(size_t) {
        return get_inp_gen_f32_range<ctype>(0.1, 5);
    }
};
658

M
Megvii Engine Team 已提交
659 660 661 662 663 664 665 666 667 668 669 670
template <>
struct CheckerConfig<FAST_TANH_GRAD> : public CheckerConfig<FAST_TANH> {
    static constexpr bool allow_inp_grad(size_t idx) {
        MGB_MARK_USED_VAR(idx);
        return false;
    }
};

template <>
struct CheckerConfig<SILU_GRAD> : public NoGradCheckerConfig {};
template <>
struct CheckerConfig<GELU_GRAD> : public NoGradCheckerConfig {};
M
Megvii Engine Team 已提交
671 672
template <>
struct CheckerConfig<PRELU> : public NoZeroCheckerConfig<0> {};
M
Megvii Engine Team 已提交
673

M
Megvii Engine Team 已提交
674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700
template <>
struct CheckerConfig<ASINH_GRAD> : public NoGradCheckerConfig {};
template <>
struct CheckerConfig<ACOSH_GRAD> : public NoGradCheckerConfig {
    template <typename ctype>
    static InputGenerator get_inp_gen(size_t) {
        return get_inp_gen_f32_range<ctype>(1.05, 5);
    }
};
template <>
struct CheckerConfig<ATANH_GRAD> : public NoGradCheckerConfig {
    template <typename ctype>
    static InputGenerator get_inp_gen(size_t) {
        return get_inp_gen_f32_range<ctype>(-0.95, 0.95);
    }
};
template <>
struct CheckerConfig<RELU6_GRAD> : public NoGradCheckerConfig {};
template <>
struct CheckerConfig<SOFTPLUS_GRAD> : public NoGradCheckerConfig {};
template <>
struct CheckerConfig<HSIGMOID_GRAD> : public NoGradCheckerConfig {
    template <typename ctype>
    static InputGenerator get_inp_gen(size_t) {
        return get_inp_gen_f32_range<ctype>(-2.95, 2.95);
    }
};
M
Megvii Engine Team 已提交
701 702 703
/* ======================= ternary config ======================= */
template <>
struct CheckerConfig<COND_LEQ_MOV> : public BinaryInputMinGap<false> {};
M
Megvii Engine Team 已提交
704

705 706
template <>
struct CheckerConfig<COND_LT_MOV> : public BinaryInputMinGap<false> {};
M
Megvii Engine Team 已提交
707

M
Megvii Engine Team 已提交
708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753
template <>
struct CheckerConfig<PRELU_GRAD> : public NoGradCheckerConfig {};

template <>
struct CheckerConfig<CLIP> : public CheckerConfig<void> {
    template <typename ctype, class Checker>
    static void do_update_checker(Checker& checker) {
        auto icoord = [](const typename Checker::NumInpArray& inp) {
            auto p0 = inp[0]->template ptr<ctype>(), p1 = inp[1]->template ptr<ctype>(),
                 p2 = inp[2]->template ptr<ctype>();
            for (size_t i = 0, it = inp[0]->shape().total_nr_elems(); i < it; ++i) {
                if (p1[i] > p2[i]) {
                    std::swap(p1[i], p2[i]);
                }
                if (p1[i] + 1 > p2[i]) {
                    p2[i] = p1[i] + 1;
                }
                if (std::abs(p1[i] - p0[i]) < 1) {
                    if (p1[i] < p0[i])
                        p0[i] += 1;
                    else
                        p0[i] -= 1;
                }
                if (std::abs(p2[i] - p0[i]) < 1) {
                    if (p2[i] < p0[i])
                        p0[i] += 1;
                    else
                        p0[i] -= 1;
                }
            }
        };
        checker.set_input_coordinator(icoord);
    }

    template <class Checker>
    static void update_checker(Checker& checker) {
        using ctype = typename Checker::ctype;
        return do_update_checker<ctype>(checker);
    }

    template <class Opt>
    static void update_opt(Opt& opt) {
        opt.numdiff_eps = 1e-3;
        opt.numdiff_max_err = 0.1;
    }
};
M
Megvii Engine Team 已提交
754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808
/* ======================= test runner ======================= */
namespace detail {
template <typename dtype, class Trait>
struct enable_for_dtype_impl;

template <class Trait>
struct enable_for_dtype_impl<dtype::Float32, Trait> {
    static constexpr bool value = Trait::ALLOW_FLOAT;
};
template <>
struct enable_for_dtype_impl<dtype::Float32, void> {
    static constexpr bool value = false;
};
template <class Trait>
struct enable_for_dtype_impl<dtype::Int32, Trait> {
    static constexpr bool value = Trait::ALLOW_INT;
};
template <>
struct enable_for_dtype_impl<dtype::Int32, void> {
    static constexpr bool value = false;
};
template <class Trait>
struct enable_for_dtype_impl<dtype::Bool, Trait> {
    static constexpr bool value = Trait::ALLOW_BOOL;
};
}  // namespace detail

//! whether to enable test for specific dtype and Trait
template <typename dtype, class Trait>
constexpr bool enable_for_dtype = detail::enable_for_dtype_impl<dtype, Trait>::value;

template <typename Trait, typename dtype, bool enable = enable_for_dtype<dtype, Trait>>
struct TestRunner;

template <typename Trait, typename dtype>
struct TestRunner<Trait, dtype, true> {
    static void run();
};
template <typename Trait, typename dtype>
struct TestRunner<Trait, dtype, false> {
    static void run() {}
};
template <typename dtype>
struct TestRunner<void, dtype, false> {
    static void run() {}
};

template <typename Trait>
class TestOprBasicArithUnaryElemwise : public ::testing::Test {};
template <typename Trait>
class TestOprBasicArithBinaryElemwise : public ::testing::Test {};
template <typename Trait>
class TestOprBasicArithTernaryElemwise : public ::testing::Test {};

typedef ::testing::Types<
809 810 811
#define DEF_TRAIT(_mode, _expr) _mode,
#include "./elemwise_unary_trait_def.inl"
#undef DEF_TRAIT
M
Megvii Engine Team 已提交
812 813 814
        void  // extra void to consume last comma
        >
        UnaryTraitTypes;
815
TYPED_TEST_SUITE(TestOprBasicArithUnaryElemwise, UnaryTraitTypes);
816

M
Megvii Engine Team 已提交
817
typedef ::testing::Types<
818 819 820
#define DEF_TRAIT(_mode, _expr) _mode,
#include "./elemwise_binary_trait_def.inl"
#undef DEF_TRAIT
M
Megvii Engine Team 已提交
821 822 823
        void  // extra void to consume last comma
        >
        BinaryTraitTypes;
824
TYPED_TEST_SUITE(TestOprBasicArithBinaryElemwise, BinaryTraitTypes);
825

M
Megvii Engine Team 已提交
826
typedef ::testing::Types<
827 828 829
#define DEF_TRAIT(_mode, _expr) _mode,
#include "./elemwise_ternary_trait_def.inl"
#undef DEF_TRAIT
M
Megvii Engine Team 已提交
830 831 832
        void  // extra void to consume last comma
        >
        TernaryTraitTypes;
833
TYPED_TEST_SUITE(TestOprBasicArithTernaryElemwise, TernaryTraitTypes);
834

M
Megvii Engine Team 已提交
835
}  // anonymous namespace
836

M
Megvii Engine Team 已提交
837
template <typename Trait, typename dtype>
838 839 840 841 842 843 844 845 846 847 848 849
void TestRunner<Trait, dtype, true>::run() {
    {
        Mode mode = Trait::MODE;
        // copy to temporary var to avoid undefined reference when linking
        tested_mode.insert(mode);
    }

    using ctype = typename DTypeTrait<dtype>::ctype;

    HostTensorGenerator<> gen;
    using Config = CheckerConfig<Trait>;

M
Megvii Engine Team 已提交
850 851 852 853
    static constexpr bool TEST_REV_INP =
            Trait::ARITY == 2 &&
            Config::allow_inp_grad(0) == Config::allow_inp_grad(1) &&
            Config::enable_binary_inp_swap();
854
    using Checker = AutoOprChecker<Trait::ARITY, TEST_REV_INP + 1, dtype>;
M
Megvii Engine Team 已提交
855
    auto make_graph = [&](const typename Checker::SymInpArray& inputs) {
856 857 858 859 860 861 862 863 864 865
        typename Checker::SymOutArray out;
        SymbolVarArray vinp(inputs.begin(), inputs.end());
        out[0] = opr::Elemwise::make(vinp, Trait::MODE);
        if (TEST_REV_INP) {
            std::swap(vinp[0], vinp[1]);
            out[1] = opr::Elemwise::make(vinp, Trait::MODE);
        }
        return out;
    };

M
Megvii Engine Team 已提交
866 867
    auto fwd = [&](typename Checker::NumOutArray& dest,
                   typename Checker::NumInpArray inp) {
868 869 870 871 872
        dest[0].resize(inp[0]->shape());
        if (TEST_REV_INP)
            dest[1].resize(inp[0]->shape());

        std::array<const ctype*, Trait::ARITY> iptr;
M
Megvii Engine Team 已提交
873
        for (size_t i = 0; i < Trait::ARITY; ++i)
874 875 876 877 878
            iptr[i] = inp[i]->template ptr<ctype>();

        size_t sz = dest[0].shape().total_nr_elems();

        ctype* optr = dest[0].template ptr<ctype>();
M
Megvii Engine Team 已提交
879
        for (size_t i = 0; i < sz; ++i)
880 881 882 883 884
            optr[i] = Trait::apply(iptr, i);

        if (TEST_REV_INP) {
            std::swap(iptr[0], iptr[1]);
            ctype* optr = dest[1].template ptr<ctype>();
M
Megvii Engine Team 已提交
885
            for (size_t i = 0; i < sz; ++i)
886 887 888 889 890 891
                optr[i] = Trait::apply(iptr, i);
        }
    };

    Checker checker{make_graph, fwd};
    checker.set_extra_err_msg(ssprintf("mode=%s", Trait::NAME));
M
Megvii Engine Team 已提交
892
    for (size_t i = 0; i < Trait::ARITY; ++i) {
893 894 895 896 897 898 899 900
        auto func = Config::template get_inp_gen<ctype>(i);
        if (func.valid())
            checker.set_input_generator(i, func.val());

        checker.set_input_allow_grad(i, Config::allow_inp_grad(i));
    }

    TensorShape shapes[] = {{1}, {23, 3}, {666}};
M
Megvii Engine Team 已提交
901 902 903 904
    if (Trait::ARITY == 4) {
        checker.disable_graph_opt();
        shapes[0] = {32};
    }
905 906 907
    typename Checker::RunOptions opt;
    Config::update_opt(opt);
    Config::update_checker(checker);
M
Megvii Engine Team 已提交
908
    for (auto&& ishp : shapes) {
909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939
        typename Checker::ShapeInpArray inp;
        std::fill(inp.begin(), inp.end(), ishp);
        checker.run(inp, opt);
    }
}

TYPED_TEST(TestOprBasicArithUnaryElemwise, Int32) {
    TestRunner<TypeParam, dtype::Int32>::run();
}
TYPED_TEST(TestOprBasicArithBinaryElemwise, Int32) {
    TestRunner<TypeParam, dtype::Int32>::run();
}
TYPED_TEST(TestOprBasicArithTernaryElemwise, Int32) {
    TestRunner<TypeParam, dtype::Int32>::run();
}

TYPED_TEST(TestOprBasicArithUnaryElemwise, Float32) {
    set_rand_seed(19931102);
    TestRunner<TypeParam, dtype::Float32>::run();
}
TYPED_TEST(TestOprBasicArithBinaryElemwise, Float32) {
    set_rand_seed(19931150);
    TestRunner<TypeParam, dtype::Float32>::run();
}
TYPED_TEST(TestOprBasicArithTernaryElemwise, Float32) {
    set_rand_seed(19931102);
    TestRunner<TypeParam, dtype::Float32>::run();
}

TEST(TestOprBasicArithElemwise, CheckAllModeTested) {
    size_t nr_member = opr::Elemwise::Param::MODE_NR_MEMBER;
940 941
    ASSERT_EQ(nr_member, tested_mode.size() + 7);
    // Not using TestRunner: NOT, AND, OR, XOR, NEQ, ISNAN, ISINF
942
}
M
Megvii Engine Team 已提交
943 944 945 946 947 948 949 950 951 952 953 954 955
#define TEST_OPR_BASIC_ARITH_UNARY_BOOL(_mode, _op)                  \
    TEST(TestOprBasicArithElemwise, _mode) {                         \
        HostTensorGenerator<dtype::Bool> gen;                        \
        auto host_x = gen({2, 1});                                   \
        auto ptr = host_x->ptr<dt_bool>();                           \
        for (size_t i = 0; i < 2; ++i) {                             \
            ptr[i] = (i & 1);                                        \
        }                                                            \
        auto graph = ComputingGraph::make();                         \
        using Mode = opr::Elemwise::Mode;                            \
        auto x = opr::Host2DeviceCopy::make(*graph, host_x),         \
             y = opr::Elemwise::make({x}, Mode::_mode);              \
        HostTensorND host_y;                                         \
M
Megvii Engine Team 已提交
956
        auto func = graph->compile({make_callback_copy(y, host_y)}); \
M
Megvii Engine Team 已提交
957 958 959 960 961 962 963
        func->execute();                                             \
        ASSERT_EQ(TensorShape({2, 1}), host_y.shape());              \
        auto ptry = host_y.ptr<dt_bool>();                           \
        for (int i = 0; i < 2; i++) {                                \
            ASSERT_EQ(_op ptr[i], ptry[i]);                          \
        }                                                            \
    }
M
Megvii Engine Team 已提交
964 965 966

TEST_OPR_BASIC_ARITH_UNARY_BOOL(NOT, !)

M
Megvii Engine Team 已提交
967 968 969 970
#define TEST_OPR_BASIC_ARITH_BINARY_BOOL(_mode, _op)                         \
    TEST(TestOprBasicArithElemwise, _mode) {                                 \
        HostTensorGenerator<dtype::Bool> gen;                                \
        auto host_x1 = gen({2, 2}), host_x2 = gen({2, 2});                   \
M
Megvii Engine Team 已提交
971
        auto ptr1 = host_x1->ptr<dt_bool>(), ptr2 = host_x2->ptr<dt_bool>(); \
M
Megvii Engine Team 已提交
972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989
        for (size_t i = 0; i < 4; ++i) {                                     \
            ptr1[i] = (i < 2);                                               \
            ptr2[i] = (i & 1);                                               \
        }                                                                    \
        auto graph = ComputingGraph::make();                                 \
        using Mode = opr::Elemwise::Mode;                                    \
        auto x1 = opr::Host2DeviceCopy::make(*graph, host_x1),               \
             x2 = opr::Host2DeviceCopy::make(*graph, host_x2),               \
             y = opr::Elemwise::make({x1, x2}, Mode::_mode);                 \
        HostTensorND host_y;                                                 \
        auto func = graph->compile({make_callback_copy(y, host_y)});         \
        func->execute();                                                     \
        ASSERT_EQ(TensorShape({2, 2}), host_y.shape());                      \
        auto ptry = host_y.ptr<dt_bool>();                                   \
        for (int i = 0; i < 4; i++) {                                        \
            ASSERT_EQ(ptr1[i] _op ptr2[i], ptry[i]);                         \
        }                                                                    \
    }
M
Megvii Engine Team 已提交
990 991 992 993

TEST_OPR_BASIC_ARITH_BINARY_BOOL(AND, &&)
TEST_OPR_BASIC_ARITH_BINARY_BOOL(OR, ||)
TEST_OPR_BASIC_ARITH_BINARY_BOOL(XOR, ^)
994 995 996
TEST_OPR_BASIC_ARITH_BINARY_BOOL(LT, <)
TEST_OPR_BASIC_ARITH_BINARY_BOOL(LEQ, <=)
TEST_OPR_BASIC_ARITH_BINARY_BOOL(EQ, ==)
997 998 999 1000

TEST(TestOprBasicArithElemwise, FuseMulAdd3Shapes) {
    using Checker = AutoOprChecker<3, 1>;

M
Megvii Engine Team 已提交
1001 1002 1003
    opr::Elemwise* opr;
    auto make_graph =
            [&](const typename Checker::SymInpArray& i) -> Checker::SymOutArray {
1004 1005 1006 1007 1008 1009
        i[0].node()->owner_graph()->options().graph_opt_level = 0;
        auto ret = opr::Elemwise::make(i, Mode::FUSE_MUL_ADD3);
        opr = &ret.node()->owner_opr()->cast_final_safe<opr::Elemwise>();
        return {ret};
    };

M
Megvii Engine Team 已提交
1010 1011
    auto fwd = [&](typename Checker::NumOutArray& dest,
                   typename Checker::NumInpArray inp) {
1012 1013 1014 1015 1016 1017 1018 1019 1020 1021
        auto graph = ComputingGraph::make();
        graph->options().graph_opt_level = false;
        auto i = [&](size_t idx) {
            return opr::Host2DeviceCopy::make(*graph, inp[idx]);
        };
        auto ans = i(0) * i(1) + i(2);
        graph->compile({make_callback_copy(ans, dest[0])})->execute();
    };

    Checker checker{make_graph, fwd};
M
Megvii Engine Team 已提交
1022 1023
    checker.run({TensorShape{1, 2}, {2, 1}, {1, 2}})
            .run({TensorShape{1, 2}, {2, 1}, {1}});
1024 1025 1026 1027 1028 1029 1030 1031
    ASSERT_FALSE(opr->fuse_badlayout_warn_printed());
    checker.run({TensorShape{1, 1, 4}, {1, 3, 1}, {2, 1, 1}});
    ASSERT_TRUE(opr->fuse_badlayout_warn_printed());
}

TEST(TestOprBasicArithElemwise, FuseMulAdd4Shapes) {
    using Checker = AutoOprChecker<4, 1>;

M
Megvii Engine Team 已提交
1032 1033 1034
    opr::Elemwise* opr;
    auto make_graph =
            [&](const typename Checker::SymInpArray& i) -> Checker::SymOutArray {
1035 1036 1037 1038 1039 1040
        i[0].node()->owner_graph()->options().graph_opt_level = 0;
        auto ret = opr::Elemwise::make(i, Mode::FUSE_MUL_ADD4);
        opr = &ret.node()->owner_opr()->cast_final_safe<opr::Elemwise>();
        return {ret};
    };

M
Megvii Engine Team 已提交
1041 1042
    auto fwd = [&](typename Checker::NumOutArray& dest,
                   typename Checker::NumInpArray inp) {
1043 1044 1045 1046 1047 1048 1049 1050 1051 1052
        auto graph = ComputingGraph::make();
        graph->options().graph_opt_level = false;
        auto i = [&](size_t idx) {
            return opr::Host2DeviceCopy::make(*graph, inp[idx]);
        };
        auto ans = i(0) * i(1) + i(2) * i(3);
        graph->compile({make_callback_copy(ans, dest[0])})->execute();
    };

    Checker checker{make_graph, fwd};
M
Megvii Engine Team 已提交
1053 1054 1055 1056 1057
    checker.run({TensorShape{1, 32}, {1, 32}, {1, 32}, {1, 32}})
            .run({TensorShape{1, 1, 1, 1, 1, 32},
                  {1, 1, 1, 1, 1, 32},
                  {1, 1, 1, 1, 1, 32},
                  {1, 1, 1, 1, 1, 32}});
1058
    ASSERT_FALSE(opr->fuse_badlayout_warn_printed());
M
Megvii Engine Team 已提交
1059
    checker.run({TensorShape{1, 32}, {32, 1}, {32, 32}, {32, 32}});
1060 1061 1062 1063 1064 1065 1066 1067
    ASSERT_TRUE(opr->fuse_badlayout_warn_printed());
}

TEST(TestOprBasicArithElemwise, WritableFwdForSameStorage) {
    HostTensorGenerator<> gen;

    auto run = [&](int idx_val, bool should_overwrite) {
        auto host_x = gen({100});
M
Megvii Engine Team 已提交
1068
        auto make_y = [&](ComputingGraph& graph) {
1069 1070 1071
            using S = opr::Subtensor;
            auto x = opr::Host2DeviceCopy::make_no_fwd(graph, host_x),
                 idx = x.make_scalar(idx_val),
M
Megvii Engine Team 已提交
1072 1073 1074
                 sub0 = S::make(x, {S::AxisIndexer::make_interval(0, None, idx, None)}),
                 sub1 = S::make(
                         x, {S::AxisIndexer::make_interval(0, -idx, None, None)}),
1075 1076 1077 1078
                 y = sub0 + sub1;
            auto chk_overwrite = [sub0, sub1, y]() {
                auto py = y.node()->prev_dev_ptr();
                return sub0.node()->prev_dev_ptr() == py ||
M
Megvii Engine Team 已提交
1079
                       sub1.node()->prev_dev_ptr() == py;
1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109
            };
            return std::make_pair(y, chk_overwrite);
        };
        auto g0 = ComputingGraph::make(), g1 = ComputingGraph::make();
        g1->options().seq_opt.enable_mem_plan_opt = false;
        auto y0 = make_y(*g0), y1 = make_y(*g1);
        HostTensorND host_y0, host_y1;
        auto f0 = g0->compile({make_callback_copy(y0.first, host_y0)}),
             f1 = g1->compile({make_callback_copy(y1.first, host_y1)});

        f0->execute();
        f1->execute();
        ASSERT_EQ(host_y1.shape(), TensorShape{static_cast<size_t>(idx_val)});
        MGB_ASSERT_TENSOR_EQ(host_y1, host_y0);
        ASSERT_EQ(should_overwrite, y0.second());
        ASSERT_FALSE(y1.second());
    };

    run(10, true);
    run(90, false);
}

TEST(TestOprBasicArithElemwise, NonContigInput) {
    HostTensorGenerator<> gen;

    auto graph = ComputingGraph::make();
    constexpr size_t SIZE = 100;
    auto host_x = gen({SIZE});
    using S = opr::Subtensor;
    auto x = opr::Host2DeviceCopy::make(*graph, host_x),
M
Megvii Engine Team 已提交
1110 1111
         xsub = S::make(
                 x, {S::AxisIndexer::make_interval(0, None, None, x.make_scalar(2))}),
1112 1113 1114 1115 1116 1117 1118 1119
         y = xsub + x.make_scalar(1.f);
    HostTensorND host_y;
    auto func = graph->compile({make_callback_copy(y, host_y)});
    func->execute();
    ASSERT_FALSE(xsub.node()->dev_tensor().layout().is_contiguous());

    ASSERT_EQ(SIZE / 2, host_y.layout().total_nr_elems());
    auto px = host_x->ptr<float>(), py = host_y.ptr<float>();
M
Megvii Engine Team 已提交
1120
    for (size_t i = 0; i < SIZE / 2; ++i) {
1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141
        MGB_ASSERT_FLOAT_EQ(px[i * 2] + 1, py[i]);
    }
}

TEST(TestOprBasicArithElemwise, CommutableDedup) {
    auto cn = CompNode::load("xpux");
    auto graph = ComputingGraph::make();
    auto host_x = std::make_shared<HostTensorND>(cn, TensorShape{100}),
         host_y = std::make_shared<HostTensorND>(cn, TensorShape{100});
    auto x = opr::Host2DeviceCopy::make(*graph, host_x),
         y = opr::Host2DeviceCopy::make(*graph, host_y);
    auto mk = [](Mode mode, SymbolVar x, SymbolVar y) {
        return opr::Elemwise::make({x, y}, mode);
    };
#define CHK(_a, _b) ASSERT_EQ((_a).node(), (_b).node())
    CHK(x + y, y + x);
    CHK(x * y, y * x);
    CHK(mk(Mode::EQ, x, y), mk(Mode::EQ, y, x));
    CHK(mk(Mode::MIN, x, y), mk(Mode::MIN, y, x));
    CHK(mk(Mode::MAX, x, y), mk(Mode::MAX, y, x));
    CHK(mk(Mode::LOG_SUM_EXP, x, y), mk(Mode::LOG_SUM_EXP, y, x));
M
Megvii Engine Team 已提交
1142
    CHK(x<y, y> x);
1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155
#undef CHK
    ASSERT_NE((x - y).node(), (y - x).node());
}

TEST(TestLayoutUtil, CollectiveCollapse) {
    using namespace opr;
    auto shp2layout = [](const TensorShapeArray& tshps) {
        TensorLayoutArray tlayouts(tshps.size());
        for (size_t i = 0; i < tshps.size(); i++) {
            tlayouts[i] = TensorLayout(tshps[i], dtype::Float32());
        }
        return tlayouts;
    };
M
Megvii Engine Team 已提交
1156
    auto check = [](const TensorLayoutArray& res, const TensorLayoutArray& std) {
1157 1158 1159 1160
        for (size_t i = 0; i < res.size(); i++) {
            ASSERT_EQ(std[i], res[i]);
        }
    };
M
Megvii Engine Team 已提交
1161
    TensorShapeArray tshps1 = {{3, 3}, {3, 3}, {3, 3}};
1162 1163 1164 1165 1166 1167
    auto cc_res1 = Elemwise::collective_collapse(shp2layout(tshps1));
    TensorShapeArray std_res1 = {{9}, {9}, {9}};
    check(cc_res1, shp2layout(std_res1));

    TensorShapeArray tshps2 = {{3, 3, 3}, {1, 3, 3}};
    auto cc_res2 = Elemwise::collective_collapse(shp2layout(tshps2));
M
Megvii Engine Team 已提交
1168
    TensorShapeArray std_res2{{3, 9}, {1, 9}};
1169 1170 1171 1172
    check(cc_res2, shp2layout(std_res2));

    TensorShapeArray tshp3 = {{3, 3, 3}, {3, 3, 1}};
    auto cc_res3 = Elemwise::collective_collapse(shp2layout(tshp3));
M
Megvii Engine Team 已提交
1173
    TensorShapeArray std_res3{{9, 3}, {9, 1}};
1174 1175 1176 1177
    check(cc_res3, shp2layout(std_res3));

    TensorShapeArray tshp4 = {{3, 3, 3, 3}, {1, 3, 3, 1}};
    auto cc_res4 = Elemwise::collective_collapse(shp2layout(tshp4));
M
Megvii Engine Team 已提交
1178
    TensorShapeArray std_res4{{3, 9, 3}, {1, 9, 1}};
1179 1180 1181
    check(cc_res4, shp2layout(std_res4));

    TensorLayoutArray inp5 = {
M
Megvii Engine Team 已提交
1182 1183
            TensorLayout(TensorShape{3, 3}, {1, 3}, dtype::Float32()),
            TensorLayout(TensorShape{3, 3}, {1, 3}, dtype::Float32())};
1184 1185 1186 1187 1188
    auto cc_res5 = Elemwise::collective_collapse(inp5);
    auto std_res5 = inp5;
    check(cc_res5, std_res5);
}

1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201
TEST(TestOprBasicArithElemwise, EmptyInputOutputUnary) {
    HostTensorGenerator<> gen;
    auto graph = ComputingGraph::make();
    auto host_x = gen({3, 0, 1, 3});
    auto x = opr::Host2DeviceCopy::make(*graph, host_x),
         y = opr::Elemwise::make(
                 {x}, opr::Elemwise::Param(opr::Elemwise::Param::Mode::RELU));
    HostTensorND host_y;
    auto func = graph->compile({make_callback_copy(y, host_y)});

    ASSERT_NO_THROW(func->execute().wait());
    ASSERT_TRUE(host_y.empty());
    ASSERT_TRUE(host_y.shape().is_empty());
1202
    MGB_ASSERT_SHAPE_EQ(host_y.shape(), TensorShape({3, 0, 1, 3}));
1203 1204 1205 1206 1207 1208
}

TEST(TestOprBasicArithElemwise, EmptyInputOutputBinary) {
    HostTensorGenerator<> gen;
    auto graph = ComputingGraph::make();
    auto host_x = gen({0, 8, 1, 7}), host_y = gen({0, 8, 1, 7});
1209

1210
    auto x = opr::Host2DeviceCopy::make(*graph, host_x),
M
Megvii Engine Team 已提交
1211
         y = opr::Host2DeviceCopy::make(*graph, host_y), z = x + y;
1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223
    HostTensorND host_z;
    auto func = graph->compile({make_callback_copy(z, host_z)});

    // Invalid broadcast
    host_y->resize({0, 9, 1, 7});
    ASSERT_ANY_THROW(func->execute().wait());

    // Broadcast to 0
    host_y->resize({1, 8, 0, 7});
    ASSERT_NO_THROW(func->execute().wait());
    ASSERT_TRUE(host_z.empty());
    ASSERT_TRUE(host_z.shape().is_empty());
1224
    MGB_ASSERT_SHAPE_EQ(host_z.shape(), TensorShape({0, 8, 0, 7}));
1225 1226 1227 1228 1229 1230

    // Broadcast to 0 (2)
    host_y->resize({2, 8, 1, 7});
    ASSERT_NO_THROW(func->execute().wait());
    ASSERT_TRUE(host_z.empty());
    ASSERT_TRUE(host_z.shape().is_empty());
1231
    MGB_ASSERT_SHAPE_EQ(host_z.shape(), TensorShape({0, 8, 1, 7}));
1232 1233 1234 1235 1236 1237 1238

    // Scalar broadcast
    z = x + x.make_scalar(1.f);
    func = graph->compile({make_callback_copy(z, host_z)});
    ASSERT_NO_THROW(func->execute().wait());
    ASSERT_TRUE(host_z.empty());
    ASSERT_TRUE(host_z.shape().is_empty());
1239
    MGB_ASSERT_SHAPE_EQ(host_z.shape(), TensorShape({0, 8, 1, 7}));
1240 1241
}

1242 1243 1244
TEST(TestOprBasicArithElemwise, PerformEmptyIO) {
    auto cn = CompNode::load("xpu0");
    HostTensorGenerator<> gen;
M
Megvii Engine Team 已提交
1245
    auto host_x1 = gen({2, 0, 3, 4}), host_x2 = gen({1});
1246 1247 1248 1249 1250 1251 1252 1253 1254 1255
    auto dev_x1 = std::make_shared<DeviceTensorND>(cn),
         dev_x2 = std::make_shared<DeviceTensorND>(cn);
    dev_x1->copy_from(*host_x1);
    dev_x2->copy_from(*host_x2);

    auto dev_y = std::make_shared<DeviceTensorND>(cn, dev_x1->dtype());
    dev_y->resize(dev_x1->shape());
    auto&& dnn_opr = opr::intl::create_megdnn_opr<megdnn::Elemwise>(cn);

    // test unary mode
M
Megvii Engine Team 已提交
1256
    for (auto mode : {Mode::NEGATE, Mode::EXP, Mode::LOG}) {
1257 1258 1259 1260 1261 1262 1263 1264
        SmallVector<DeviceTensorND> inputs = {*dev_x1};
        ASSERT_NO_THROW(opr::Elemwise::perform(mode, *dev_y, inputs, dnn_opr));
        ASSERT_TRUE(dev_y->empty());
        ASSERT_TRUE(dev_y->shape().is_empty());
        MGB_ASSERT_SHAPE_EQ(dev_y->shape(), dev_x1->shape());
    }

    // test binary mode
M
Megvii Engine Team 已提交
1265
    for (auto mode : {Mode::ADD, Mode::MUL, Mode::LT}) {
1266 1267 1268 1269 1270 1271 1272 1273
        SmallVector<DeviceTensorND> inputs = {*dev_x1, *dev_x2};
        ASSERT_NO_THROW(opr::Elemwise::perform(mode, *dev_y, inputs, dnn_opr));
        ASSERT_TRUE(dev_y->empty());
        ASSERT_TRUE(dev_y->shape().is_empty());
        MGB_ASSERT_SHAPE_EQ(dev_y->shape(), dev_x1->shape());
    }
}

1274
// vim: syntax=cpp.doxygen foldmethod=marker foldmarker=f{{{,f}}}