opr_impl.cpp 12.3 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48
/**
 * \file dnn/src/arm_common/conv_bias/opr_impl.cpp
 * MegEngine is Licensed under the Apache License, Version 2.0 (the "License")
 *
 * Copyright (c) 2014-2020 Megvii Inc. All rights reserved.
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT ARRANTIES OR CONDITIONS OF ANY KIND, either express or
 * implied.
 */

#include "src/arm_common/conv_bias/int8/algos.h"
#include "src/arm_common/conv_bias/int8x8x16/algos.h"
#include "src/arm_common/conv_bias/quint8/algos.h"

#include "src/arm_common/conv_bias/opr_impl.h"
#include "src/common/metahelper.h"
#include "src/common/utils.h"
#include "src/naive/handle.h"

#include "src/arm_common/convolution/opr_impl.h"
#include "src/arm_common/matrix_mul/opr_impl.h"
#include "src/common/opr_delegate.h"

#include "include/megdnn/oprs/nn.h"
#include "src/arm_common/conv_bias/f16/algos.h"
#include "src/arm_common/conv_bias/fp32/algos.h"
#include "src/arm_common/conv_bias/int8/stride1.h"
#include "src/arm_common/conv_bias/int8/stride2.h"
#include "src/arm_common/conv_bias/quint8/stride1.h"
#include "src/arm_common/conv_bias/quint8/stride2.h"
#include "src/arm_common/convolution/opr_impl.h"

using namespace megdnn;
using namespace arm_common;

namespace {
uint8_t arm_common_algo_type_storage;
}  // anonymous namespace

class ConvBiasImpl::AlgoPack : NonCopyableObj {
    AlgoQU8DirectStride2 qu8_direct_stride2_large_group{true};
    AlgoQU8DirectStride2 qu8_direct_stride2_small_group{false};
    AlgoQU8DirectStride1 qu8_direct_stride1_large_group{true};
    AlgoQU8DirectStride1 qu8_direct_stride1_small_group{false};
    AlgoS8DirectStride2 s8_direct_stride2_large_group{true};
    AlgoS8DirectStride2 s8_direct_stride2_small_group{false};
49
    AlgoS8DirectNCHW44 s8_direct_nchw44;
50
    AlgoS8DirectNCHWNCHW44 s8_direct_nchw_nchw44;
51 52 53 54 55 56 57 58 59 60 61 62 63 64
    AlgoS8DirectStride1 s8_direct_stride1_large_group{true};
    AlgoS8DirectStride1 s8_direct_stride1_small_group{false};
    AlgoS8ChanWiseStride1NCHW44 s8_channel_wise_stride1_nchw44;
    AlgoS8ChanWiseStride2NCHW44 s8_channel_wise_stride2_nchw44;

#if __ARM_FEATURE_DOTPROD
    AlgoDotS8DirectStride1 ds8_direct_stride1_large_group{true};
    AlgoDotS8DirectStride1 ds8_direct_stride1_small_group{false};
    AlgoDotS8DirectStride2 ds8_direct_stride2_large_group{true};
    AlgoDotS8DirectStride2 ds8_direct_stride2_small_group{false};
    AlgoDotU8DirectStride1 du8_direct_stride1_large_group{true};
    AlgoDotU8DirectStride1 du8_direct_stride1_small_group{false};
    AlgoDotU8DirectStride2 du8_direct_stride2_large_group{true};
    AlgoDotU8DirectStride2 du8_direct_stride2_small_group{false};
65 66

    AlgoDotS8Direct_NCHW44 ds8_direct_nchw44;
67
    AlgoDotS8DirectNCHWNCHW44 ds8_direct_nchw_nchw44;
68 69
#endif

70
    AlgoF32DirectNCHWNCHW44 f32_direct_stride2_nchw_nchw44;
71 72 73
    AlgoF32ChannelWiseNCHW44 f32_chanel_wise_nchw44;
    AlgoF32DirectNCHW44 f32_direct_nchw44;

74 75 76 77 78 79
    AlgoF32Direct f32_direct_large_group{true};
    AlgoF32Direct f32_direct_small_group{false};
    AlgoF32DirectStride2 f32_direct_stride2_large_group{true};
    AlgoF32DirectStride2 f32_direct_stride2_small_group{false};
    AlgoF32DirectStride1 f32_direct_stride1_large_group{true};
    AlgoF32DirectStride1 f32_direct_stride1_small_group{false};
80

81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105
    AlgoI8x8x16Direct i8x8x16_direct_large_group{true};
    AlgoI8x8x16Direct i8x8x16_direct_small_group{false};
    AlgoI8x8x16Stride2 i8x8x16_stride2_large_group{true};
    AlgoI8x8x16Stride2 i8x8x16_stride2_small_group{false};
    AlgoI8x8x16Stride2Filter2 i8x8x16_stride2_filter2;
#if __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
    AlgoF16Direct f16_direct_large_group{true};
    AlgoF16Direct f16_direct_small_group{false};
    AlgoF16DirectStride1 f16_direct_stride1_large_group{true};
    AlgoF16DirectStride1 f16_direct_stride1_small_group{false};
#endif

    SmallVector<std::unique_ptr<AlgoBase>> refhold;

public:
    AlgoPack() {
#if __ARM_FEATURE_DOTPROD
        direct_algos.emplace_back(&ds8_direct_stride1_large_group);
        direct_algos.emplace_back(&ds8_direct_stride1_small_group);
        direct_algos.emplace_back(&ds8_direct_stride2_large_group);
        direct_algos.emplace_back(&ds8_direct_stride2_small_group);
        direct_algos.emplace_back(&du8_direct_stride1_large_group);
        direct_algos.emplace_back(&du8_direct_stride1_small_group);
        direct_algos.emplace_back(&du8_direct_stride2_large_group);
        direct_algos.emplace_back(&du8_direct_stride2_small_group);
106 107

        direct_algos.emplace_back(&ds8_direct_nchw44);
108
        direct_algos.emplace_back(&ds8_direct_nchw_nchw44);
109 110 111 112 113 114 115
#endif
        direct_algos.emplace_back(&qu8_direct_stride2_large_group);
        direct_algos.emplace_back(&qu8_direct_stride2_small_group);
        direct_algos.emplace_back(&qu8_direct_stride1_large_group);
        direct_algos.emplace_back(&qu8_direct_stride1_small_group);
        direct_algos.emplace_back(&s8_direct_stride2_large_group);
        direct_algos.emplace_back(&s8_direct_stride2_small_group);
116
        direct_algos.emplace_back(&s8_direct_nchw44);
117
        direct_algos.emplace_back(&s8_direct_nchw_nchw44);
118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134
        direct_algos.emplace_back(&s8_direct_stride1_large_group);
        direct_algos.emplace_back(&s8_direct_stride1_small_group);

        direct_algos.emplace_back(&s8_channel_wise_stride1_nchw44);
        direct_algos.emplace_back(&s8_channel_wise_stride2_nchw44);

#if __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
        direct_algos.emplace_back(&f16_direct_stride1_large_group);
        direct_algos.emplace_back(&f16_direct_stride1_small_group);
        direct_algos.emplace_back(&f16_direct_large_group);
        direct_algos.emplace_back(&f16_direct_small_group);
#endif
        direct_algos.emplace_back(&i8x8x16_direct_large_group);
        direct_algos.emplace_back(&i8x8x16_direct_small_group);
        direct_algos.emplace_back(&i8x8x16_stride2_filter2);
        direct_algos.emplace_back(&i8x8x16_stride2_large_group);
        direct_algos.emplace_back(&i8x8x16_stride2_small_group);
135

136
        direct_algos.emplace_back(&f32_direct_stride2_nchw_nchw44);
137
        direct_algos.emplace_back(&f32_chanel_wise_nchw44);
138
        direct_algos.emplace_back(&f32_direct_nchw44);
139

140 141 142 143 144 145 146 147 148 149 150 151 152 153 154
        direct_algos.emplace_back(&f32_direct_stride1_large_group);
        direct_algos.emplace_back(&f32_direct_stride1_small_group);
        direct_algos.emplace_back(&f32_direct_stride2_large_group);
        direct_algos.emplace_back(&f32_direct_stride2_small_group);
        direct_algos.emplace_back(&f32_direct_large_group);
        direct_algos.emplace_back(&f32_direct_small_group);

        static CpuOprDelegationStorage<2> storage;
        auto matmul_opr = storage.get<MatrixMul, 0>();
        auto&& matmul_algos =
                static_cast<arm_common::MatrixMulImpl*>(matmul_opr)
                        ->algo_pack();
        for (auto&& algo : matmul_algos) {
            if (algo->type() == nullptr)
                continue;
155
            for (uint32_t tile_size : {16, 8, 24, 32}) {
156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175
                refhold.emplace_back(new AlgoFP32WinogradF23_4x4(
                        static_cast<fallback::MatrixMulImpl::AlgoBase*>(algo),
                        tile_size));
                winograd_algos.emplace_back(refhold.back().get());
                refhold.emplace_back(new AlgoFP32WinogradF63(
                        static_cast<fallback::MatrixMulImpl::AlgoBase*>(algo),
                        tile_size));
                winograd_algos.emplace_back(refhold.back().get());
                refhold.emplace_back(new AlgoFP32WinogradF63_4x4(
                        static_cast<fallback::MatrixMulImpl::AlgoBase*>(algo),
                        tile_size));
                winograd_algos.emplace_back(refhold.back().get());
                refhold.emplace_back(new AlgoFP32WinogradF54(
                        static_cast<fallback::MatrixMulImpl::AlgoBase*>(algo),
                        tile_size));
                winograd_algos.emplace_back(refhold.back().get());
                refhold.emplace_back(new AlgoFP32WinogradF45(
                        static_cast<fallback::MatrixMulImpl::AlgoBase*>(algo),
                        tile_size));
                winograd_algos.emplace_back(refhold.back().get());
176 177 178 179 180 181 182 183
                refhold.emplace_back(new AlgoFP32WinogradF23_4x4_NCHW44(
                        static_cast<fallback::MatrixMulImpl::AlgoBase*>(algo),
                        tile_size));
                winograd_algos.emplace_back(refhold.back().get());
                refhold.emplace_back(new AlgoFP32WinogradF63_4x4_NCHW44(
                        static_cast<fallback::MatrixMulImpl::AlgoBase*>(algo),
                        tile_size));
                winograd_algos.emplace_back(refhold.back().get());
184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205
#if __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
                refhold.emplace_back(new AlgoFP16WinogradF23(
                        static_cast<fallback::MatrixMulImpl::AlgoBase*>(algo),
                        tile_size));
                winograd_algos.emplace_back(refhold.back().get());
                refhold.emplace_back(new AlgoFP16WinogradF45(
                        static_cast<fallback::MatrixMulImpl::AlgoBase*>(algo),
                        tile_size));
                winograd_algos.emplace_back(refhold.back().get());
                refhold.emplace_back(new AlgoFP16WinogradF63(
                        static_cast<fallback::MatrixMulImpl::AlgoBase*>(algo),
                        tile_size));
                winograd_algos.emplace_back(refhold.back().get());
                refhold.emplace_back(new AlgoFP16WinogradF23_8x8(
                        static_cast<fallback::MatrixMulImpl::AlgoBase*>(algo),
                        tile_size));
                winograd_algos.emplace_back(refhold.back().get());
#endif
                refhold.emplace_back(new AlgoS8WinogradF23_8x8(
                        static_cast<fallback::MatrixMulImpl::AlgoBase*>(algo),
                        tile_size));
                winograd_algos.emplace_back(refhold.back().get());
206 207 208 209 210 211 212 213
                refhold.emplace_back(new AlgoS8CF32WinogradF23_4x4_NCHW44(
                        static_cast<fallback::MatrixMulImpl::AlgoBase*>(algo),
                        tile_size));
                winograd_algos.emplace_back(refhold.back().get());
                refhold.emplace_back(new AlgoS8WinogradF23_8x8_NCHW44(
                        static_cast<fallback::MatrixMulImpl::AlgoBase*>(algo),
                        tile_size));
                winograd_algos.emplace_back(refhold.back().get());
214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266
            }
        }
    }
    SmallVector<AlgoBase*> direct_algos;
    SmallVector<AlgoBase*> winograd_algos;
};

SmallVector<ConvBiasImpl::AlgoBase*> ConvBiasImpl::algo_pack() {
    static AlgoPack sl_algo_pack;
    auto&& algos = fallback::ConvBiasImpl::algo_pack();
    algos.insert(algos.begin(), sl_algo_pack.direct_algos.begin(),
                 sl_algo_pack.direct_algos.end());
    algos.insert(algos.end(), sl_algo_pack.winograd_algos.begin(),
                 sl_algo_pack.winograd_algos.end());
    return std::move(algos);
}

void* const ConvBiasImpl::sm_arm_common_algo_type =
        &arm_common_algo_type_storage;

bool ConvBiasImpl::is_matmul_quantized_prefer(
        const ConvBiasImpl::NCBKernSizeParam& param) {
    // fallback::ConvBiasImpl::NCBKernParam conv_ncb_param;
    fallback::ConvBiasImpl::NCBKernSizeParam conv_ncb_param(
            param, 0, param::MatrixMul::Format::DEFAULT, {}, 0,
            BiasMode::NO_BIAS, param::ConvBias::NonlineMode::IDENTITY);
    conv_ncb_param.dst_type = param.bias_type;
    conv_ncb_param.filter_meta.group = 1;

    bool conv_direct_unusable = false;
    if (param.dst_type.enumv() == DTypeEnum::QuantizedS8 ||
        param.dst_type.enumv() == DTypeEnum::QuantizedS32) {
        conv_direct_unusable =
                !arm_common::direct_int8_stride1::can_conv_direct_stride1_int8(
                        conv_ncb_param) &&
                !arm_common::direct_int8_stride2::can_conv_direct_stride2_int8(
                        conv_ncb_param);
    } else if (param.dst_type.enumv() == DTypeEnum::Quantized8Asymm) {
        conv_direct_unusable =
                !arm_common::direct_quint8_stride1::
                        can_conv_direct_stride1_quint8(conv_ncb_param) &&
                !arm_common::direct_quint8_stride2::
                        can_conv_direct_stride2_quint8(conv_ncb_param);
    }
    return conv_direct_unusable;
}

const char* ConvBiasImpl::get_algorithm_set_name() const {
    // arm common version 0
    return "AC0";
}

// vim: syntax=cpp.doxygen