quantize_kernel.cpp 6.9 KB
Newer Older
T
Tian 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14
/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */

15
#ifdef QUANT_OP
16

17
#include "operators/kernel/quantize_kernel.h"
18
#include <cmath>
19
#include "operators/math/quantize.h"
T
Tian 已提交
20

21 22
#if defined(__ARM_NEON__) || defined(__ARM_NEON)
#include <arm_neon.h>
H
hjchen2 已提交
23
#endif
H
Refine  
hjchen2 已提交
24

25 26 27
namespace paddle_mobile {
namespace operators {

H
hjchen2 已提交
28
#if defined(__ARM_NEON__) || defined(__ARM_NEON)
29
#ifndef __aarch64__
H
hjchen2 已提交
30
inline float32_t vmaxvq_f32(float32x4_t r) {
31 32 33 34 35
  float32x2_t v = vmax_f32(vget_high_f32(r), vget_low_f32(r));
  return vget_lane_f32(vpmax_f32(v, v), 0);
}
#endif

36
template <RoundType R>
37 38
inline void QuantizeOffline(const Tensor *input, const float scale,
                            const float max_abs, Tensor *output) {
39
  const float *x = input->data<const float>();
H
hjchen2 已提交
40
  int8_t *y = output->mutable_data<int8_t>();
41
  size_t remain = input->numel();
H
hjchen2 已提交
42
#if defined(__ARM_NEON__) || defined(__ARM_NEON)
43 44
  size_t loop = remain >> 4;
  remain = remain & 0xF;
45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86
  float32x4_t __scale = vdupq_n_f32(scale);
  float32x4_t __postive_max = vdupq_n_f32(max_abs);
  float32x4_t __negtive_max = vdupq_n_f32(-max_abs);
  #pragma omp parallel for
  for (size_t i = 0; i < loop; ++i) {
    const float *local_x = x + (i << 4);
    int8_t *local_y = y + (i << 4);
    float32x4_t r0 = vld1q_f32(local_x);
    float32x4_t r1 = vld1q_f32(local_x + 4);
    float32x4_t r2 = vld1q_f32(local_x + 8);
    float32x4_t r3 = vld1q_f32(local_x + 12);
    r0 = vmaxq_f32(vminq_f32(r0, __postive_max), __negtive_max);
    r1 = vmaxq_f32(vminq_f32(r1, __postive_max), __negtive_max);
    r2 = vmaxq_f32(vminq_f32(r2, __postive_max), __negtive_max);
    r3 = vmaxq_f32(vminq_f32(r3, __postive_max), __negtive_max);
    r0 = vmulq_f32(r0, __scale);
    r1 = vmulq_f32(r1, __scale);
    r2 = vmulq_f32(r2, __scale);
    r3 = vmulq_f32(r3, __scale);
    int32x4_t q0 = math::vRoundq_f32<R>(r0);
    int32x4_t q1 = math::vRoundq_f32<R>(r1);
    int32x4_t q2 = math::vRoundq_f32<R>(r2);
    int32x4_t q3 = math::vRoundq_f32<R>(r3);
    int16x4_t d0 = vmovn_s32(q0);
    int16x4_t d1 = vmovn_s32(q1);
    int16x4_t d2 = vmovn_s32(q2);
    int16x4_t d3 = vmovn_s32(q3);
    int16x8_t q5 = vcombine_s16(d0, d1);
    int16x8_t q6 = vcombine_s16(d2, d3);
    int8x8_t d5 = vmovn_s16(q5);
    int8x8_t d6 = vmovn_s16(q6);
    vst1_s8(local_y, d5);
    vst1_s8(local_y + 8, d6);
  }
  x += (loop << 4);
  y += (loop << 4);
#endif
  for (size_t i = 0; i < remain; ++i) {
    float x_temp = std::max(std::min(x[i], max_abs), -max_abs);
    y[i] = math::Round<R>(x_temp * scale);
  }
}
H
hjchen2 已提交
87

88 89 90 91 92 93 94 95 96
template <RoundType R>
inline void QuantizeOnline(const Tensor *input, const float scale,
                           Tensor *output) {
  const float *x = input->data<const float>();
  int8_t *y = output->mutable_data<int8_t>();
  size_t remain = input->numel();
#if defined(__ARM_NEON__) || defined(__ARM_NEON)
  size_t loop = remain >> 4;
  remain = remain & 0xF;
97
  float32x4_t __scale = vdupq_n_f32(scale);
H
hjchen2 已提交
98
  #pragma omp parallel for
99
  for (size_t i = 0; i < loop; ++i) {
H
hjchen2 已提交
100 101 102 103 104 105
    const float *local_x = x + (i << 4);
    int8_t *local_y = y + (i << 4);
    float32x4_t r0 = vld1q_f32(local_x);
    float32x4_t r1 = vld1q_f32(local_x + 4);
    float32x4_t r2 = vld1q_f32(local_x + 8);
    float32x4_t r3 = vld1q_f32(local_x + 12);
106 107 108 109
    r0 = vmulq_f32(r0, __scale);
    r1 = vmulq_f32(r1, __scale);
    r2 = vmulq_f32(r2, __scale);
    r3 = vmulq_f32(r3, __scale);
110 111 112 113
    int32x4_t q0 = math::vRoundq_f32<R>(r0);
    int32x4_t q1 = math::vRoundq_f32<R>(r1);
    int32x4_t q2 = math::vRoundq_f32<R>(r2);
    int32x4_t q3 = math::vRoundq_f32<R>(r3);
114 115 116 117
    int16x4_t d0 = vmovn_s32(q0);
    int16x4_t d1 = vmovn_s32(q1);
    int16x4_t d2 = vmovn_s32(q2);
    int16x4_t d3 = vmovn_s32(q3);
H
hjchen2 已提交
118 119
    int16x8_t q5 = vcombine_s16(d0, d1);
    int16x8_t q6 = vcombine_s16(d2, d3);
H
Refine  
hjchen2 已提交
120 121
    int8x8_t d5 = vmovn_s16(q5);
    int8x8_t d6 = vmovn_s16(q6);
H
hjchen2 已提交
122 123
    vst1_s8(local_y, d5);
    vst1_s8(local_y + 8, d6);
124
  }
H
hjchen2 已提交
125 126
  x += (loop << 4);
  y += (loop << 4);
127
#endif
128
  for (size_t i = 0; i < remain; ++i) {
129
    y[i] = math::Round<R>(x[i] * scale);
130 131 132
  }
}

133 134 135 136 137 138 139 140 141 142 143
template <RoundType R>
static void Quantize(const Tensor *input, const float max_abs,
                     const bool offline, Tensor *output) {
  float scale = 127.f / max_abs;
  if (offline) {
    QuantizeOffline<R>(input, scale, max_abs, output);
  } else {
    QuantizeOnline<R>(input, scale, output);
  }
}

144 145
float find_abs_max(const Tensor *input) {
  float max_abs = 0.f;
146
  const float *x = input->data<const float>();
147
  size_t remain = input->numel();
148
#if defined(__ARM_NEON__) || defined(__ARM_NEON)
149 150 151
  size_t loop = remain >> 4;
  remain = remain & 0xF;
  float32x4_t __max = {0.f, 0.f, 0.f, 0.f};
H
hjchen2 已提交
152

153 154 155 156 157 158 159 160 161 162 163 164 165
  for (size_t i = 0; i < loop; ++i, x += 16) {
    float32x4_t r0 = vld1q_f32(x);
    float32x4_t r1 = vld1q_f32(x + 4);
    float32x4_t r2 = vld1q_f32(x + 8);
    float32x4_t r3 = vld1q_f32(x + 12);
    r0 = vabsq_f32(r0);
    r1 = vabsq_f32(r1);
    r2 = vabsq_f32(r2);
    r3 = vabsq_f32(r3);
    r0 = vmaxq_f32(r0, r1);
    r1 = vmaxq_f32(r2, r3);
    r0 = vmaxq_f32(r0, r1);
    __max = vmaxq_f32(r0, __max);
166
  }
167
  max_abs = vmaxvq_f32(__max);
168
#endif
169
  for (size_t i = 0; i < remain; ++i) {
170
    max_abs = std::max(max_abs, static_cast<float>(fabs(x[i])));
H
hjchen2 已提交
171
  }
172
  return max_abs;
H
hjchen2 已提交
173
}
174

175 176 177 178 179 180 181
}  // namespace operators
}  // namespace paddle_mobile
#endif  // __ARM_NEON__

namespace paddle_mobile {
namespace operators {

182
template <>
183 184 185 186
bool QuantizeKernel<CPU, float>::Init(QuantizeParam<CPU> *param) {
  return true;
}

187
template <>
L
liuruilong 已提交
188
void QuantizeKernel<CPU, float>::Compute(const QuantizeParam<CPU> &param) {
189 190
  const LoDTensor *input = param.input_;
  LoDTensor *output = param.output_;
191
  Tensor *output_scale = param.online_scale_;
H
hjchen2 已提交
192
  float max_abs = 0.f;
193 194
  if (param.offline_) {
    max_abs = param.offline_scale_->data<float>()[0];
195 196 197
  } else {
    max_abs = find_abs_max(input);
  }
H
hjchen2 已提交
198
  max_abs = std::max(max_abs, 1e-6f);
199
  param.online_scale_->mutable_data<float>()[0] = max_abs;
200 201
  switch (param.round_type_) {
    case ROUND_NEAREST_TO_EVEN:
202
      Quantize<ROUND_NEAREST_TO_EVEN>(input, max_abs, param.offline_, output);
203 204
      break;
    case ROUND_NEAREST_TOWARDS_ZERO:
205 206
      Quantize<ROUND_NEAREST_TOWARDS_ZERO>(input, max_abs, param.offline_,
                                           output);
207 208
      break;
    case ROUND_NEAREST_AWAY_ZERO:
209
      Quantize<ROUND_NEAREST_AWAY_ZERO>(input, max_abs, param.offline_, output);
210
      break;
211 212 213 214
    default:
      LOG(kLOG_ERROR) << "round type is not supported.";
      break;
  }
215
  output->set_lod(input->lod());
216 217 218
}

}  // namespace operators
219
}  // namespace paddle_mobile
220

221
#endif  // QUANT_OP