!3726 add lite op PriorBox

Merge pull request !3726 from zhaozhenlong/lite/op/prior_box

mindspore/lite/schema/model.fbs

@@ -173,7 +173,8 @@ union PrimitiveType {
     Div,
     Where,
     OneHot,
-    Lstm
+    Lstm,
+    PriorBox
 }
 
 enum QuantType: int {

mindspore/lite/schema/ops.fbs

@@ -722,3 +722,17 @@ table OneHot {
 table Lstm{
     bidirection: bool = false;
 }
+
+table PriorBox {
+    min_sizes: [int];
+    max_sizes: [int];
+    aspect_ratios: [float];
+    variances: [float];
+    image_size_w: int;
+    image_size_h: int;
+    step_w: float;
+    step_h: float;
+    clip: bool = true;
+    flip: bool = true;
+    offset: float;
+}

mindspore/lite/src/ops/ops.cc

@@ -131,6 +131,8 @@ Primitive *Primitive::CreatePrimitive(schema::Primitive *primitive) {
       return new lite::Resize(const_cast<schema::Primitive *>(primitive));
     case schema::PrimitiveType_OneHot:
       return new lite::OneHot(const_cast<schema::Primitive *>(primitive));
+    case schema::PrimitiveType_PriorBox:
+      return new lite::PriorBox(const_cast<schema::Primitive *>(primitive));
     default:
       break;
   }

mindspore/lite/src/ops/ops.h

@@ -660,7 +660,14 @@ class StridedSlice : public Primitive {
   std::vector<int> new_axis_mask_;
   std::vector<int> shrink_axis_mask_;
 };
+
+class PriorBox : public Primitive {
+ public:
+  explicit PriorBox(schema::Primitive *primitive) : Primitive(primitive) {}
+  const schema::PriorBox *GetAttrbute() const { return this->primitive->value_as_PriorBox(); }
+  int InferShape(std::vector<tensor::Tensor *> inputs, std::vector<tensor::Tensor *> outputs) override;
+};
+
 }  // namespace lite
 }  // namespace mindspore
 #endif  // MINDSPORE_LITE_SRC_OPS_OPS_H_
-

mindspore/lite/src/ops/prior_box.cc

+/**
+ * Copyright 2019-2020 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "src/ops/ops.h"
+#include "include/errorcode.h"
+#include "utils/log_adapter.h"
+#include "src/ir/tensor.h"
+
+namespace mindspore::lite {
+namespace {
+constexpr int kPriorBoxPoints = 4;
+constexpr int kPriorBoxN = 1;
+constexpr int kPriorBoxW = 1;
+constexpr int kPriorBoxC = 2;
+}  // namespace
+
+int PriorBox::InferShape(std::vector<tensor::Tensor *> inputs_, std::vector<tensor::Tensor *> outputs_) {
+  auto param = GetAttrbute();
+  MS_ASSERT(param != nullptr);
+  std::vector<float> different_aspect_ratios{1.0f};
+  auto aspect_ratios = param->aspect_ratios();
+  MS_ASSERT(aspect_ratios != nullptr);
+  for (auto i = 0; i < aspect_ratios->size(); i++) {
+    float ratio = (*aspect_ratios)[i];
+    bool exist = std::any_of(different_aspect_ratios.begin(), different_aspect_ratios.end(), [&](float v) {
+      return abs(ratio - v) < 1e-6;
+    });
+    if (!exist) {
+      different_aspect_ratios.emplace_back(ratio);
+      if (param->flip()) {
+        different_aspect_ratios.emplace_back(1.0f / ratio);
+      }
+    }
+  }
+  int32_t num_priors_box = param->min_sizes()->size() * different_aspect_ratios.size() + param->max_sizes()->size();
+  auto input = inputs_.at(0);
+  MS_ASSERT(input != nullptr);
+  int32_t h = input->Height() * input->Width() * num_priors_box * kPriorBoxPoints;
+
+  std::vector<int> output_shape{kPriorBoxN, h, kPriorBoxW, kPriorBoxC};
+  auto output = outputs_.at(0);
+  MS_ASSERT(output != nullptr);
+
+  output->set_shape(output_shape);
+  output->set_data_type(kNumberTypeFloat32);
+  output->SetFormat(input->GetFormat());
+  return RET_OK;
+}
+}  // namespace mindspore::lite

mindspore/lite/src/populate_parameter.cc

@@ -62,6 +62,7 @@
 #include "src/runtime/kernel/arm/opclib/fp32/unsqueeze.h"
 #include "src/runtime/kernel/arm/opclib/fp32/one_hot.h"
 #include "src/runtime/kernel/arm/opclib/fp32/strided_slice.h"
+#include "src/runtime/kernel/arm/base/prior_box.h"
 
 namespace mindspore::kernel {
 FillParameter *PopulateFillParam(const lite::Primitive *primitive) {
@@ -990,6 +991,60 @@ OpParameter *PopulateAddNParam(const lite::Primitive *primitive) {
   return parameter;
 }
 
+PriorBoxParameter *PopulatePriorBoxParameter(const lite::Primitive *primitive) {
+  PriorBoxParameter *param = new (std::nothrow) PriorBoxParameter();
+  if (param == nullptr) {
+    MS_LOG(ERROR) << "new PriorBoxParameter failed.";
+    return nullptr;
+  }
+  param->op_parameter_.type_ = primitive->Type();
+  auto prior_box_param = primitive->Value()->value_as_PriorBox();
+
+  if (prior_box_param->min_sizes()->size() > PRIOR_BOX_MAX_NUM) {
+    MS_LOG(ERROR) << "PriorBox min_sizes size exceeds max num " << PRIOR_BOX_MAX_NUM << ", got "
+                  << prior_box_param->min_sizes();
+    delete (param);
+    return nullptr;
+  }
+  param->min_sizes_size = prior_box_param->min_sizes()->size();
+  if (prior_box_param->max_sizes()->size() > PRIOR_BOX_MAX_NUM) {
+    MS_LOG(ERROR) << "PriorBox max_sizes size exceeds max num " << PRIOR_BOX_MAX_NUM << ", got "
+                  << prior_box_param->max_sizes();
+    delete (param);
+    return nullptr;
+  }
+  param->max_sizes_size = prior_box_param->max_sizes()->size();
+  (void)memcpy(param->max_sizes, prior_box_param->max_sizes()->data(),
+               prior_box_param->max_sizes()->size() * sizeof(int32_t));
+  (void)memcpy(param->min_sizes, prior_box_param->min_sizes()->data(),
+               prior_box_param->min_sizes()->size() * sizeof(int32_t));
+
+  if (prior_box_param->aspect_ratios()->size() > PRIOR_BOX_MAX_NUM) {
+    MS_LOG(ERROR) << "PriorBox aspect_ratios size exceeds max num " << PRIOR_BOX_MAX_NUM << ", got "
+                  << prior_box_param->aspect_ratios();
+    delete (param);
+    return nullptr;
+  }
+  param->aspect_ratios_size = prior_box_param->aspect_ratios()->size();
+  (void)memcpy(param->aspect_ratios, prior_box_param->aspect_ratios()->data(),
+               prior_box_param->aspect_ratios()->size() * sizeof(float));
+  if (prior_box_param->variances()->size() != PRIOR_BOX_VAR_NUM) {
+    MS_LOG(ERROR) << "PriorBox variances size should be " << PRIOR_BOX_VAR_NUM << ", got "
+                  << prior_box_param->variances()->size();
+    delete (param);
+    return nullptr;
+  }
+  (void)memcpy(param->variances, prior_box_param->variances()->data(), PRIOR_BOX_VAR_NUM * sizeof(float));
+  param->flip = prior_box_param->flip();
+  param->clip = prior_box_param->clip();
+  param->offset = prior_box_param->offset();
+  param->image_size_h = prior_box_param->image_size_h();
+  param->image_size_w = prior_box_param->image_size_w();
+  param->step_h = prior_box_param->step_h();
+  param->step_w = prior_box_param->step_w();
+  return param;
+}
+
 OpParameter *PopulateParameter(const lite::Primitive *primitive) {
   MS_EXCEPTION_IF_NULL(primitive);
   auto op_type = primitive->Type();
@@ -1109,6 +1164,8 @@ OpParameter *PopulateParameter(const lite::Primitive *primitive) {
       return reinterpret_cast<OpParameter *>(PopulateOneHotParameter(primitive));
     case schema::PrimitiveType_AddN:
       return reinterpret_cast<OpParameter *>(PopulateAddNParam(primitive));
+    case schema::PrimitiveType_PriorBox:
+      return reinterpret_cast<OpParameter *>(PopulatePriorBoxParameter(primitive));
     default:
       break;
   }

mindspore/lite/src/runtime/kernel/arm/base/prior_box.cc

+/**
+ * Copyright 2020 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <vector>
+#include <cmath>
+#include "src/runtime/kernel/arm/base/prior_box.h"
+#include "schema/model_generated.h"
+#include "src/kernel_factory.h"
+#include "include/errorcode.h"
+#include "include/context.h"
+#include "src/runtime/runtime_api.h"
+
+using mindspore::lite::KernelRegistrar;
+using mindspore::lite::RET_ERROR;
+using mindspore::lite::RET_NULL_PTR;
+using mindspore::lite::RET_OK;
+using mindspore::schema::PrimitiveType_PriorBox;
+
+namespace mindspore::kernel {
+namespace {
+constexpr int kInputNum = 2;
+constexpr int kOutputNum = 1;
+}  // namespace
+int PriorBoxCPUKernel::Init() {
+  if (prior_box_param_ == nullptr) {
+    MS_LOG(ERROR) << "PriorBoxParameter nullptr";
+    return RET_NULL_PTR;
+  }
+  MS_ASSERT(inputs_.size() == kInputNum);
+  MS_ASSERT(outputs_.size() == kOutputNum);
+
+  auto ret = GeneratePriorBox();
+
+  return ret;
+}
+
+int PriorBoxCPUKernel::GeneratePriorBox() {
+  const int fmap_w = inputs_[0]->Width();
+  const int fmap_h = inputs_[0]->Height();
+
+  const int image_w = prior_box_param_->image_size_w > 0 ? prior_box_param_->image_size_w : inputs_[1]->Width();
+  const int image_h = prior_box_param_->image_size_h > 0 ? prior_box_param_->image_size_h : inputs_[1]->Height();
+
+  const float step_w =
+    prior_box_param_->step_w > 0.0f ? prior_box_param_->step_w : static_cast<float>(image_w) / fmap_w;
+  const float step_h =
+    prior_box_param_->step_h > 0.0f ? prior_box_param_->step_h : static_cast<float>(image_h) / fmap_h;
+
+  std::vector<float> different_aspect_ratios{1.0f};
+  auto aspect_ratios = prior_box_param_->aspect_ratios;
+  MS_ASSERT(aspect_ratios != nullptr);
+  for (auto i = 0; i < prior_box_param_->aspect_ratios_size; i++) {
+    float ratio = aspect_ratios[i];
+    bool exist = std::any_of(different_aspect_ratios.begin(), different_aspect_ratios.end(),
+                             [&](float v) { return abs(ratio - v) < 1e-6; });
+    if (!exist) {
+      different_aspect_ratios.emplace_back(ratio);
+      if (prior_box_param_->flip) {
+        different_aspect_ratios.emplace_back(1.0f / ratio);
+      }
+    }
+  }
+
+  for (int i = 0; i < fmap_h; i++) {
+    float cy = i + prior_box_param_->offset;
+    for (int j = 0; j < fmap_w; j++) {
+      float cx = j + prior_box_param_->offset;
+      for (auto k = 0; k < prior_box_param_->min_sizes_size; k++) {
+        float min_size = prior_box_param_->min_sizes[k];
+        output_.emplace_back((cx - min_size / step_w * 0.5f) / fmap_w);
+        output_.emplace_back((cy - min_size / step_h * 0.5f) / fmap_h);
+        output_.emplace_back((cx + min_size / step_w * 0.5f) / fmap_w);
+        output_.emplace_back((cy + min_size / step_h * 0.5f) / fmap_h);
+
+        if (prior_box_param_->max_sizes_size > 0) {
+          float max_size = prior_box_param_->max_sizes[k];
+          float prime = sqrt(min_size * max_size);
+          output_.emplace_back((cx - prime / step_w * 0.5f) / fmap_w);
+          output_.emplace_back((cy - prime / step_h * 0.5f) / fmap_h);
+          output_.emplace_back((cx + prime / step_w * 0.5f) / fmap_w);
+          output_.emplace_back((cy + prime / step_h * 0.5f) / fmap_h);
+        }
+
+        for (auto v : different_aspect_ratios) {
+          if (abs(v - 1.0f) < 1e-6) {
+            continue;
+          }
+          float as_square_root = sqrt(v);
+          float box_w = min_size * as_square_root;
+          float box_h = min_size / as_square_root;
+          output_.emplace_back((cx - box_w / step_w * 0.5f) / fmap_w);
+          output_.emplace_back((cy - box_h / step_h * 0.5f) / fmap_h);
+          output_.emplace_back((cx + box_w / step_w * 0.5f) / fmap_w);
+          output_.emplace_back((cy + box_h / step_h * 0.5f) / fmap_h);
+        }
+      }
+    }
+  }
+
+  // do clip
+  if (prior_box_param_->clip) {
+    for (auto item : output_) {
+      if (item > 1.0f) {
+        item = 1.0f;
+      }
+      if (item < 0.0f) {
+        item = 0.0f;
+      }
+    }
+  }
+
+  // variance
+  for (auto i = 0; i < outputs_[0]->Height() / PRIOR_BOX_VAR_NUM; i++) {
+    for (auto j = 0; j < PRIOR_BOX_VAR_NUM; j++) {
+      output_.emplace_back(prior_box_param_->variances[j]);
+    }
+  }
+  return RET_OK;
+}
+
+int PriorBoxCPUKernel::PriorBoxImpl(int task_id) {
+  auto src = output_.data();
+  auto output = outputs_.at(0);
+  if (output == nullptr) {
+    return RET_NULL_PTR;
+  }
+  auto ret = PriorBox(src, reinterpret_cast<float *>(output->Data()), output_.size(), task_id, thread_count_);
+  return ret;
+}
+
+int RunPriorBox(int task_id, LiteParallelGroupEnv *penv, void *cdata) {
+  auto prior_box = reinterpret_cast<PriorBoxCPUKernel *>(cdata);
+
+  auto error_code = prior_box->PriorBoxImpl(task_id);
+  if (error_code != RET_OK) {
+    MS_LOG(ERROR) << "Resize Run error task_id[" << task_id << "] error_code[" << error_code << "]";
+    return RET_ERROR;
+  }
+  return RET_OK;
+}
+
+int PriorBoxCPUKernel::Run() {
+  int error_code = LiteBackendParallelLaunch(RunPriorBox, this, thread_count_);
+  if (error_code != RET_OK) {
+    MS_LOG(ERROR) << "PriorBox run error, error_code[" << error_code << "]";
+    return RET_ERROR;
+  }
+  return RET_OK;
+}
+
+kernel::LiteKernel *CpuPriorBoxKernelCreator(const std::vector<lite::tensor::Tensor *> &inputs,
+                                             const std::vector<lite::tensor::Tensor *> &outputs,
+                                             OpParameter *opParameter, const Context *ctx,
+                                             const kernel::KernelKey &desc) {
+  if (opParameter == nullptr) {
+    MS_LOG(ERROR) << "Input opParameter is nullptr!";
+    return nullptr;
+  }
+  if (desc.type != schema::PrimitiveType_PriorBox) {
+    MS_LOG(ERROR) << "PriorBox invalid desc type " << desc.type;
+    return nullptr;
+  }
+  auto *kernel = new (std::nothrow) PriorBoxCPUKernel(opParameter, inputs, outputs, ctx);
+  if (kernel == nullptr) {
+    MS_LOG(ERROR) << "new PriorBoxCPUKernel fail!";
+    return nullptr;
+  }
+  auto ret = kernel->Init();
+  if (ret != RET_OK) {
+    delete kernel;
+    MS_LOG(ERROR) << "Init kernel failed, name: " << opParameter->name_ << ", type: "
+                  << schema::EnumNamePrimitiveType(static_cast<schema::PrimitiveType>(opParameter->type_));
+    return nullptr;
+  }
+  return kernel;
+}
+
+REG_KERNEL(kCPU, kNumberTypeFloat32, PrimitiveType_PriorBox, CpuPriorBoxKernelCreator)
+REG_KERNEL(kCPU, kNumberTypeInt8, PrimitiveType_PriorBox, CpuPriorBoxKernelCreator)
+}  // namespace mindspore::kernel

mindspore/lite/src/runtime/kernel/arm/base/prior_box.h

+/**
+ * Copyright 2020 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef MINDSPORE_LITE_SRC_RUNTIME_KERNEL_ARM_BASE_PRIOR_BOX_H_
+#define MINDSPORE_LITE_SRC_RUNTIME_KERNEL_ARM_BASE_PRIOR_BOX_H_
+
+#include <vector>
+#include "src/lite_kernel.h"
+#include "src/runtime/kernel/arm/opclib/reshape_parameter.h"
+#include "src/runtime/kernel/arm/opclib/prior_box.h"
+
+using mindspore::lite::Context;
+
+namespace mindspore::kernel {
+class PriorBoxCPUKernel : public LiteKernel {
+ public:
+  PriorBoxCPUKernel(OpParameter *parameter, const std::vector<lite::tensor::Tensor *> &inputs,
+                    const std::vector<lite::tensor::Tensor *> &outputs, const Context *ctx)
+      : LiteKernel(parameter, inputs, outputs), ctx_(ctx), thread_count_(ctx->threadNum) {
+    prior_box_param_ = reinterpret_cast<PriorBoxParameter *>(opParameter);
+  }
+  ~PriorBoxCPUKernel() = default;
+
+  int Init() override;
+  int ReSize() override { return 0; }
+  int Run() override;
+  int PriorBoxImpl(int task_id);
+
+ protected:
+  int thread_count_;
+  const Context *ctx_;
+
+ private:
+  std::vector<float> output_;
+  PriorBoxParameter *prior_box_param_;
+  int GeneratePriorBox();
+};
+}  // namespace mindspore::kernel
+
+#endif  // MINDSPORE_LITE_SRC_RUNTIME_KERNEL_ARM_BASE_PRIOR_BOX_H_

mindspore/lite/src/runtime/kernel/arm/opclib/prior_box.cc

+/**
+ * Copyright 2020 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <memory.h>
+#include "src/runtime/kernel/arm/opclib/errorcode.h"
+#include "src/runtime/kernel/arm/opclib/prior_box.h"
+
+int PriorBox(const float *input_data, float *output_data, const size_t size, const int tid, const int thread_num) {
+  size_t unit_size = size / thread_num;
+  if (tid == thread_num - 1) {
+    size_t tail_size = size - unit_size * tid;
+    (void)memcpy(output_data + tid * unit_size, input_data + tid * unit_size, tail_size * sizeof(float));
+  } else {
+    (void)memcpy(output_data + tid * unit_size, input_data + tid * unit_size, unit_size * sizeof(float));
+  }
+  return OPCLIB_OK;
+}

mindspore/lite/src/runtime/kernel/arm/opclib/prior_box.h

+/**
+ * Copyright 2020 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef MINDSPORE_LITE_SRC_RUNTIME_KERNEL_ARM_OPCLIB_PRIOR_BOX_H_
+#define MINDSPORE_LITE_SRC_RUNTIME_KERNEL_ARM_OPCLIB_PRIOR_BOX_H_
+
+#ifdef ENABLE_NEON
+#include <arm_neon.h>
+#endif
+#include <memory.h>
+#include "src/runtime/kernel/arm/opclib/op_base.h"
+#define PRIOR_BOX_MAX_NUM 8
+#define PRIOR_BOX_VAR_NUM 4
+struct PriorBoxParameter {
+  OpParameter op_parameter_;
+  int32_t min_sizes_size;
+  int32_t min_sizes[PRIOR_BOX_MAX_NUM];
+  int32_t max_sizes_size;
+  int32_t max_sizes[PRIOR_BOX_MAX_NUM];
+  int32_t aspect_ratios_size;
+  float aspect_ratios[PRIOR_BOX_MAX_NUM];
+  float variances[PRIOR_BOX_VAR_NUM];
+  int32_t image_size_w;
+  int32_t image_size_h;
+  float step_w;
+  float step_h;
+  bool clip;
+  bool flip;
+  float offset;
+};
+
+int PriorBox(const float *input_data, float *output_data, const size_t size, const int tid, const int thread_num);
+#endif  // MINDSPORE_LITE_SRC_RUNTIME_KERNEL_ARM_OPCLIB_PRIOR_BOX_H_