Perception: prepare code for multi-scale camera object detection (#8649)

* prepare code for multi-scale camera object detection

* add constexpr for const value

modules/perception/camera/lib/obstacle/detector/yolo/proto/yolo.proto

@@ -50,27 +50,40 @@ message ModelParam {
 }
 
 message NetworkParam {
-    optional string loc_blob = 1 [default = "loc_pred"];
-    optional string obj_blob = 2 [default = "obj_pred"];
-    optional string cls_blob = 3 [default = "cls_pred"];
-    optional string ori_blob = 4 [default = "ori_pred"];
-    optional string dim_blob = 5 [default = "dim_pred"];
-    optional string lof_blob = 6 [default = "lof_pred"];
-    optional string lor_blob = 7 [default = "lor_pred"];
-    optional string input_blob = 8 [default = "data"];
-    optional string rois_blob = 10 [default = "rois_pred"];
-    optional string feat_blob = 11 [default = "conv_feat"];
-    optional string box_blob = 13 [default = "box_pred"];
-    optional string iou_blob = 14 [default = "iou_pred"];
-    optional string brvis_blob = 15 [default = "brvis_pred"];
-    optional string brswt_blob = 16 [default = "brswt_pred"];
-    optional string ltvis_blob = 17 [default = "ltvis_pred"];
-    optional string ltswt_blob = 18 [default = "ltswt_pred"];
-    optional string rtvis_blob = 19 [default = "rtvis_pred"];
-    optional string rtswt_blob = 20 [default = "rtswt_pred"];
-    optional string area_id_blob = 23 [default = "area_id_pred"];
-    optional string visible_ratio_blob = 21 [default = "vis_pred"];
-    optional string cut_off_ratio_blob = 22 [default = "cut_pred"];
+    optional string det1_loc_blob = 1 [default = "detect1_loc_pred"];
+    optional string det1_obj_blob = 2 [default = "detect1_obj_pred"];
+    optional string det1_cls_blob = 3 [default = "detect1_cls_pred"];
+    optional string det1_ori_blob = 4 [default = "detect1_ori_pred"];
+    optional string det1_ori_conf_blob = 5 [default = "detect1_ori_conf_pred"];
+    optional string det1_dim_blob = 6 [default = "detect1_dim_pred"];
+    optional string det2_loc_blob = 7 [default = "detect2_loc_pred"];
+    optional string det2_obj_blob = 8 [default = "detect2_obj_pred"];
+    optional string det2_cls_blob = 9 [default = "detect2_cls_pred"];
+    optional string det2_ori_blob = 10 [default = "detect2_ori_pred"];
+    optional string det2_ori_conf_blob = 11 [default = "detect2_ori_conf_pred"];
+    optional string det2_dim_blob = 12 [default = "detect2_dim_pred"];
+    optional string det3_loc_blob = 13 [default = "detect3_loc_pred"];
+    optional string det3_obj_blob = 14 [default = "detect3_obj_pred"];
+    optional string det3_cls_blob = 15 [default = "detect3_cls_pred"];
+    optional string det3_ori_blob = 16 [default = "detect3_ori_pred"];
+    optional string det3_ori_conf_blob = 17 [default = "detect3_ori_conf_pred"];
+    optional string det3_dim_blob = 18 [default = "detect3_dim_pred"];
+    optional string lof_blob = 19 [default = "lof_pred"];
+    optional string lor_blob = 20 [default = "lor_pred"];
+    optional string input_blob = 21 [default = "data"];
+    optional string rois_blob = 22 [default = "rois_pred"];
+    optional string feat_blob = 23 [default = "conv_feat"];
+    optional string box_blob = 24 [default = "box_pred"];
+    optional string iou_blob = 25 [default = "iou_pred"];
+    optional string brvis_blob = 26 [default = "brvis_pred"];
+    optional string brswt_blob = 27 [default = "brswt_pred"];
+    optional string ltvis_blob = 28 [default = "ltvis_pred"];
+    optional string ltswt_blob = 29 [default = "ltswt_pred"];
+    optional string rtvis_blob = 30 [default = "rtvis_pred"];
+    optional string rtswt_blob = 31 [default = "rtswt_pred"];
+    optional string area_id_blob = 32 [default = "area_id_pred"];
+    optional string visible_ratio_blob = 33 [default = "vis_pred"];
+    optional string cut_off_ratio_blob = 34 [default = "cut_pred"];
 }
 
 message NMSParam {

modules/perception/camera/lib/obstacle/detector/yolo/region_output.cu

@@ -107,9 +107,12 @@ __global__ void get_object_kernel(int n,
                                   bool with_frbox,
                                   bool with_lights,
                                   bool with_ratios,
+                                  bool multi_scale,
                                   int num_areas,
                                   float *res_box_data,
-                                  float *res_cls_data) {
+                                  float *res_cls_data,
+                                  int res_cls_offset,
+                                  int all_scales_num_candidates) {
   for (int i = blockIdx.x * blockDim.x + threadIdx.x; i < (n);
        i += blockDim.x * gridDim.x) {
     int box_block = kBoxBlockSize;
@@ -126,21 +129,26 @@ __global__ void get_object_kernel(int n,
     float scale = obj_data[loc_index];
     float cx = (w + sigmoid_gpu(loc_data[offset_loc + 0])) / width;
     float cy = (h + sigmoid_gpu(loc_data[offset_loc + 1])) / height;
-    float hw = exp(loc_data[offset_loc + 2]) * anchor_data[2 * c] / width * 0.5;
+    float hw = 
+        exp(max(minExpPower, min(loc_data[offset_loc + 2], maxExpPower))) *
+        anchor_data[2 * c] / width * 0.5;
     float hh =
-        exp(loc_data[offset_loc + 3]) * anchor_data[2 * c + 1] / height * 0.5;
-
+        exp(max(minExpPower, min(loc_data[offset_loc + 3], maxExpPower))) *
+        anchor_data[2 * c + 1] / height * 0.5;
+   
     float max_prob = 0.f;
     int max_index = 0;
     for (int k = 0; k < num_classes; ++k) {
       float prob = cls_data[offset_cls + k] * scale;
-      res_cls_data[k * width * height * num_anchors + i] = prob;
+      res_cls_data[k * all_scales_num_candidates
+                   + res_cls_offset + i] = prob;
       if (prob > max_prob) {
         max_prob = prob;
         max_index = k;
       }
     }
-    res_cls_data[num_classes * width * height * num_anchors + i] = max_prob;
+    res_cls_data[num_classes * all_scales_num_candidates
+                 + res_cls_offset + i] = max_prob;
 
     auto &&dst_ptr = res_box_data + i * box_block;
     hw += expand_data[max_index];
@@ -154,6 +162,9 @@ __global__ void get_object_kernel(int n,
       dst_ptr[4] = atan2(ori_data[offset_ori + 1], ori_data[offset_ori]);
 
       int offset_dim = loc_index * 3;
+      if (multi_scale){
+        offset_dim = loc_index * num_classes * 3 + max_index * 3;
+      }
       dst_ptr[5] = dim_data[offset_dim + 0];
       dst_ptr[6] = dim_data[offset_dim + 1];
       dst_ptr[7] = dim_data[offset_dim + 2];
@@ -439,23 +450,56 @@ void get_objects_gpu(const YoloBlobs &yolo_blobs,
                      base::Blob<bool> *overlapped,
                      base::Blob<int> *idx_sm,
                      std::vector<base::ObjectPtr> *objects) {
+  bool multi_scale = false;
+  if (yolo_blobs.det2_obj_blob){
+    multi_scale = true;
+  }
   int num_classes = types.size();
-  int batch = yolo_blobs.obj_blob->shape(0);
-  int height = yolo_blobs.obj_blob->shape(1);
-  int width = yolo_blobs.obj_blob->shape(2);
+  int batch = yolo_blobs.det1_obj_blob->shape(0);
   int num_anchor = yolo_blobs.anchor_blob->shape(2);
-  int num_candidates = height * width * num_anchor;
-
+  int num_anchor_per_scale = num_anchor;
+  if (multi_scale){
+    num_anchor_per_scale /= numScales;
+  }
   CHECK_EQ(batch, 1) << "batch size should be 1!";
-  const float *loc_data = yolo_blobs.loc_blob->gpu_data();
-  const float *obj_data = yolo_blobs.obj_blob->gpu_data();
-  const float *cls_data = yolo_blobs.cls_blob->gpu_data();
-
-  const float *ori_data = get_gpu_data(
-          model_param.with_box3d(), *yolo_blobs.ori_blob);
-  const float *dim_data = get_gpu_data(
-          model_param.with_box3d(), *yolo_blobs.dim_blob);
+ 
+  std::vector<int> height_vec, width_vec, num_candidates_vec;
+  height_vec.push_back(yolo_blobs.det1_obj_blob->shape(1));
+  width_vec.push_back(yolo_blobs.det1_obj_blob->shape(2));
+  if (multi_scale){
+    height_vec.push_back(yolo_blobs.det2_obj_blob->shape(1));
+    height_vec.push_back(yolo_blobs.det3_obj_blob->shape(1));
+    width_vec.push_back(yolo_blobs.det2_obj_blob->shape(2));
+    width_vec.push_back(yolo_blobs.det3_obj_blob->shape(2));
+  }
+  for (size_t i=0; i<height_vec.size(); i++){
+    num_candidates_vec.push_back(
+      height_vec[i] * width_vec[i] * num_anchor_per_scale);
+  }
 
+  const float* loc_data_vec[3] = {yolo_blobs.det1_loc_blob->gpu_data(),
+    yolo_blobs.det2_loc_blob? yolo_blobs.det2_loc_blob->gpu_data() : nullptr,
+    yolo_blobs.det3_loc_blob? yolo_blobs.det3_loc_blob->gpu_data() : nullptr};
+  const float* obj_data_vec[3] = {yolo_blobs.det1_obj_blob->gpu_data(),
+    yolo_blobs.det2_obj_blob? yolo_blobs.det2_obj_blob->gpu_data() : nullptr,
+    yolo_blobs.det3_obj_blob? yolo_blobs.det3_obj_blob->gpu_data() : nullptr};
+  const float* cls_data_vec[3] = {yolo_blobs.det1_cls_blob->gpu_data(),
+    yolo_blobs.det2_cls_blob? yolo_blobs.det2_cls_blob->gpu_data() : nullptr,
+    yolo_blobs.det3_cls_blob? yolo_blobs.det3_cls_blob->gpu_data() : nullptr};
+  const float* ori_data_vec[3] = {get_gpu_data(model_param.with_box3d(),
+                                    *yolo_blobs.det1_ori_blob),
+                    multi_scale? get_gpu_data(model_param.with_box3d(),
+                                    *yolo_blobs.det2_ori_blob) : nullptr,
+                    multi_scale? get_gpu_data(model_param.with_box3d(),
+                                    *yolo_blobs.det3_ori_blob) : nullptr};
+  const float* dim_data_vec[3] = {get_gpu_data(model_param.with_box3d(),
+                                    *yolo_blobs.det1_dim_blob),
+                    multi_scale? get_gpu_data(model_param.with_box3d(),
+                                    *yolo_blobs.det2_dim_blob) : nullptr,
+                    multi_scale? get_gpu_data(model_param.with_box3d(),
+                                    *yolo_blobs.det3_dim_blob) : nullptr};
+
+  //TODO[KaWai]: add 3 scale frbox data and light data.
   const float *lof_data = get_gpu_data(
           model_param.with_frbox(), *yolo_blobs.lof_blob);
   const float *lor_data = get_gpu_data(
@@ -476,35 +520,57 @@ void get_objects_gpu(const YoloBlobs &yolo_blobs,
   const float *rtvis_data = get_gpu_data(with_lights, *yolo_blobs.rtvis_blob);
   const float *rtswt_data = get_gpu_data(with_lights, *yolo_blobs.rtswt_blob);
 
+  int all_scales_num_candidates = 0; 
+  for (size_t i = 0; i < num_candidates_vec.size(); i++){
+    all_scales_num_candidates += num_candidates_vec[i];
+  }
   yolo_blobs.res_box_blob->Reshape(
-      std::vector<int>{1, 1, num_candidates, kBoxBlockSize});
+      std::vector<int>{1, 1, all_scales_num_candidates, kBoxBlockSize});
   yolo_blobs.res_cls_blob->Reshape(
-      std::vector<int>{1, 1, num_classes + 1, num_candidates});
+      std::vector<int>{1, 1, num_classes + 1, all_scales_num_candidates});
 
   float *res_box_data = yolo_blobs.res_box_blob->mutable_gpu_data();
   float *res_cls_data = yolo_blobs.res_cls_blob->mutable_gpu_data();
   const int thread_size = 512;
-
-  int block_size = (num_candidates + thread_size - 1) / thread_size;
-  get_object_kernel <<< block_size, thread_size, 0, stream >>> (
-          num_candidates, loc_data, obj_data, cls_data, ori_data, dim_data,
-          lof_data, lor_data, area_id_data,
-          visible_ratio_data, cut_off_ratio_data,
-          brvis_data, brswt_data, ltvis_data, ltswt_data,
-          rtvis_data, rtswt_data,
-          yolo_blobs.anchor_blob->gpu_data(),
-          yolo_blobs.expand_blob->gpu_data(),
-          width, height,
-          num_anchor, num_classes, model_param.confidence_threshold(),
-          light_vis_conf_threshold, light_swt_conf_threshold,
-          model_param.with_box3d(), model_param.with_frbox(),
-          model_param.with_lights(), model_param.with_ratios(),
-          model_param.num_areas(),
-          res_box_data, res_cls_data);
-  cudaStreamSynchronize(stream);
+  //TODO[KaWai]: use different stream to process scales in parallel.
+  int num_candidates_offset = 0;
+  for (int i = 0; i < num_candidates_vec.size(); i++){
+    int block_size = (num_candidates_vec[i] + thread_size - 1) / thread_size;
+    const float *loc_data = loc_data_vec[i];
+    const float *obj_data = obj_data_vec[i];
+    const float *cls_data = cls_data_vec[i];
+    const float *ori_data = ori_data_vec[i];
+    const float *dim_data = dim_data_vec[i];
+    const float *anchor_data = yolo_blobs.anchor_blob->gpu_data()
+                               + num_anchor_per_scale * 2 * i;
+    const float *expand_data = yolo_blobs.expand_blob->gpu_data();
+    const int width = width_vec[i];
+    const int height = height_vec[i];
+    get_object_kernel <<< block_size, thread_size, 0, stream >>> (
+            num_candidates_vec[i], loc_data, obj_data,
+            cls_data, ori_data, dim_data,
+            lof_data, lor_data, area_id_data,
+            visible_ratio_data, cut_off_ratio_data,
+            brvis_data, brswt_data, ltvis_data, ltswt_data,
+            rtvis_data, rtswt_data,
+            anchor_data,
+            yolo_blobs.expand_blob->gpu_data(),
+            width, height, num_anchor_per_scale,
+            num_classes, model_param.confidence_threshold(),
+            light_vis_conf_threshold, light_swt_conf_threshold,
+            model_param.with_box3d(), model_param.with_frbox(),
+            model_param.with_lights(), model_param.with_ratios(),
+            multi_scale,
+            model_param.num_areas(),
+            res_box_data + num_candidates_offset * kBoxBlockSize,
+            res_cls_data, num_candidates_offset,
+            all_scales_num_candidates);
+    cudaStreamSynchronize(stream);
+    num_candidates_offset += num_candidates_vec[i];
+  }
   const float *cpu_cls_data = yolo_blobs.res_cls_blob->cpu_data();
 
-  std::vector<int> all_indices(num_candidates);
+  std::vector<int> all_indices(all_scales_num_candidates);
   std::iota(all_indices.begin(), all_indices.end(), 0);
   std::vector<int> rest_indices;
 
@@ -515,7 +581,7 @@ void get_objects_gpu(const YoloBlobs &yolo_blobs,
   int num_kept = 0;
   // inter-cls NMS
   apply_nms_gpu(res_box_data,
-                cpu_cls_data + num_classes * num_candidates,
+                cpu_cls_data + num_classes * all_scales_num_candidates,
                 all_indices,
                 kBoxBlockSize,
                 nms.inter_cls_conf_thresh,
@@ -527,7 +593,7 @@ void get_objects_gpu(const YoloBlobs &yolo_blobs,
                 stream);
   for (int k = 0; k < num_classes; ++k) {
     apply_nms_gpu(res_box_data,
-                  cpu_cls_data + k * num_candidates,
+                  cpu_cls_data + k * all_scales_num_candidates,
                   rest_indices,
                   kBoxBlockSize,
                   model_param.confidence_threshold(),
@@ -538,8 +604,9 @@ void get_objects_gpu(const YoloBlobs &yolo_blobs,
                   idx_sm,
                   stream);
     num_kept += indices[types[k]].size();
-    std::vector<float> conf_score(cpu_cls_data + k * num_candidates,
-                                  cpu_cls_data + (k + 1) * num_candidates);
+    std::vector<float> conf_score(
+                         cpu_cls_data + k * all_scales_num_candidates,
+                         cpu_cls_data + (k + 1) * all_scales_num_candidates);
     conf_scores.insert(std::make_pair(types[k], conf_score));
     cudaStreamSynchronize(stream);
   }

modules/perception/camera/lib/obstacle/detector/yolo/region_output.h

@@ -74,11 +74,25 @@ struct NMSParam {
   std::string type = BoxVote;
 };
 struct YoloBlobs {
-  std::shared_ptr<base::Blob<float>> loc_blob;
-  std::shared_ptr<base::Blob<float>> obj_blob;
-  std::shared_ptr<base::Blob<float>> cls_blob;
-  std::shared_ptr<base::Blob<float>> ori_blob;
-  std::shared_ptr<base::Blob<float>> dim_blob;
+  std::shared_ptr<base::Blob<float>> det1_loc_blob;
+  std::shared_ptr<base::Blob<float>> det1_obj_blob;
+  std::shared_ptr<base::Blob<float>> det1_cls_blob;
+  std::shared_ptr<base::Blob<float>> det1_ori_conf_blob;
+  std::shared_ptr<base::Blob<float>> det1_ori_blob;
+  std::shared_ptr<base::Blob<float>> det1_dim_blob;
+  std::shared_ptr<base::Blob<float>> det2_loc_blob;
+  std::shared_ptr<base::Blob<float>> det2_obj_blob;
+  std::shared_ptr<base::Blob<float>> det2_cls_blob;
+  std::shared_ptr<base::Blob<float>> det2_ori_conf_blob;
+  std::shared_ptr<base::Blob<float>> det2_ori_blob;
+  std::shared_ptr<base::Blob<float>> det2_dim_blob;
+  std::shared_ptr<base::Blob<float>> det3_loc_blob;
+  std::shared_ptr<base::Blob<float>> det3_obj_blob;
+  std::shared_ptr<base::Blob<float>> det3_cls_blob;
+  std::shared_ptr<base::Blob<float>> det3_ori_conf_blob;
+  std::shared_ptr<base::Blob<float>> det3_ori_blob;
+  std::shared_ptr<base::Blob<float>> det3_dim_blob;
+
   std::shared_ptr<base::Blob<float>> lof_blob;
   std::shared_ptr<base::Blob<float>> lor_blob;
   std::shared_ptr<base::Blob<float>> brvis_blob;
@@ -102,6 +116,11 @@ struct MinDims {
   float min_3d_width = 0.0f;
 };
 
+constexpr float minExpPower = -10.0f;
+constexpr float maxExpPower = 5.0f;
+constexpr int anchorSizeFactor = 2;
+constexpr int numScales = 3;
+
 __host__ __device__ float sigmoid_gpu(float x);
 __host__ __device__ float bbox_size_gpu(const float *bbox,
                                         const bool normalized);

modules/perception/camera/lib/obstacle/detector/yolo/yolo_obstacle_detector.cc

@@ -91,11 +91,24 @@ bool YoloObstacleDetector::InitNet(const yolo::YoloParam &yolo_param,
   // init Net
   auto const &net_param = yolo_param.net_param();
   input_names.push_back(net_param.input_blob());
-  output_names.push_back(net_param.loc_blob());
-  output_names.push_back(net_param.obj_blob());
-  output_names.push_back(net_param.cls_blob());
-  output_names.push_back(net_param.ori_blob());
-  output_names.push_back(net_param.dim_blob());
+  output_names.push_back(net_param.det1_loc_blob());
+  output_names.push_back(net_param.det1_obj_blob());
+  output_names.push_back(net_param.det1_cls_blob());
+  output_names.push_back(net_param.det1_ori_conf_blob());
+  output_names.push_back(net_param.det1_ori_blob());
+  output_names.push_back(net_param.det1_dim_blob());
+  output_names.push_back(net_param.det2_loc_blob());
+  output_names.push_back(net_param.det2_obj_blob());
+  output_names.push_back(net_param.det2_cls_blob());
+  output_names.push_back(net_param.det2_ori_conf_blob());
+  output_names.push_back(net_param.det2_ori_blob());
+  output_names.push_back(net_param.det2_dim_blob());
+  output_names.push_back(net_param.det3_loc_blob());
+  output_names.push_back(net_param.det3_obj_blob());
+  output_names.push_back(net_param.det3_cls_blob());
+  output_names.push_back(net_param.det3_ori_conf_blob());
+  output_names.push_back(net_param.det3_ori_blob());
+  output_names.push_back(net_param.det3_dim_blob());
   output_names.push_back(net_param.lof_blob());
   output_names.push_back(net_param.lor_blob());
   output_names.push_back(net_param.brvis_blob());
@@ -132,11 +145,26 @@ bool YoloObstacleDetector::InitNet(const yolo::YoloParam &yolo_param,
 }
 
 void YoloObstacleDetector::InitYoloBlob(const yolo::NetworkParam &net_param) {
-  auto obj_blob = inference_->get_blob(net_param.obj_blob());
-  int output_height = obj_blob->shape(1);
-  int output_width = obj_blob->shape(2);
+  auto obj_blob_scale1 = inference_->get_blob(net_param.det1_obj_blob());
+  auto obj_blob_scale2 = inference_->get_blob(net_param.det2_obj_blob());
+  auto obj_blob_scale3 = inference_->get_blob(net_param.det3_obj_blob());
+  int output_height_scale1 = obj_blob_scale1->shape(1);
+  int output_width_scale1 = obj_blob_scale1->shape(2);
   int obj_size =
-      output_height * output_width * static_cast<int>(anchors_.size()) / 2;
+    output_height_scale1 * output_width_scale1 *
+    static_cast<int>(anchors_.size()) / anchorSizeFactor;
+  if (obj_blob_scale2) {
+    int output_height_scale2 = obj_blob_scale2->shape(1);
+    int output_width_scale2 = obj_blob_scale2->shape(2);
+    int output_height_scale3 = obj_blob_scale3->shape(1);
+    int output_width_scale3 = obj_blob_scale3->shape(2);
+    obj_size =
+      (output_height_scale1 * output_width_scale1 +
+       output_height_scale2 * output_width_scale2 +
+       output_height_scale3 * output_width_scale3) *
+      static_cast<int>(anchors_.size()) / anchorSizeFactor / numScales;
+  }
+
   yolo_blobs_.res_box_blob.reset(
       new base::Blob<float>(1, 1, obj_size, kBoxBlockSize));
   yolo_blobs_.res_cls_blob.reset(new base::Blob<float>(
@@ -159,16 +187,43 @@ void YoloObstacleDetector::InitYoloBlob(const yolo::NetworkParam &net_param) {
 
   image_.reset(new base::Image8U(height_, width_, base::Color::RGB));
 
-  yolo_blobs_.loc_blob =
-      inference_->get_blob(yolo_param_.net_param().loc_blob());
-  yolo_blobs_.obj_blob =
-      inference_->get_blob(yolo_param_.net_param().obj_blob());
-  yolo_blobs_.cls_blob =
-      inference_->get_blob(yolo_param_.net_param().cls_blob());
-  yolo_blobs_.ori_blob =
-      inference_->get_blob(yolo_param_.net_param().ori_blob());
-  yolo_blobs_.dim_blob =
-      inference_->get_blob(yolo_param_.net_param().dim_blob());
+  yolo_blobs_.det1_loc_blob =
+      inference_->get_blob(yolo_param_.net_param().det1_loc_blob());
+  yolo_blobs_.det1_obj_blob =
+      inference_->get_blob(yolo_param_.net_param().det1_obj_blob());
+  yolo_blobs_.det1_cls_blob =
+      inference_->get_blob(yolo_param_.net_param().det1_cls_blob());
+  yolo_blobs_.det1_ori_conf_blob =
+      inference_->get_blob(yolo_param_.net_param().det1_ori_conf_blob());
+  yolo_blobs_.det1_ori_blob =
+      inference_->get_blob(yolo_param_.net_param().det1_ori_blob());
+  yolo_blobs_.det1_dim_blob =
+      inference_->get_blob(yolo_param_.net_param().det1_dim_blob());
+  yolo_blobs_.det2_loc_blob =
+      inference_->get_blob(yolo_param_.net_param().det2_loc_blob());
+  yolo_blobs_.det2_obj_blob =
+      inference_->get_blob(yolo_param_.net_param().det2_obj_blob());
+  yolo_blobs_.det2_cls_blob =
+      inference_->get_blob(yolo_param_.net_param().det2_cls_blob());
+  yolo_blobs_.det2_ori_conf_blob =
+      inference_->get_blob(yolo_param_.net_param().det2_ori_conf_blob());
+  yolo_blobs_.det2_ori_blob =
+      inference_->get_blob(yolo_param_.net_param().det2_ori_blob());
+  yolo_blobs_.det2_dim_blob =
+      inference_->get_blob(yolo_param_.net_param().det2_dim_blob());
+  yolo_blobs_.det3_loc_blob =
+      inference_->get_blob(yolo_param_.net_param().det3_loc_blob());
+  yolo_blobs_.det3_obj_blob =
+      inference_->get_blob(yolo_param_.net_param().det3_obj_blob());
+  yolo_blobs_.det3_cls_blob =
+      inference_->get_blob(yolo_param_.net_param().det3_cls_blob());
+  yolo_blobs_.det3_ori_conf_blob =
+      inference_->get_blob(yolo_param_.net_param().det3_ori_conf_blob());
+  yolo_blobs_.det3_ori_blob =
+      inference_->get_blob(yolo_param_.net_param().det3_ori_blob());
+  yolo_blobs_.det3_dim_blob =
+      inference_->get_blob(yolo_param_.net_param().det3_dim_blob());
+
   yolo_blobs_.lof_blob =
       inference_->get_blob(yolo_param_.net_param().lof_blob());
   yolo_blobs_.lor_blob =
@@ -286,7 +341,8 @@ bool YoloObstacleDetector::Detect(const ObstacleDetectorOptions &options,
 
   /////////////////////////// detection part ///////////////////////////
   inference_->Infer();
-
+  AINFO << "Network Forward: "
+        << static_cast<double>(timer.Toc()) * 0.001 << "ms";
   get_objects_gpu(yolo_blobs_, stream_, types_, nms_, yolo_param_.model_param(),
                   light_vis_conf_threshold_, light_swt_conf_threshold_,
                   overlapped_.get(), idx_sm_.get(), &(frame->detected_objects));

modules/perception/production/data/perception/camera/models/yolo_obstacle_detector/3d-r4-half/deploy.pt

@@ -1542,7 +1542,7 @@ layer {
   name: "slice"
   type: "Slice"
   bottom: "conv_final_permute"
-  top: "loc_pred"
+  top: "detect1_loc_pred"
   top: "obj_perm"
   top: "cls_perm"
   slice_param {
@@ -1578,7 +1578,7 @@ layer {
   name: "cls_pred"
   type: "Reshape"
   bottom: "cls_pred_prob"
-  top: "cls_pred"
+  top: "detect1_cls_pred"
   reshape_param {
     shape {
       dim: 0
@@ -1592,7 +1592,7 @@ layer {
   name: "obj_pred"
   type: "Sigmoid"
   bottom: "obj_perm"
-  top: "obj_pred"
+  top: "detect1_obj_pred"
 }
 ###########################
 ######## BBOX3D ###########
@@ -1621,7 +1621,7 @@ layer {
   name: "ori_pred"
   type: "Permute"
   bottom: "ori_origin"
-  top: "ori_pred"
+  top: "detect1_ori_pred"
   permute_param {
     order: 0
     order: 2
@@ -1653,7 +1653,7 @@ layer {
   name: "dim_pred"
   type: "Permute"
   bottom: "dim_origin"
-  top: "dim_pred"
+  top: "detect1_dim_pred"
   permute_param {
     order: 0
     order: 2