Merge branch 'dygraph' into robustscanner_branch

.gitignore

@@ -31,4 +31,4 @@ paddleocr.egg-info/
 /deploy/android_demo/app/.cxx/
 /deploy/android_demo/app/cache/
 test_tipc/web/models/
-test_tipc/web/node_modules/
+test_tipc/web/node_modules/
\ No newline at end of file

configs/table/SLANet.yml

+Global:
+  use_gpu: true
+  epoch_num: 100
+  log_smooth_window: 20
+  print_batch_step: 20
+  save_model_dir: ./output/SLANet
+  save_epoch_step: 400
+  # evaluation is run every 1000 iterations after the 0th iteration
+  eval_batch_step: [0, 1000]
+  cal_metric_during_train: True
+  pretrained_model:
+  checkpoints:
+  save_inference_dir: ./output/SLANet/infer
+  use_visualdl: False
+  infer_img: doc/table/table.jpg
+  # for data or label process
+  character_dict_path: ppocr/utils/dict/table_structure_dict.txt
+  character_type: en
+  max_text_length: &max_text_length 500
+  box_format: &box_format 'xyxy' # 'xywh', 'xyxy', 'xyxyxyxy'
+  infer_mode: False
+  use_sync_bn: True
+  save_res_path: 'output/infer'
+
+Optimizer:
+  name: Adam
+  beta1: 0.9
+  beta2: 0.999
+  clip_norm: 5.0
+  lr:
+    name: Piecewise
+    learning_rate: 0.001
+    decay_epochs : [40, 50]
+    values : [0.001, 0.0001, 0.00005]
+  regularizer:
+    name: 'L2'
+    factor: 0.00000
+
+Architecture:
+  model_type: table
+  algorithm: SLANet
+  Backbone:
+    name: PPLCNet
+    scale: 1.0
+    pretrained: true
+    use_ssld: true
+  Neck:
+    name: CSPPAN
+    out_channels: 96
+  Head:
+    name: SLAHead
+    hidden_size: 256
+    max_text_length: *max_text_length
+    loc_reg_num: &loc_reg_num 4
+
+Loss:
+  name: SLALoss
+  structure_weight: 1.0
+  loc_weight: 2.0
+  loc_loss: smooth_l1
+
+PostProcess:
+  name: TableLabelDecode
+  merge_no_span_structure: &merge_no_span_structure True
+
+Metric:
+  name: TableMetric
+  main_indicator: acc
+  compute_bbox_metric: False
+  loc_reg_num: *loc_reg_num
+  box_format: *box_format
+
+Train:
+  dataset:
+    name: PubTabDataSet
+    data_dir: train_data/table/pubtabnet/train/
+    label_file_list: [train_data/table/pubtabnet/PubTabNet_2.0.0_train.jsonl]
+    transforms:
+      - DecodeImage: # load image
+          img_mode: BGR
+          channel_first: False
+      - TableLabelEncode:
+          learn_empty_box: False
+          merge_no_span_structure: *merge_no_span_structure
+          replace_empty_cell_token: False
+          loc_reg_num: *loc_reg_num
+          max_text_length: *max_text_length
+      - TableBoxEncode:
+          in_box_format: *box_format
+          out_box_format: *box_format
+      - ResizeTableImage:
+          max_len: 488
+      - NormalizeImage:
+          scale: 1./255.
+          mean: [0.485, 0.456, 0.406]
+          std: [0.229, 0.224, 0.225]
+          order: 'hwc'
+      - PaddingTableImage:
+          size: [488, 488]
+      - ToCHWImage:
+      - KeepKeys:
+          keep_keys: [ 'image', 'structure', 'bboxes', 'bbox_masks', 'shape' ]
+  loader:
+    shuffle: True
+    batch_size_per_card: 48
+    drop_last: True
+    num_workers: 1
+
+Eval:
+  dataset:
+    name: PubTabDataSet
+    data_dir: train_data/table/pubtabnet/val/
+    label_file_list: [train_data/table/pubtabnet/PubTabNet_2.0.0_val.jsonl]
+    transforms:
+      - DecodeImage: # load image
+          img_mode: BGR
+          channel_first: False
+      - TableLabelEncode:
+          learn_empty_box: False
+          merge_no_span_structure: *merge_no_span_structure
+          replace_empty_cell_token: False
+          loc_reg_num: *loc_reg_num
+          max_text_length: *max_text_length
+      - TableBoxEncode:
+          in_box_format: *box_format
+          out_box_format: *box_format
+      - ResizeTableImage:
+          max_len: 488
+      - NormalizeImage:
+          scale: 1./255.
+          mean: [0.485, 0.456, 0.406]
+          std: [0.229, 0.224, 0.225]
+          order: 'hwc'
+      - PaddingTableImage:
+          size: [488, 488]
+      - ToCHWImage:
+      - KeepKeys:
+          keep_keys: [ 'image', 'structure', 'bboxes', 'bbox_masks', 'shape' ]
+  loader:
+    shuffle: False
+    drop_last: False
+    batch_size_per_card: 48
+    num_workers: 1

configs/table/table_master.yml

@@ -8,16 +8,15 @@ Global:
   eval_batch_step: [0,  6259]
   cal_metric_during_train: true
   pretrained_model: null
-  checkpoints: 
+  checkpoints:
   save_inference_dir: output/table_master/infer
   use_visualdl: false
   infer_img: ppstructure/docs/table/table.jpg
   save_res_path: ./output/table_master
   character_dict_path: ppocr/utils/dict/table_master_structure_dict.txt
   infer_mode: false
-  max_text_length: 500
-  process_total_num: 0
-  process_cut_num: 0
+  max_text_length: &max_text_length 500
+  box_format: &box_format 'xywh' # 'xywh', 'xyxy', 'xyxyxyxy'
 
 
 Optimizer:
@@ -52,7 +51,8 @@ Architecture:
     headers: 8
     dropout: 0
     d_ff: 2024
-    max_text_length: 500
+    max_text_length: *max_text_length
+    loc_reg_num: &loc_reg_num 4
 
 Loss:
   name: TableMasterLoss
@@ -61,11 +61,13 @@ Loss:
 PostProcess:
   name: TableMasterLabelDecode
   box_shape: pad
+  merge_no_span_structure: &merge_no_span_structure True
 
 Metric:
   name: TableMetric
   main_indicator: acc
   compute_bbox_metric: False
+  box_format: *box_format
 
 Train:
   dataset:
@@ -78,15 +80,18 @@ Train:
           channel_first: False
       - TableMasterLabelEncode:
           learn_empty_box: False
-          merge_no_span_structure: True
+          merge_no_span_structure: *merge_no_span_structure
           replace_empty_cell_token: True
+          loc_reg_num: *loc_reg_num
+          max_text_length: *max_text_length
       - ResizeTableImage:
           max_len: 480
           resize_bboxes: True
       - PaddingTableImage:
           size: [480, 480]
       - TableBoxEncode:
-          use_xywh: True
+          in_box_format: *box_format
+          out_box_format: *box_format
       - NormalizeImage:
           scale: 1./255.
           mean: [0.5, 0.5, 0.5]
@@ -112,15 +117,18 @@ Eval:
           channel_first: False
       - TableMasterLabelEncode:
           learn_empty_box: False
-          merge_no_span_structure: True
+          merge_no_span_structure: *merge_no_span_structure
           replace_empty_cell_token: True
+          loc_reg_num: *loc_reg_num
+          max_text_length: *max_text_length
       - ResizeTableImage:
           max_len: 480
           resize_bboxes: True
       - PaddingTableImage:
           size: [480, 480]
       - TableBoxEncode:
-          use_xywh: True
+          in_box_format: *box_format
+          out_box_format: *box_format
       - NormalizeImage:
           scale: 1./255.
           mean: [0.5, 0.5, 0.5]

configs/table/table_mv3.yml

@@ -17,10 +17,9 @@ Global:
   # for data or label process
   character_dict_path: ppocr/utils/dict/table_structure_dict.txt
   character_type: en
-  max_text_length: 800
+  max_text_length: &max_text_length 800
+  box_format: &box_format 'xyxy' # 'xywh', 'xyxy', 'xyxyxyxy'
   infer_mode: False
-  process_total_num: 0
-  process_cut_num: 0
 
 Optimizer:
   name: Adam
@@ -44,7 +43,8 @@ Architecture:
     name: TableAttentionHead
     hidden_size: 256
     loc_type: 2
-    max_text_length: 800
+    max_text_length: *max_text_length
+    loc_reg_num: &loc_reg_num 4
 
 Loss:
   name: TableAttentionLoss
@@ -72,6 +72,8 @@ Train:
           learn_empty_box: False
           merge_no_span_structure: False
           replace_empty_cell_token: False
+          loc_reg_num: *loc_reg_num
+          max_text_length: *max_text_length
       - TableBoxEncode:
       - ResizeTableImage:
           max_len: 488
@@ -94,8 +96,8 @@ Train:
 Eval:
   dataset:
     name: PubTabDataSet
-    data_dir: /home/zhoujun20/table/PubTabNe/pubtabnet/val/
-    label_file_list: [/home/zhoujun20/table/PubTabNe/pubtabnet/val_500.jsonl]
+    data_dir: train_data/table/pubtabnet/val/
+    label_file_list: [train_data/table/pubtabnet/PubTabNet_2.0.0_val.jsonl]
     transforms:
       - DecodeImage: # load image
           img_mode: BGR
@@ -104,6 +106,8 @@ Eval:
           learn_empty_box: False
           merge_no_span_structure: False
           replace_empty_cell_token: False
+          loc_reg_num: *loc_reg_num
+          max_text_length: *max_text_length
       - TableBoxEncode:
       - ResizeTableImage:
           max_len: 488

deploy/cpp_infer/include/args.h

@@ -30,7 +30,8 @@ DECLARE_string(image_dir);
 DECLARE_string(type);
 // detection related
 DECLARE_string(det_model_dir);
-DECLARE_int32(max_side_len);
+DECLARE_string(limit_type);
+DECLARE_int32(limit_side_len);
 DECLARE_double(det_db_thresh);
 DECLARE_double(det_db_box_thresh);
 DECLARE_double(det_db_unclip_ratio);
@@ -48,7 +49,13 @@ DECLARE_int32(rec_batch_num);
 DECLARE_string(rec_char_dict_path);
 DECLARE_int32(rec_img_h);
 DECLARE_int32(rec_img_w);
+// structure model related
+DECLARE_string(table_model_dir);
+DECLARE_int32(table_max_len);
+DECLARE_int32(table_batch_num);
+DECLARE_string(table_char_dict_path);
 // forward related
 DECLARE_bool(det);
 DECLARE_bool(rec);
 DECLARE_bool(cls);
+DECLARE_bool(table);
\ No newline at end of file

deploy/cpp_infer/include/ocr_det.h

@@ -41,8 +41,8 @@ public:
   explicit DBDetector(const std::string &model_dir, const bool &use_gpu,
                       const int &gpu_id, const int &gpu_mem,
                       const int &cpu_math_library_num_threads,
-                      const bool &use_mkldnn, const int &max_side_len,
-                      const double &det_db_thresh,
+                      const bool &use_mkldnn, const string &limit_type,
+                      const int &limit_side_len, const double &det_db_thresh,
                       const double &det_db_box_thresh,
                       const double &det_db_unclip_ratio,
                       const std::string &det_db_score_mode,
@@ -54,7 +54,8 @@ public:
     this->cpu_math_library_num_threads_ = cpu_math_library_num_threads;
     this->use_mkldnn_ = use_mkldnn;
 
-    this->max_side_len_ = max_side_len;
+    this->limit_type_ = limit_type;
+    this->limit_side_len_ = limit_side_len;
 
     this->det_db_thresh_ = det_db_thresh;
     this->det_db_box_thresh_ = det_db_box_thresh;
@@ -84,7 +85,8 @@ private:
   int cpu_math_library_num_threads_ = 4;
   bool use_mkldnn_ = false;
 
-  int max_side_len_ = 960;
+  string limit_type_ = "max";
+  int limit_side_len_ = 960;
 
   double det_db_thresh_ = 0.3;
   double det_db_box_thresh_ = 0.5;
@@ -106,7 +108,7 @@ private:
   Permute permute_op_;
 
   // post-process
-  PostProcessor post_processor_;
+  DBPostProcessor post_processor_;
 };
 
 } // namespace PaddleOCR
\ No newline at end of file

deploy/cpp_infer/include/paddleocr.h

@@ -47,11 +47,7 @@ public:
   ocr(std::vector<cv::String> cv_all_img_names, bool det = true,
       bool rec = true, bool cls = true);
 
-private:
-  DBDetector *detector_ = nullptr;
-  Classifier *classifier_ = nullptr;
-  CRNNRecognizer *recognizer_ = nullptr;
-
+protected:
   void det(cv::Mat img, std::vector<OCRPredictResult> &ocr_results,
            std::vector<double> &times);
   void rec(std::vector<cv::Mat> img_list,
@@ -62,6 +58,11 @@ private:
            std::vector<double> &times);
   void log(std::vector<double> &det_times, std::vector<double> &rec_times,
            std::vector<double> &cls_times, int img_num);
+
+private:
+  DBDetector *detector_ = nullptr;
+  Classifier *classifier_ = nullptr;
+  CRNNRecognizer *recognizer_ = nullptr;
 };
 
 } // namespace PaddleOCR

deploy/cpp_infer/include/paddlestructure.h

+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include "opencv2/core.hpp"
+#include "opencv2/imgcodecs.hpp"
+#include "opencv2/imgproc.hpp"
+#include "paddle_api.h"
+#include "paddle_inference_api.h"
+#include <chrono>
+#include <iomanip>
+#include <iostream>
+#include <ostream>
+#include <vector>
+
+#include <cstring>
+#include <fstream>
+#include <numeric>
+
+#include <include/paddleocr.h>
+#include <include/preprocess_op.h>
+#include <include/structure_table.h>
+#include <include/utility.h>
+
+using namespace paddle_infer;
+
+namespace PaddleOCR {
+
+class PaddleStructure : public PPOCR {
+public:
+  explicit PaddleStructure();
+  ~PaddleStructure();
+  std::vector<std::vector<StructurePredictResult>>
+  structure(std::vector<cv::String> cv_all_img_names, bool layout = false,
+            bool table = true);
+
+private:
+  StructureTableRecognizer *recognizer_ = nullptr;
+
+  void table(cv::Mat img, StructurePredictResult &structure_result,
+             std::vector<double> &time_info_table,
+             std::vector<double> &time_info_det,
+             std::vector<double> &time_info_rec,
+             std::vector<double> &time_info_cls);
+  std::string
+  rebuild_table(std::vector<std::string> rec_html_tags,
+                std::vector<std::vector<std::vector<int>>> rec_boxes,
+                std::vector<OCRPredictResult> &ocr_result);
+
+  float iou(std::vector<std::vector<int>> &box1,
+            std::vector<std::vector<int>> &box2);
+  float dis(std::vector<std::vector<int>> &box1,
+            std::vector<std::vector<int>> &box2);
+
+  static bool comparison_dis(const std::vector<float> &dis1,
+                             const std::vector<float> &dis2) {
+    if (dis1[1] < dis2[1]) {
+      return true;
+    } else if (dis1[1] == dis2[1]) {
+      return dis1[0] < dis2[0];
+    } else {
+      return false;
+    }
+  }
+};
+
+} // namespace PaddleOCR
\ No newline at end of file

deploy/cpp_infer/include/postprocess_op.h

@@ -34,7 +34,7 @@ using namespace std;
 
 namespace PaddleOCR {
 
-class PostProcessor {
+class DBPostProcessor {
 public:
   void GetContourArea(const std::vector<std::vector<float>> &box,
                       float unclip_ratio, float &distance);
@@ -90,4 +90,21 @@ private:
   }
 };
 
+class TablePostProcessor {
+public:
+  void init(std::string label_path);
+  void
+  Run(std::vector<float> &loc_preds, std::vector<float> &structure_probs,
+      std::vector<float> &rec_scores, std::vector<int> &loc_preds_shape,
+      std::vector<int> &structure_probs_shape,
+      std::vector<std::vector<std::string>> &rec_html_tag_batch,
+      std::vector<std::vector<std::vector<std::vector<int>>>> &rec_boxes_batch,
+      std::vector<int> &width_list, std::vector<int> &height_list);
+
+private:
+  std::vector<std::string> label_list_;
+  std::string end = "eos";
+  std::string beg = "sos";
+};
+
 } // namespace PaddleOCR

deploy/cpp_infer/include/preprocess_op.h

@@ -48,11 +48,12 @@ class PermuteBatch {
 public:
   virtual void Run(const std::vector<cv::Mat> imgs, float *data);
 };
-    
+
 class ResizeImgType0 {
 public:
-  virtual void Run(const cv::Mat &img, cv::Mat &resize_img, int max_size_len,
-                   float &ratio_h, float &ratio_w, bool use_tensorrt);
+  virtual void Run(const cv::Mat &img, cv::Mat &resize_img, string limit_type,
+                   int limit_side_len, float &ratio_h, float &ratio_w,
+                   bool use_tensorrt);
 };
 
 class CrnnResizeImg {
@@ -69,4 +70,16 @@ public:
                    const std::vector<int> &rec_image_shape = {3, 48, 192});
 };
 
+class TableResizeImg {
+public:
+  virtual void Run(const cv::Mat &img, cv::Mat &resize_img,
+                   const int max_len = 488);
+};
+
+class TablePadImg {
+public:
+  virtual void Run(const cv::Mat &img, cv::Mat &resize_img,
+                   const int max_len = 488);
+};
+
 } // namespace PaddleOCR
\ No newline at end of file

deploy/cpp_infer/include/structure_table.h

+// Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include "opencv2/core.hpp"
+#include "opencv2/imgcodecs.hpp"
+#include "opencv2/imgproc.hpp"
+#include "paddle_api.h"
+#include "paddle_inference_api.h"
+#include <chrono>
+#include <iomanip>
+#include <iostream>
+#include <ostream>
+#include <vector>
+
+#include <cstring>
+#include <fstream>
+#include <numeric>
+
+#include <include/postprocess_op.h>
+#include <include/preprocess_op.h>
+#include <include/utility.h>
+
+using namespace paddle_infer;
+
+namespace PaddleOCR {
+
+class StructureTableRecognizer {
+public:
+  explicit StructureTableRecognizer(
+      const std::string &model_dir, const bool &use_gpu, const int &gpu_id,
+      const int &gpu_mem, const int &cpu_math_library_num_threads,
+      const bool &use_mkldnn, const string &label_path,
+      const bool &use_tensorrt, const std::string &precision,
+      const int &table_batch_num, const int &table_max_len) {
+    this->use_gpu_ = use_gpu;
+    this->gpu_id_ = gpu_id;
+    this->gpu_mem_ = gpu_mem;
+    this->cpu_math_library_num_threads_ = cpu_math_library_num_threads;
+    this->use_mkldnn_ = use_mkldnn;
+    this->use_tensorrt_ = use_tensorrt;
+    this->precision_ = precision;
+    this->table_batch_num_ = table_batch_num;
+    this->table_max_len_ = table_max_len;
+
+    this->post_processor_.init(label_path);
+    LoadModel(model_dir);
+  }
+
+  // Load Paddle inference model
+  void LoadModel(const std::string &model_dir);
+
+  void Run(std::vector<cv::Mat> img_list,
+           std::vector<std::vector<std::string>> &rec_html_tags,
+           std::vector<float> &rec_scores,
+           std::vector<std::vector<std::vector<std::vector<int>>>> &rec_boxes,
+           std::vector<double> &times);
+
+private:
+  std::shared_ptr<Predictor> predictor_;
+
+  bool use_gpu_ = false;
+  int gpu_id_ = 0;
+  int gpu_mem_ = 4000;
+  int cpu_math_library_num_threads_ = 4;
+  bool use_mkldnn_ = false;
+  int table_max_len_ = 488;
+
+  std::vector<float> mean_ = {0.485f, 0.456f, 0.406f};
+  std::vector<float> scale_ = {1 / 0.229f, 1 / 0.224f, 1 / 0.225f};
+  bool is_scale_ = true;
+
+  bool use_tensorrt_ = false;
+  std::string precision_ = "fp32";
+  int table_batch_num_ = 1;
+
+  // pre-process
+  TableResizeImg resize_op_;
+  Normalize normalize_op_;
+  PermuteBatch permute_op_;
+  TablePadImg pad_op_;
+
+  // post-process
+  TablePostProcessor post_processor_;
+
+}; // class StructureTableRecognizer
+
+} // namespace PaddleOCR
\ No newline at end of file

deploy/cpp_infer/include/utility.h

@@ -40,6 +40,14 @@ struct OCRPredictResult {
   int cls_label = -1;
 };
 
+struct StructurePredictResult {
+  std::vector<int> box;
+  std::string type;
+  std::vector<OCRPredictResult> text_res;
+  std::string html;
+  float html_score = -1;
+};
+
 class Utility {
 public:
   static std::vector<std::string> ReadDict(const std::string &path);
@@ -68,6 +76,22 @@ public:
   static void CreateDir(const std::string &path);
 
   static void print_result(const std::vector<OCRPredictResult> &ocr_result);
+
+  static cv::Mat crop_image(cv::Mat &img, std::vector<int> &area);
+
+  static void sorted_boxes(std::vector<OCRPredictResult> &ocr_result);
+
+private:
+  static bool comparison_box(const OCRPredictResult &result1,
+                             const OCRPredictResult &result2) {
+    if (result1.box[0][1] < result2.box[0][1]) {
+      return true;
+    } else if (result1.box[0][1] == result2.box[0][1]) {
+      return result1.box[0][0] < result2.box[0][0];
+    } else {
+      return false;
+    }
+  }
 };
 
 } // namespace PaddleOCR
\ No newline at end of file

deploy/cpp_infer/readme.md

@@ -171,6 +171,9 @@ inference/
 |-- cls
 |   |--inference.pdiparams
 |   |--inference.pdmodel
+|-- table
+|   |--inference.pdiparams
+|   |--inference.pdmodel
 ```
 
 
@@ -275,6 +278,17 @@ Specifically,
     --cls=true \
 ```
 
+
+##### 7. table
+```shell
+./build/ppocr --det_model_dir=inference/det_db \
+    --rec_model_dir=inference/rec_rcnn \
+    --table_model_dir=inference/table \
+    --image_dir=../../ppstructure/docs/table/table.jpg \
+    --type=structure \
+    --table=true
+```
+
 More parameters are as follows,
 
 - Common parameters
@@ -293,9 +307,9 @@ More parameters are as follows,
 
 |parameter|data type|default|meaning|
 | :---: | :---: | :---: | :---: |
-|det|bool|true|前向是否执行文字检测|
-|rec|bool|true|前向是否执行文字识别|
-|cls|bool|false|前向是否执行文字方向分类|
+|det|bool|true|Whether to perform text detection in the forward direction|
+|rec|bool|true|Whether to perform text recognition in the forward direction|
+|cls|bool|false|Whether to perform text direction classification in the forward direction|
 
 
 - Detection related parameters
@@ -329,6 +343,15 @@ More parameters are as follows,
 |rec_img_h|int|48|image height of recognition|
 |rec_img_w|int|320|image width of recognition|
 
+- Table recognition related parameters
+
+|parameter|data type|default|meaning|
+| :---: | :---: | :---: | :---: |
+|table_model_dir|string|-|Address of table recognition inference model|
+|table_char_dict_path|string|../../ppocr/utils/dict/table_structure_dict.txt|dictionary file|
+|table_max_len|int|488|The size of the long side of the input image of the table recognition model, the final input image size of the network is（table_max_len，table_max_len）|
+
+
 * Multi-language inference is also supported in PaddleOCR, you can refer to [recognition tutorial](../../doc/doc_en/recognition_en.md) for more supported languages and models in PaddleOCR. Specifically, if you want to infer using multi-language models, you just need to modify values of `rec_char_dict_path` and `rec_model_dir`.
 
 
@@ -344,6 +367,12 @@ predict img: ../../doc/imgs/12.jpg
 The detection visualized image saved in ./output//12.jpg
 ```
 
+- table
+
+```bash
+predict img: ../../ppstructure/docs/table/table.jpg
+0       type: table, region: [0,0,371,293], res: <html><body><table><thead><tr><td>Methods</td><td>R</td><td>P</td><td>F</td><td>FPS</td></tr></thead><tbody><tr><td>SegLink [26]</td><td>70.0</td><td>86.0</td><td>77.0</td><td>8.9</td></tr><tr><td>PixelLink [4]</td><td>73.2</td><td>83.0</td><td>77.8</td><td>-</td></tr><tr><td>TextSnake [18]</td><td>73.9</td><td>83.2</td><td>78.3</td><td>1.1</td></tr><tr><td>TextField [37]</td><td>75.9</td><td>87.4</td><td>81.3</td><td>5.2 </td></tr><tr><td>MSR[38]</td><td>76.7</td><td>87.4</td><td>81.7</td><td>-</td></tr><tr><td>FTSN [3]</td><td>77.1</td><td>87.6</td><td>82.0</td><td>-</td></tr><tr><td>LSE[30]</td><td>81.7</td><td>84.2</td><td>82.9</td><td>-</td></tr><tr><td>CRAFT [2]</td><td>78.2</td><td>88.2</td><td>82.9</td><td>8.6</td></tr><tr><td>MCN [16]</td><td>79</td><td>88</td><td>83</td><td>-</td></tr><tr><td>ATRR[35]</td><td>82.1</td><td>85.2</td><td>83.6</td><td>-</td></tr><tr><td>PAN [34]</td><td>83.8</td><td>84.4</td><td>84.1</td><td>30.2</td></tr><tr><td>DB[12]</td><td>79.2</td><td>91.5</td><td>84.9</td><td>32.0</td></tr><tr><td>DRRG [41]</td><td>82.30</td><td>88.05</td><td>85.08</td><td>-</td></tr><tr><td>Ours (SynText)</td><td>80.68</td><td>85.40</td><td>82.97</td><td>12.68</td></tr><tr><td>Ours (MLT-17)</td><td>84.54</td><td>86.62</td><td>85.57</td><td>12.31</td></tr></tbody></table></body></html>
+```
 
 <a name="3"></a>
 ## 3. FAQ

deploy/cpp_infer/readme_ch.md

@@ -181,6 +181,9 @@ inference/
 |-- cls
 |   |--inference.pdiparams
 |   |--inference.pdmodel
+|-- table
+|   |--inference.pdiparams
+|   |--inference.pdmodel
 ```
 
 <a name="22"></a>
@@ -285,6 +288,16 @@ CUDNN_LIB_DIR=/your_cudnn_lib_dir
     --cls=true \
 ```
 
+##### 7. 表格识别
+```shell
+./build/ppocr --det_model_dir=inference/det_db \
+    --rec_model_dir=inference/rec_rcnn \
+    --table_model_dir=inference/table \
+    --image_dir=../../ppstructure/docs/table/table.jpg \
+    --type=structure \
+    --table=true
+```
+
 更多支持的可调节参数解释如下：
 
 - 通用参数
@@ -328,21 +341,32 @@ CUDNN_LIB_DIR=/your_cudnn_lib_dir
 |cls_thresh|float|0.9|方向分类器的得分阈值|
 |cls_batch_num|int|1|方向分类器batchsize|
 
-- 识别模型相关
+- 文字识别模型相关
 
 |参数名称|类型|默认参数|意义|
 | :---: | :---: | :---: | :---: |
-|rec_model_dir|string|-|识别模型inference model地址|
+|rec_model_dir|string|-|文字识别模型inference model地址|
 |rec_char_dict_path|string|../../ppocr/utils/ppocr_keys_v1.txt|字典文件|
-|rec_batch_num|int|6|识别模型batchsize|
-|rec_img_h|int|48|识别模型输入图像高度|
-|rec_img_w|int|320|识别模型输入图像宽度|
+|rec_batch_num|int|6|文字识别模型batchsize|
+|rec_img_h|int|48|文字识别模型输入图像高度|
+|rec_img_w|int|320|文字识别模型输入图像宽度|
+
+
+- 表格识别模型相关
+
+|参数名称|类型|默认参数|意义|
+| :---: | :---: | :---: | :---: |
+|table_model_dir|string|-|表格识别模型inference model地址|
+|table_char_dict_path|string|../../ppocr/utils/dict/table_structure_dict.txt|字典文件|
+|table_max_len|int|488|表格识别模型输入图像长边大小，最终网络输入图像大小为（table_max_len，table_max_len）|
 
 
 * PaddleOCR也支持多语言的预测，更多支持的语言和模型可以参考[识别文档](../../doc/doc_ch/recognition.md)中的多语言字典与模型部分，如果希望进行多语言预测，只需将修改`rec_char_dict_path`（字典文件路径）以及`rec_model_dir`（inference模型路径）字段即可。
 
 最终屏幕上会输出检测结果如下。
 
+- ocr
+
 ```bash
 predict img: ../../doc/imgs/12.jpg
 ../../doc/imgs/12.jpg
@@ -353,6 +377,13 @@ predict img: ../../doc/imgs/12.jpg
 The detection visualized image saved in ./output//12.jpg
 ```
 
+- table
+
+```bash
+predict img: ../../ppstructure/docs/table/table.jpg
+0       type: table, region: [0,0,371,293], res: <html><body><table><thead><tr><td>Methods</td><td>R</td><td>P</td><td>F</td><td>FPS</td></tr></thead><tbody><tr><td>SegLink [26]</td><td>70.0</td><td>86.0</td><td>77.0</td><td>8.9</td></tr><tr><td>PixelLink [4]</td><td>73.2</td><td>83.0</td><td>77.8</td><td>-</td></tr><tr><td>TextSnake [18]</td><td>73.9</td><td>83.2</td><td>78.3</td><td>1.1</td></tr><tr><td>TextField [37]</td><td>75.9</td><td>87.4</td><td>81.3</td><td>5.2 </td></tr><tr><td>MSR[38]</td><td>76.7</td><td>87.4</td><td>81.7</td><td>-</td></tr><tr><td>FTSN [3]</td><td>77.1</td><td>87.6</td><td>82.0</td><td>-</td></tr><tr><td>LSE[30]</td><td>81.7</td><td>84.2</td><td>82.9</td><td>-</td></tr><tr><td>CRAFT [2]</td><td>78.2</td><td>88.2</td><td>82.9</td><td>8.6</td></tr><tr><td>MCN [16]</td><td>79</td><td>88</td><td>83</td><td>-</td></tr><tr><td>ATRR[35]</td><td>82.1</td><td>85.2</td><td>83.6</td><td>-</td></tr><tr><td>PAN [34]</td><td>83.8</td><td>84.4</td><td>84.1</td><td>30.2</td></tr><tr><td>DB[12]</td><td>79.2</td><td>91.5</td><td>84.9</td><td>32.0</td></tr><tr><td>DRRG [41]</td><td>82.30</td><td>88.05</td><td>85.08</td><td>-</td></tr><tr><td>Ours (SynText)</td><td>80.68</td><td>85.40</td><td>82.97</td><td>12.68</td></tr><tr><td>Ours (MLT-17)</td><td>84.54</td><td>86.62</td><td>85.57</td><td>12.31</td></tr></tbody></table></body></html>
+```
+
 <a name="3"></a>
 ## 3. FAQ
 

deploy/cpp_infer/src/args.cpp

@@ -30,7 +30,8 @@ DEFINE_string(
     "Perform ocr or structure, the value is selected in ['ocr','structure'].");
 // detection related
 DEFINE_string(det_model_dir, "", "Path of det inference model.");
-DEFINE_int32(max_side_len, 960, "max_side_len of input image.");
+DEFINE_string(limit_type, "max", "limit_type of input image.");
+DEFINE_int32(limit_side_len, 960, "limit_side_len of input image.");
 DEFINE_double(det_db_thresh, 0.3, "Threshold of det_db_thresh.");
 DEFINE_double(det_db_box_thresh, 0.6, "Threshold of det_db_box_thresh.");
 DEFINE_double(det_db_unclip_ratio, 1.5, "Threshold of det_db_unclip_ratio.");
@@ -50,7 +51,16 @@ DEFINE_string(rec_char_dict_path, "../../ppocr/utils/ppocr_keys_v1.txt",
 DEFINE_int32(rec_img_h, 48, "rec image height");
 DEFINE_int32(rec_img_w, 320, "rec image width");
 
+// structure model related
+DEFINE_string(table_model_dir, "", "Path of table struture inference model.");
+DEFINE_int32(table_max_len, 488, "max len size of input image.");
+DEFINE_int32(table_batch_num, 1, "table_batch_num.");
+DEFINE_string(table_char_dict_path,
+              "../../ppocr/utils/dict/table_structure_dict.txt",
+              "Path of dictionary.");
+
 // ocr forward related
 DEFINE_bool(det, true, "Whether use det in forward.");
 DEFINE_bool(rec, true, "Whether use rec in forward.");
-DEFINE_bool(cls, false, "Whether use cls in forward.");
\ No newline at end of file
+DEFINE_bool(cls, false, "Whether use cls in forward.");
+DEFINE_bool(table, false, "Whether use table structure in forward.");
\ No newline at end of file

deploy/cpp_infer/src/main.cpp

@@ -19,6 +19,7 @@
 
 #include <include/args.h>
 #include <include/paddleocr.h>
+#include <include/paddlestructure.h>
 
 using namespace PaddleOCR;
 
@@ -32,6 +33,12 @@ void check_params() {
     }
   }
   if (FLAGS_rec) {
+    std::cout
+        << "In PP-OCRv3, rec_image_shape parameter defaults to '3, 48, 320',"
+           "if you are using recognition model with PP-OCRv2 or an older "
+           "version, "
+           "please set --rec_image_shape='3,32,320"
+        << std::endl;
     if (FLAGS_rec_model_dir.empty() || FLAGS_image_dir.empty()) {
       std::cout << "Usage[rec]: ./ppocr "
                    "--rec_model_dir=/PATH/TO/REC_INFERENCE_MODEL/ "
@@ -47,6 +54,17 @@ void check_params() {
       exit(1);
     }
   }
+  if (FLAGS_table) {
+    if (FLAGS_table_model_dir.empty() || FLAGS_det_model_dir.empty() ||
+        FLAGS_rec_model_dir.empty() || FLAGS_image_dir.empty()) {
+      std::cout << "Usage[table]: ./ppocr "
+                << "--det_model_dir=/PATH/TO/DET_INFERENCE_MODEL/ "
+                << "--rec_model_dir=/PATH/TO/REC_INFERENCE_MODEL/ "
+                << "--table_model_dir=/PATH/TO/TABLE_INFERENCE_MODEL/ "
+                << "--image_dir=/PATH/TO/INPUT/IMAGE/" << std::endl;
+      exit(1);
+    }
+  }
   if (FLAGS_precision != "fp32" && FLAGS_precision != "fp16" &&
       FLAGS_precision != "int8") {
     cout << "precison should be 'fp32'(default), 'fp16' or 'int8'. " << endl;
@@ -54,21 +72,7 @@ void check_params() {
   }
 }
 
-int main(int argc, char **argv) {
-  // Parsing command-line
-  google::ParseCommandLineFlags(&argc, &argv, true);
-  check_params();
-
-  if (!Utility::PathExists(FLAGS_image_dir)) {
-    std::cerr << "[ERROR] image path not exist! image_dir: " << FLAGS_image_dir
-              << endl;
-    exit(1);
-  }
-
-  std::vector<cv::String> cv_all_img_names;
-  cv::glob(FLAGS_image_dir, cv_all_img_names);
-  std::cout << "total images num: " << cv_all_img_names.size() << endl;
-
+void ocr(std::vector<cv::String> &cv_all_img_names) {
   PPOCR ocr = PPOCR();
 
   std::vector<std::vector<OCRPredictResult>> ocr_results =
@@ -109,3 +113,49 @@ int main(int argc, char **argv) {
     }
   }
 }
+
+void structure(std::vector<cv::String> &cv_all_img_names) {
+  PaddleOCR::PaddleStructure engine = PaddleOCR::PaddleStructure();
+  std::vector<std::vector<StructurePredictResult>> structure_results =
+      engine.structure(cv_all_img_names, false, FLAGS_table);
+  for (int i = 0; i < cv_all_img_names.size(); i++) {
+    cout << "predict img: " << cv_all_img_names[i] << endl;
+    for (int j = 0; j < structure_results[i].size(); j++) {
+      std::cout << j << "\ttype: " << structure_results[i][j].type
+                << ", region: [";
+      std::cout << structure_results[i][j].box[0] << ","
+                << structure_results[i][j].box[1] << ","
+                << structure_results[i][j].box[2] << ","
+                << structure_results[i][j].box[3] << "], res: ";
+      if (structure_results[i][j].type == "table") {
+        std::cout << structure_results[i][j].html << std::endl;
+      } else {
+        Utility::print_result(structure_results[i][j].text_res);
+      }
+    }
+  }
+}
+
+int main(int argc, char **argv) {
+  // Parsing command-line
+  google::ParseCommandLineFlags(&argc, &argv, true);
+  check_params();
+
+  if (!Utility::PathExists(FLAGS_image_dir)) {
+    std::cerr << "[ERROR] image path not exist! image_dir: " << FLAGS_image_dir
+              << endl;
+    exit(1);
+  }
+
+  std::vector<cv::String> cv_all_img_names;
+  cv::glob(FLAGS_image_dir, cv_all_img_names);
+  std::cout << "total images num: " << cv_all_img_names.size() << endl;
+
+  if (FLAGS_type == "ocr") {
+    ocr(cv_all_img_names);
+  } else if (FLAGS_type == "structure") {
+    structure(cv_all_img_names);
+  } else {
+    std::cout << "only value in ['ocr','structure'] is supported" << endl;
+  }
+}

deploy/cpp_infer/src/ocr_det.cpp

@@ -47,7 +47,7 @@ void DBDetector::LoadModel(const std::string &model_dir) {
           {"elementwise_add_7", {1, 56, 2, 2}},
           {"nearest_interp_v2_0.tmp_0", {1, 256, 2, 2}}};
       std::map<std::string, std::vector<int>> max_input_shape = {
-          {"x", {1, 3, this->max_side_len_, this->max_side_len_}},
+          {"x", {1, 3, 1536, 1536}},
           {"conv2d_92.tmp_0", {1, 120, 400, 400}},
           {"conv2d_91.tmp_0", {1, 24, 200, 200}},
           {"conv2d_59.tmp_0", {1, 96, 400, 400}},
@@ -109,7 +109,8 @@ void DBDetector::Run(cv::Mat &img,
   img.copyTo(srcimg);
 
   auto preprocess_start = std::chrono::steady_clock::now();
-  this->resize_op_.Run(img, resize_img, this->max_side_len_, ratio_h, ratio_w,
+  this->resize_op_.Run(img, resize_img, this->limit_type_,
+                       this->limit_side_len_, ratio_h, ratio_w,
                        this->use_tensorrt_);
 
   this->normalize_op_.Run(&resize_img, this->mean_, this->scale_,

deploy/cpp_infer/src/paddleocr.cpp

@@ -23,10 +23,10 @@ PPOCR::PPOCR() {
   if (FLAGS_det) {
     this->detector_ = new DBDetector(
         FLAGS_det_model_dir, FLAGS_use_gpu, FLAGS_gpu_id, FLAGS_gpu_mem,
-        FLAGS_cpu_threads, FLAGS_enable_mkldnn, FLAGS_max_side_len,
-        FLAGS_det_db_thresh, FLAGS_det_db_box_thresh, FLAGS_det_db_unclip_ratio,
-        FLAGS_det_db_score_mode, FLAGS_use_dilation, FLAGS_use_tensorrt,
-        FLAGS_precision);
+        FLAGS_cpu_threads, FLAGS_enable_mkldnn, FLAGS_limit_type,
+        FLAGS_limit_side_len, FLAGS_det_db_thresh, FLAGS_det_db_box_thresh,
+        FLAGS_det_db_unclip_ratio, FLAGS_det_db_score_mode, FLAGS_use_dilation,
+        FLAGS_use_tensorrt, FLAGS_precision);
   }
 
   if (FLAGS_cls && FLAGS_use_angle_cls) {
@@ -56,7 +56,8 @@ void PPOCR::det(cv::Mat img, std::vector<OCRPredictResult> &ocr_results,
     res.box = boxes[i];
     ocr_results.push_back(res);
   }
-
+  // sort boex from top to bottom, from left to right
+  Utility::sorted_boxes(ocr_results);
   times[0] += det_times[0];
   times[1] += det_times[1];
   times[2] += det_times[2];

deploy/cpp_infer/src/paddlestructure.cpp

+// Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <include/args.h>
+#include <include/paddlestructure.h>
+
+#include "auto_log/autolog.h"
+#include <numeric>
+#include <sys/stat.h>
+
+namespace PaddleOCR {
+
+PaddleStructure::PaddleStructure() {
+  if (FLAGS_table) {
+    this->recognizer_ = new StructureTableRecognizer(
+        FLAGS_table_model_dir, FLAGS_use_gpu, FLAGS_gpu_id, FLAGS_gpu_mem,
+        FLAGS_cpu_threads, FLAGS_enable_mkldnn, FLAGS_table_char_dict_path,
+        FLAGS_use_tensorrt, FLAGS_precision, FLAGS_table_batch_num,
+        FLAGS_table_max_len);
+  }
+};
+
+std::vector<std::vector<StructurePredictResult>>
+PaddleStructure::structure(std::vector<cv::String> cv_all_img_names,
+                           bool layout, bool table) {
+  std::vector<double> time_info_det = {0, 0, 0};
+  std::vector<double> time_info_rec = {0, 0, 0};
+  std::vector<double> time_info_cls = {0, 0, 0};
+  std::vector<double> time_info_table = {0, 0, 0};
+
+  std::vector<std::vector<StructurePredictResult>> structure_results;
+
+  if (!Utility::PathExists(FLAGS_output) && FLAGS_det) {
+    mkdir(FLAGS_output.c_str(), 0777);
+  }
+  for (int i = 0; i < cv_all_img_names.size(); ++i) {
+    std::vector<StructurePredictResult> structure_result;
+    cv::Mat srcimg = cv::imread(cv_all_img_names[i], cv::IMREAD_COLOR);
+    if (!srcimg.data) {
+      std::cerr << "[ERROR] image read failed! image path: "
+                << cv_all_img_names[i] << endl;
+      exit(1);
+    }
+    if (layout) {
+    } else {
+      StructurePredictResult res;
+      res.type = "table";
+      res.box = std::vector<int>(4, 0);
+      res.box[2] = srcimg.cols;
+      res.box[3] = srcimg.rows;
+      structure_result.push_back(res);
+    }
+    cv::Mat roi_img;
+    for (int i = 0; i < structure_result.size(); i++) {
+      // crop image
+      roi_img = Utility::crop_image(srcimg, structure_result[i].box);
+      if (structure_result[i].type == "table") {
+        this->table(roi_img, structure_result[i], time_info_table,
+                    time_info_det, time_info_rec, time_info_cls);
+      }
+    }
+    structure_results.push_back(structure_result);
+  }
+  return structure_results;
+};
+
+void PaddleStructure::table(cv::Mat img,
+                            StructurePredictResult &structure_result,
+                            std::vector<double> &time_info_table,
+                            std::vector<double> &time_info_det,
+                            std::vector<double> &time_info_rec,
+                            std::vector<double> &time_info_cls) {
+  // predict structure
+  std::vector<std::vector<std::string>> structure_html_tags;
+  std::vector<float> structure_scores(1, 0);
+  std::vector<std::vector<std::vector<std::vector<int>>>> structure_boxes;
+  std::vector<double> structure_imes;
+  std::vector<cv::Mat> img_list;
+  img_list.push_back(img);
+  this->recognizer_->Run(img_list, structure_html_tags, structure_scores,
+                         structure_boxes, structure_imes);
+  time_info_table[0] += structure_imes[0];
+  time_info_table[1] += structure_imes[1];
+  time_info_table[2] += structure_imes[2];
+
+  std::vector<OCRPredictResult> ocr_result;
+  std::string html;
+  int expand_pixel = 3;
+
+  for (int i = 0; i < img_list.size(); i++) {
+    // det
+    this->det(img_list[i], ocr_result, time_info_det);
+    // crop image
+    std::vector<cv::Mat> rec_img_list;
+    for (int j = 0; j < ocr_result.size(); j++) {
+      int x_collect[4] = {ocr_result[j].box[0][0], ocr_result[j].box[1][0],
+                          ocr_result[j].box[2][0], ocr_result[j].box[3][0]};
+      int y_collect[4] = {ocr_result[j].box[0][1], ocr_result[j].box[1][1],
+                          ocr_result[j].box[2][1], ocr_result[j].box[3][1]};
+      int left = int(*std::min_element(x_collect, x_collect + 4));
+      int right = int(*std::max_element(x_collect, x_collect + 4));
+      int top = int(*std::min_element(y_collect, y_collect + 4));
+      int bottom = int(*std::max_element(y_collect, y_collect + 4));
+      std::vector<int> box{max(0, left - expand_pixel),
+                           max(0, top - expand_pixel),
+                           min(img_list[i].cols, right + expand_pixel),
+                           min(img_list[i].rows, bottom + expand_pixel)};
+      cv::Mat crop_img = Utility::crop_image(img_list[i], box);
+      rec_img_list.push_back(crop_img);
+    }
+    // rec
+    this->rec(rec_img_list, ocr_result, time_info_rec);
+    // rebuild table
+    html = this->rebuild_table(structure_html_tags[i], structure_boxes[i],
+                               ocr_result);
+    structure_result.html = html;
+    structure_result.html_score = structure_scores[i];
+  }
+};
+
+std::string PaddleStructure::rebuild_table(
+    std::vector<std::string> structure_html_tags,
+    std::vector<std::vector<std::vector<int>>> structure_boxes,
+    std::vector<OCRPredictResult> &ocr_result) {
+  // match text in same cell
+  std::vector<std::vector<string>> matched(structure_boxes.size(),
+                                           std::vector<std::string>());
+
+  for (int i = 0; i < ocr_result.size(); i++) {
+    std::vector<std::vector<float>> dis_list(structure_boxes.size(),
+                                             std::vector<float>(3, 100000.0));
+    for (int j = 0; j < structure_boxes.size(); j++) {
+      int x_collect[4] = {ocr_result[i].box[0][0], ocr_result[i].box[1][0],
+                          ocr_result[i].box[2][0], ocr_result[i].box[3][0]};
+      int y_collect[4] = {ocr_result[i].box[0][1], ocr_result[i].box[1][1],
+                          ocr_result[i].box[2][1], ocr_result[i].box[3][1]};
+      int left = int(*std::min_element(x_collect, x_collect + 4));
+      int right = int(*std::max_element(x_collect, x_collect + 4));
+      int top = int(*std::min_element(y_collect, y_collect + 4));
+      int bottom = int(*std::max_element(y_collect, y_collect + 4));
+      std::vector<std::vector<int>> box(2, std::vector<int>(2, 0));
+      box[0][0] = left - 1;
+      box[0][1] = top - 1;
+      box[1][0] = right + 1;
+      box[1][1] = bottom + 1;
+
+      dis_list[j][0] = this->dis(box, structure_boxes[j]);
+      dis_list[j][1] = 1 - this->iou(box, structure_boxes[j]);
+      dis_list[j][2] = j;
+    }
+    // find min dis idx
+    std::sort(dis_list.begin(), dis_list.end(),
+              PaddleStructure::comparison_dis);
+    matched[dis_list[0][2]].push_back(ocr_result[i].text);
+  }
+  // get pred html
+  std::string html_str = "";
+  int td_tag_idx = 0;
+  for (int i = 0; i < structure_html_tags.size(); i++) {
+    if (structure_html_tags[i].find("</td>") != std::string::npos) {
+      if (structure_html_tags[i].find("<td></td>") != std::string::npos) {
+        html_str += "<td>";
+      }
+      if (matched[td_tag_idx].size() > 0) {
+        bool b_with = false;
+        if (matched[td_tag_idx][0].find("<b>") != std::string::npos &&
+            matched[td_tag_idx].size() > 1) {
+          b_with = true;
+          html_str += "<b>";
+        }
+        for (int j = 0; j < matched[td_tag_idx].size(); j++) {
+          std::string content = matched[td_tag_idx][j];
+          if (matched[td_tag_idx].size() > 1) {
+            // remove blank, <b> and </b>
+            if (content.length() > 0 && content.at(0) == ' ') {
+              content = content.substr(0);
+            }
+            if (content.length() > 2 && content.substr(0, 3) == "<b>") {
+              content = content.substr(3);
+            }
+            if (content.length() > 4 &&
+                content.substr(content.length() - 4) == "</b>") {
+              content = content.substr(0, content.length() - 4);
+            }
+            if (content.empty()) {
+              continue;
+            }
+            // add blank
+            if (j != matched[td_tag_idx].size() - 1 &&
+                content.at(content.length() - 1) != ' ') {
+              content += ' ';
+            }
+          }
+          html_str += content;
+        }
+        if (b_with) {
+          html_str += "</b>";
+        }
+      }
+      if (structure_html_tags[i].find("<td></td>") != std::string::npos) {
+        html_str += "</td>";
+      } else {
+        html_str += structure_html_tags[i];
+      }
+      td_tag_idx += 1;
+    } else {
+      html_str += structure_html_tags[i];
+    }
+  }
+  return html_str;
+}
+
+float PaddleStructure::iou(std::vector<std::vector<int>> &box1,
+                           std::vector<std::vector<int>> &box2) {
+  int area1 = max(0, box1[1][0] - box1[0][0]) * max(0, box1[1][1] - box1[0][1]);
+  int area2 = max(0, box2[1][0] - box2[0][0]) * max(0, box2[1][1] - box2[0][1]);
+
+  // computing the sum_area
+  int sum_area = area1 + area2;
+
+  // find the each point of intersect rectangle
+  int x1 = max(box1[0][0], box2[0][0]);
+  int y1 = max(box1[0][1], box2[0][1]);
+  int x2 = min(box1[1][0], box2[1][0]);
+  int y2 = min(box1[1][1], box2[1][1]);
+
+  // judge if there is an intersect
+  if (y1 >= y2 || x1 >= x2) {
+    return 0.0;
+  } else {
+    int intersect = (x2 - x1) * (y2 - y1);
+    return intersect / (sum_area - intersect + 0.00000001);
+  }
+}
+
+float PaddleStructure::dis(std::vector<std::vector<int>> &box1,
+                           std::vector<std::vector<int>> &box2) {
+  int x1_1 = box1[0][0];
+  int y1_1 = box1[0][1];
+  int x2_1 = box1[1][0];
+  int y2_1 = box1[1][1];
+
+  int x1_2 = box2[0][0];
+  int y1_2 = box2[0][1];
+  int x2_2 = box2[1][0];
+  int y2_2 = box2[1][1];
+
+  float dis =
+      abs(x1_2 - x1_1) + abs(y1_2 - y1_1) + abs(x2_2 - x2_1) + abs(y2_2 - y2_1);
+  float dis_2 = abs(x1_2 - x1_1) + abs(y1_2 - y1_1);
+  float dis_3 = abs(x2_2 - x2_1) + abs(y2_2 - y2_1);
+  return dis + min(dis_2, dis_3);
+}
+
+PaddleStructure::~PaddleStructure() {
+  if (this->recognizer_ != nullptr) {
+    delete this->recognizer_;
+  }
+};
+
+} // namespace PaddleOCR
\ No newline at end of file

deploy/cpp_infer/src/postprocess_op.cpp

@@ -17,8 +17,8 @@
 
 namespace PaddleOCR {
 
-void PostProcessor::GetContourArea(const std::vector<std::vector<float>> &box,
-                                   float unclip_ratio, float &distance) {
+void DBPostProcessor::GetContourArea(const std::vector<std::vector<float>> &box,
+                                     float unclip_ratio, float &distance) {
   int pts_num = 4;
   float area = 0.0f;
   float dist = 0.0f;
@@ -35,8 +35,8 @@ void PostProcessor::GetContourArea(const std::vector<std::vector<float>> &box,
   distance = area * unclip_ratio / dist;
 }
 
-cv::RotatedRect PostProcessor::UnClip(std::vector<std::vector<float>> box,
-                                      const float &unclip_ratio) {
+cv::RotatedRect DBPostProcessor::UnClip(std::vector<std::vector<float>> box,
+                                        const float &unclip_ratio) {
   float distance = 1.0;
 
   GetContourArea(box, unclip_ratio, distance);
@@ -67,7 +67,7 @@ cv::RotatedRect PostProcessor::UnClip(std::vector<std::vector<float>> box,
   return res;
 }
 
-float **PostProcessor::Mat2Vec(cv::Mat mat) {
+float **DBPostProcessor::Mat2Vec(cv::Mat mat) {
   auto **array = new float *[mat.rows];
   for (int i = 0; i < mat.rows; ++i)
     array[i] = new float[mat.cols];
@@ -81,7 +81,7 @@ float **PostProcessor::Mat2Vec(cv::Mat mat) {
 }
 
 std::vector<std::vector<int>>
-PostProcessor::OrderPointsClockwise(std::vector<std::vector<int>> pts) {
+DBPostProcessor::OrderPointsClockwise(std::vector<std::vector<int>> pts) {
   std::vector<std::vector<int>> box = pts;
   std::sort(box.begin(), box.end(), XsortInt);
 
@@ -99,7 +99,7 @@ PostProcessor::OrderPointsClockwise(std::vector<std::vector<int>> pts) {
   return rect;
 }
 
-std::vector<std::vector<float>> PostProcessor::Mat2Vector(cv::Mat mat) {
+std::vector<std::vector<float>> DBPostProcessor::Mat2Vector(cv::Mat mat) {
   std::vector<std::vector<float>> img_vec;
   std::vector<float> tmp;
 
@@ -113,20 +113,20 @@ std::vector<std::vector<float>> PostProcessor::Mat2Vector(cv::Mat mat) {
   return img_vec;
 }
 
-bool PostProcessor::XsortFp32(std::vector<float> a, std::vector<float> b) {
+bool DBPostProcessor::XsortFp32(std::vector<float> a, std::vector<float> b) {
   if (a[0] != b[0])
     return a[0] < b[0];
   return false;
 }
 
-bool PostProcessor::XsortInt(std::vector<int> a, std::vector<int> b) {
+bool DBPostProcessor::XsortInt(std::vector<int> a, std::vector<int> b) {
   if (a[0] != b[0])
     return a[0] < b[0];
   return false;
 }
 
-std::vector<std::vector<float>> PostProcessor::GetMiniBoxes(cv::RotatedRect box,
-                                                            float &ssid) {
+std::vector<std::vector<float>>
+DBPostProcessor::GetMiniBoxes(cv::RotatedRect box, float &ssid) {
   ssid = std::max(box.size.width, box.size.height);
 
   cv::Mat points;
@@ -160,8 +160,8 @@ std::vector<std::vector<float>> PostProcessor::GetMiniBoxes(cv::RotatedRect box,
   return array;
 }
 
-float PostProcessor::PolygonScoreAcc(std::vector<cv::Point> contour,
-                                     cv::Mat pred) {
+float DBPostProcessor::PolygonScoreAcc(std::vector<cv::Point> contour,
+                                       cv::Mat pred) {
   int width = pred.cols;
   int height = pred.rows;
   std::vector<float> box_x;
@@ -206,8 +206,8 @@ float PostProcessor::PolygonScoreAcc(std::vector<cv::Point> contour,
   return score;
 }
 
-float PostProcessor::BoxScoreFast(std::vector<std::vector<float>> box_array,
-                                  cv::Mat pred) {
+float DBPostProcessor::BoxScoreFast(std::vector<std::vector<float>> box_array,
+                                    cv::Mat pred) {
   auto array = box_array;
   int width = pred.cols;
   int height = pred.rows;
@@ -244,7 +244,7 @@ float PostProcessor::BoxScoreFast(std::vector<std::vector<float>> box_array,
   return score;
 }
 
-std::vector<std::vector<std::vector<int>>> PostProcessor::BoxesFromBitmap(
+std::vector<std::vector<std::vector<int>>> DBPostProcessor::BoxesFromBitmap(
     const cv::Mat pred, const cv::Mat bitmap, const float &box_thresh,
     const float &det_db_unclip_ratio, const std::string &det_db_score_mode) {
   const int min_size = 3;
@@ -321,9 +321,9 @@ std::vector<std::vector<std::vector<int>>> PostProcessor::BoxesFromBitmap(
   return boxes;
 }
 
-std::vector<std::vector<std::vector<int>>>
-PostProcessor::FilterTagDetRes(std::vector<std::vector<std::vector<int>>> boxes,
-                               float ratio_h, float ratio_w, cv::Mat srcimg) {
+std::vector<std::vector<std::vector<int>>> DBPostProcessor::FilterTagDetRes(
+    std::vector<std::vector<std::vector<int>>> boxes, float ratio_h,
+    float ratio_w, cv::Mat srcimg) {
   int oriimg_h = srcimg.rows;
   int oriimg_w = srcimg.cols;
 
@@ -352,4 +352,77 @@ PostProcessor::FilterTagDetRes(std::vector<std::vector<std::vector<int>>> boxes,
   return root_points;
 }
 
+void TablePostProcessor::init(std::string label_path) {
+  this->label_list_ = Utility::ReadDict(label_path);
+  this->label_list_.insert(this->label_list_.begin(), this->beg);
+  this->label_list_.push_back(this->end);
+}
+
+void TablePostProcessor::Run(
+    std::vector<float> &loc_preds, std::vector<float> &structure_probs,
+    std::vector<float> &rec_scores, std::vector<int> &loc_preds_shape,
+    std::vector<int> &structure_probs_shape,
+    std::vector<std::vector<std::string>> &rec_html_tag_batch,
+    std::vector<std::vector<std::vector<std::vector<int>>>> &rec_boxes_batch,
+    std::vector<int> &width_list, std::vector<int> &height_list) {
+  for (int batch_idx = 0; batch_idx < structure_probs_shape[0]; batch_idx++) {
+    // image tags and boxs
+    std::vector<std::string> rec_html_tags;
+    std::vector<std::vector<std::vector<int>>> rec_boxes;
+
+    float score = 0.f;
+    int count = 0;
+    float char_score = 0.f;
+    int char_idx = 0;
+
+    // step
+    for (int step_idx = 0; step_idx < structure_probs_shape[1]; step_idx++) {
+      std::string html_tag;
+      std::vector<std::vector<int>> rec_box;
+      // html tag
+      int step_start_idx = (batch_idx * structure_probs_shape[1] + step_idx) *
+                           structure_probs_shape[2];
+      char_idx = int(Utility::argmax(
+          &structure_probs[step_start_idx],
+          &structure_probs[step_start_idx + structure_probs_shape[2]]));
+      char_score = float(*std::max_element(
+          &structure_probs[step_start_idx],
+          &structure_probs[step_start_idx + structure_probs_shape[2]]));
+      html_tag = this->label_list_[char_idx];
+
+      if (step_idx > 0 && html_tag == this->end) {
+        break;
+      }
+      if (html_tag == this->beg) {
+        continue;
+      }
+      count += 1;
+      score += char_score;
+      rec_html_tags.push_back(html_tag);
+      // box
+      if (html_tag == "<td>" || html_tag == "<td" || html_tag == "<td></td>") {
+        for (int point_idx = 0; point_idx < loc_preds_shape[2];
+             point_idx += 2) {
+          std::vector<int> point(2, 0);
+          step_start_idx = (batch_idx * structure_probs_shape[1] + step_idx) *
+                               loc_preds_shape[2] +
+                           point_idx;
+          point[0] = int(loc_preds[step_start_idx] * width_list[batch_idx]);
+          point[1] =
+              int(loc_preds[step_start_idx + 1] * height_list[batch_idx]);
+          rec_box.push_back(point);
+        }
+        rec_boxes.push_back(rec_box);
+      }
+    }
+    score /= count;
+    if (isnan(score) || rec_boxes.size() == 0) {
+      score = -1;
+    }
+    rec_scores.push_back(score);
+    rec_boxes_batch.push_back(rec_boxes);
+    rec_html_tag_batch.push_back(rec_html_tags);
+  }
+}
+
 } // namespace PaddleOCR

deploy/cpp_infer/src/preprocess_op.cpp

@@ -69,18 +69,28 @@ void Normalize::Run(cv::Mat *im, const std::vector<float> &mean,
 }
 
 void ResizeImgType0::Run(const cv::Mat &img, cv::Mat &resize_img,
-                         int max_size_len, float &ratio_h, float &ratio_w,
-                         bool use_tensorrt) {
+                         string limit_type, int limit_side_len, float &ratio_h,
+                         float &ratio_w, bool use_tensorrt) {
   int w = img.cols;
   int h = img.rows;
-
   float ratio = 1.f;
-  int max_wh = w >= h ? w : h;
-  if (max_wh > max_size_len) {
-    if (h > w) {
-      ratio = float(max_size_len) / float(h);
-    } else {
-      ratio = float(max_size_len) / float(w);
+  if (limit_type == "min") {
+    int min_wh = min(h, w);
+    if (min_wh < limit_side_len) {
+      if (h < w) {
+        ratio = float(limit_side_len) / float(h);
+      } else {
+        ratio = float(limit_side_len) / float(w);
+      }
+    }
+  } else {
+    int max_wh = max(h, w);
+    if (max_wh > limit_side_len) {
+      if (h > w) {
+        ratio = float(limit_side_len) / float(h);
+      } else {
+        ratio = float(limit_side_len) / float(w);
+      }
     }
   }
 
@@ -143,4 +153,26 @@ void ClsResizeImg::Run(const cv::Mat &img, cv::Mat &resize_img,
   }
 }
 
+void TableResizeImg::Run(const cv::Mat &img, cv::Mat &resize_img,
+                         const int max_len) {
+  int w = img.cols;
+  int h = img.rows;
+
+  int max_wh = w >= h ? w : h;
+  float ratio = w >= h ? float(max_len) / float(w) : float(max_len) / float(h);
+
+  int resize_h = int(float(h) * ratio);
+  int resize_w = int(float(w) * ratio);
+
+  cv::resize(img, resize_img, cv::Size(resize_w, resize_h));
+}
+
+void TablePadImg::Run(const cv::Mat &img, cv::Mat &resize_img,
+                      const int max_len) {
+  int w = img.cols;
+  int h = img.rows;
+  cv::copyMakeBorder(img, resize_img, 0, max_len - h, 0, max_len - w,
+                     cv::BORDER_CONSTANT, cv::Scalar(0, 0, 0));
+}
+
 } // namespace PaddleOCR

deploy/cpp_infer/src/structure_table.cpp

+// Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <include/structure_table.h>
+
+namespace PaddleOCR {
+
+void StructureTableRecognizer::Run(
+    std::vector<cv::Mat> img_list,
+    std::vector<std::vector<std::string>> &structure_html_tags,
+    std::vector<float> &structure_scores,
+    std::vector<std::vector<std::vector<std::vector<int>>>> &structure_boxes,
+    std::vector<double> &times) {
+  std::chrono::duration<float> preprocess_diff =
+      std::chrono::steady_clock::now() - std::chrono::steady_clock::now();
+  std::chrono::duration<float> inference_diff =
+      std::chrono::steady_clock::now() - std::chrono::steady_clock::now();
+  std::chrono::duration<float> postprocess_diff =
+      std::chrono::steady_clock::now() - std::chrono::steady_clock::now();
+
+  int img_num = img_list.size();
+  for (int beg_img_no = 0; beg_img_no < img_num;
+       beg_img_no += this->table_batch_num_) {
+    // preprocess
+    auto preprocess_start = std::chrono::steady_clock::now();
+    int end_img_no = min(img_num, beg_img_no + this->table_batch_num_);
+    int batch_num = end_img_no - beg_img_no;
+    std::vector<cv::Mat> norm_img_batch;
+    std::vector<int> width_list;
+    std::vector<int> height_list;
+    for (int ino = beg_img_no; ino < end_img_no; ino++) {
+      cv::Mat srcimg;
+      img_list[ino].copyTo(srcimg);
+      cv::Mat resize_img;
+      cv::Mat pad_img;
+      this->resize_op_.Run(srcimg, resize_img, this->table_max_len_);
+      this->normalize_op_.Run(&resize_img, this->mean_, this->scale_,
+                              this->is_scale_);
+      this->pad_op_.Run(resize_img, pad_img, this->table_max_len_);
+      norm_img_batch.push_back(pad_img);
+      width_list.push_back(srcimg.cols);
+      height_list.push_back(srcimg.rows);
+    }
+
+    std::vector<float> input(
+        batch_num * 3 * this->table_max_len_ * this->table_max_len_, 0.0f);
+    this->permute_op_.Run(norm_img_batch, input.data());
+    auto preprocess_end = std::chrono::steady_clock::now();
+    preprocess_diff += preprocess_end - preprocess_start;
+    // inference.
+    auto input_names = this->predictor_->GetInputNames();
+    auto input_t = this->predictor_->GetInputHandle(input_names[0]);
+    input_t->Reshape(
+        {batch_num, 3, this->table_max_len_, this->table_max_len_});
+    auto inference_start = std::chrono::steady_clock::now();
+    input_t->CopyFromCpu(input.data());
+    this->predictor_->Run();
+    auto output_names = this->predictor_->GetOutputNames();
+    auto output_tensor0 = this->predictor_->GetOutputHandle(output_names[0]);
+    auto output_tensor1 = this->predictor_->GetOutputHandle(output_names[1]);
+    std::vector<int> predict_shape0 = output_tensor0->shape();
+    std::vector<int> predict_shape1 = output_tensor1->shape();
+
+    int out_num0 = std::accumulate(predict_shape0.begin(), predict_shape0.end(),
+                                   1, std::multiplies<int>());
+    int out_num1 = std::accumulate(predict_shape1.begin(), predict_shape1.end(),
+                                   1, std::multiplies<int>());
+    std::vector<float> loc_preds;
+    std::vector<float> structure_probs;
+    loc_preds.resize(out_num0);
+    structure_probs.resize(out_num1);
+
+    output_tensor0->CopyToCpu(loc_preds.data());
+    output_tensor1->CopyToCpu(structure_probs.data());
+    auto inference_end = std::chrono::steady_clock::now();
+    inference_diff += inference_end - inference_start;
+    // postprocess
+    auto postprocess_start = std::chrono::steady_clock::now();
+    std::vector<std::vector<std::string>> structure_html_tag_batch;
+    std::vector<float> structure_score_batch;
+    std::vector<std::vector<std::vector<std::vector<int>>>>
+        structure_boxes_batch;
+    this->post_processor_.Run(loc_preds, structure_probs, structure_score_batch,
+                              predict_shape0, predict_shape1,
+                              structure_html_tag_batch, structure_boxes_batch,
+                              width_list, height_list);
+    for (int m = 0; m < predict_shape0[0]; m++) {
+
+      structure_html_tag_batch[m].insert(structure_html_tag_batch[m].begin(),
+                                         "<table>");
+      structure_html_tag_batch[m].insert(structure_html_tag_batch[m].begin(),
+                                         "<body>");
+      structure_html_tag_batch[m].insert(structure_html_tag_batch[m].begin(),
+                                         "<html>");
+      structure_html_tag_batch[m].push_back("</table>");
+      structure_html_tag_batch[m].push_back("</body>");
+      structure_html_tag_batch[m].push_back("</html>");
+      structure_html_tags.push_back(structure_html_tag_batch[m]);
+      structure_scores.push_back(structure_score_batch[m]);
+      structure_boxes.push_back(structure_boxes_batch[m]);
+    }
+    auto postprocess_end = std::chrono::steady_clock::now();
+    postprocess_diff += postprocess_end - postprocess_start;
+    times.push_back(double(preprocess_diff.count() * 1000));
+    times.push_back(double(inference_diff.count() * 1000));
+    times.push_back(double(postprocess_diff.count() * 1000));
+  }
+}
+
+void StructureTableRecognizer::LoadModel(const std::string &model_dir) {
+  AnalysisConfig config;
+  config.SetModel(model_dir + "/inference.pdmodel",
+                  model_dir + "/inference.pdiparams");
+
+  if (this->use_gpu_) {
+    config.EnableUseGpu(this->gpu_mem_, this->gpu_id_);
+    if (this->use_tensorrt_) {
+      auto precision = paddle_infer::Config::Precision::kFloat32;
+      if (this->precision_ == "fp16") {
+        precision = paddle_infer::Config::Precision::kHalf;
+      }
+      if (this->precision_ == "int8") {
+        precision = paddle_infer::Config::Precision::kInt8;
+      }
+      config.EnableTensorRtEngine(1 << 20, 10, 3, precision, false, false);
+    }
+  } else {
+    config.DisableGpu();
+    if (this->use_mkldnn_) {
+      config.EnableMKLDNN();
+    }
+    config.SetCpuMathLibraryNumThreads(this->cpu_math_library_num_threads_);
+  }
+
+  // false for zero copy tensor
+  config.SwitchUseFeedFetchOps(false);
+  // true for multiple input
+  config.SwitchSpecifyInputNames(true);
+
+  config.SwitchIrOptim(true);
+
+  config.EnableMemoryOptim();
+  config.DisableGlogInfo();
+
+  this->predictor_ = CreatePredictor(config);
+}
+} // namespace PaddleOCR

deploy/cpp_infer/src/utility.cpp

@@ -248,4 +248,33 @@ void Utility::print_result(const std::vector<OCRPredictResult> &ocr_result) {
     std::cout << std::endl;
   }
 }
+
+cv::Mat Utility::crop_image(cv::Mat &img, std::vector<int> &area) {
+  cv::Mat crop_im;
+  int crop_x1 = std::max(0, area[0]);
+  int crop_y1 = std::max(0, area[1]);
+  int crop_x2 = std::min(img.cols - 1, area[2] - 1);
+  int crop_y2 = std::min(img.rows - 1, area[3] - 1);
+
+  crop_im = cv::Mat::zeros(area[3] - area[1], area[2] - area[0], 16);
+  cv::Mat crop_im_window =
+      crop_im(cv::Range(crop_y1 - area[1], crop_y2 + 1 - area[1]),
+              cv::Range(crop_x1 - area[0], crop_x2 + 1 - area[0]));
+  cv::Mat roi_img =
+      img(cv::Range(crop_y1, crop_y2 + 1), cv::Range(crop_x1, crop_x2 + 1));
+  crop_im_window += roi_img;
+  return crop_im;
+}
+
+void Utility::sorted_boxes(std::vector<OCRPredictResult> &ocr_result) {
+  std::sort(ocr_result.begin(), ocr_result.end(), Utility::comparison_box);
+
+  for (int i = 0; i < ocr_result.size() - 1; i++) {
+    if (abs(ocr_result[i + 1].box[0][1] - ocr_result[i].box[0][1]) < 10 &&
+        (ocr_result[i + 1].box[0][0] < ocr_result[i].box[0][0])) {
+      std::swap(ocr_result[i], ocr_result[i + 1]);
+    }
+  }
+}
+
 } // namespace PaddleOCR
\ No newline at end of file

deploy/hubserving/ocr_system/module.py

@@ -118,7 +118,7 @@ class OCRSystem(hub.Module):
                 all_results.append([])
                 continue
             starttime = time.time()
-            dt_boxes, rec_res = self.text_sys(img)
+            dt_boxes, rec_res, _ = self.text_sys(img)
             elapse = time.time() - starttime
             logger.info("Predict time: {}".format(elapse))
 

deploy/hubserving/readme.md

@@ -59,7 +59,8 @@ pip3 install paddlehub==2.1.0 --upgrade -i https://mirror.baidu.com/pypi/simple
 检测模型：./inference/ch_PP-OCRv3_det_infer/
 识别模型：./inference/ch_PP-OCRv3_rec_infer/
 方向分类器：./inference/ch_ppocr_mobile_v2.0_cls_infer/
-表格结构识别模型：./inference/en_ppocr_mobile_v2.0_table_structure_infer/
+版面分析模型：./inference/layout_infer/
+表格结构识别模型：./inference/ch_ppstructure_mobile_v2.0_SLANet_infer/
 ```  
 
 **模型路径可在`params.py`中查看和修改。** 更多模型可以从PaddleOCR提供的模型库[PP-OCR](../../doc/doc_ch/models_list.md)和[PP-Structure](../../ppstructure/docs/models_list.md)下载，也可以替换成自己训练转换好的模型。
@@ -172,7 +173,7 @@ hub serving start -c deploy/hubserving/ocr_system/config.json
 ## 3. 发送预测请求
 配置好服务端，可使用以下命令发送预测请求，获取预测结果:  
 
-```python tools/test_hubserving.py server_url image_path```  
+```python tools/test_hubserving.py --server_url=server_url --image_dir=image_path```  
 
 需要给脚本传递2个参数：  
 - **server_url**：服务地址，格式为  

deploy/hubserving/readme_en.md

@@ -61,7 +61,8 @@ Before installing the service module, you need to prepare the inference model an
 text detection model: ./inference/ch_PP-OCRv3_det_infer/
 text recognition model: ./inference/ch_PP-OCRv3_rec_infer/
 text angle classifier: ./inference/ch_ppocr_mobile_v2.0_cls_infer/
-tanle recognition: ./inference/en_ppocr_mobile_v2.0_table_structure_infer/
+layout parse model: ./inference/layout_infer/
+tanle recognition: ./inference/ch_ppstructure_mobile_v2.0_SLANet_infer/
 ```  
 
 **The model path can be found and modified in `params.py`.** More models provided by PaddleOCR can be obtained from the [model library](../../doc/doc_en/models_list_en.md). You can also use models trained by yourself.
@@ -177,7 +178,7 @@ hub serving start -c deploy/hubserving/ocr_system/config.json
 ## 3. Send prediction requests
 After the service starts, you can use the following command to send a prediction request to obtain the prediction result:  
 ```shell
-python tools/test_hubserving.py server_url image_path
+python tools/test_hubserving.py --server_url=server_url --image_dir=image_path
 ```  
 
 Two parameters need to be passed to the script:

deploy/hubserving/structure_system/module.py

@@ -119,7 +119,7 @@ class StructureSystem(hub.Module):
                 all_results.append([])
                 continue
             starttime = time.time()
-            res = self.table_sys(img)
+            res, _ = self.table_sys(img)
             elapse = time.time() - starttime
             logger.info("Predict time: {}".format(elapse))
 
@@ -144,6 +144,6 @@ class StructureSystem(hub.Module):
 if __name__ == '__main__':
     structure_system = StructureSystem()
     structure_system._initialize()
-    image_path = ['./doc/table/1.png']
+    image_path = ['./ppstructure/docs/table/1.png']
     res = structure_system.predict(paths=image_path)
     print(res)

deploy/hubserving/structure_system/params.py

@@ -23,8 +23,10 @@ def read_params():
     cfg = table_read_params()
 
     # params for layout parser model
-    cfg.layout_path_model = 'lp://PubLayNet/ppyolov2_r50vd_dcn_365e_publaynet/config'
-    cfg.layout_label_map = None
+    cfg.layout_model_dir = ''
+    cfg.layout_dict_path = './ppocr/utils/dict/layout_publaynet_dict.txt'
+    cfg.layout_score_threshold = 0.5
+    cfg.layout_nms_threshold = 0.5
 
     cfg.mode = 'structure'
     cfg.output = './output'

deploy/hubserving/structure_table/module.py

@@ -118,11 +118,11 @@ class TableSystem(hub.Module):
                 all_results.append([])
                 continue
             starttime = time.time()
-            pred_html = self.table_sys(img)
+            res, _ = self.table_sys(img)
             elapse = time.time() - starttime
             logger.info("Predict time: {}".format(elapse))
 
-            all_results.append({'html': pred_html})
+            all_results.append({'html': res['html']})
         return all_results
 
     @serving
@@ -138,6 +138,6 @@ class TableSystem(hub.Module):
 if __name__ == '__main__':
     table_system = TableSystem()
     table_system._initialize()
-    image_path = ['./doc/table/table.jpg']
+    image_path = ['./ppstructure/docs/table/table.jpg']
     res = table_system.predict(paths=image_path)
     print(res)

doc/doc_ch/algorithm_rec_sar.md

@@ -79,7 +79,7 @@ python3 tools/export_model.py -c configs/rec/rec_r31_sar.yml -o Global.pretraine
 SAR文本识别模型推理，可以执行如下命令：
 
 ```
-python3 tools/infer/predict_rec.py --image_dir="./doc/imgs_words/en/word_1.png" --rec_model_dir="./inference/rec_sar/" --rec_image_shape="3, 48, 48, 160" --rec_char_type="ch" --rec_algorithm="SAR" --rec_char_dict_path="ppocr/utils/dict90.txt" --max_text_length=30 --use_space_char=False
+python3 tools/infer/predict_rec.py --image_dir="./doc/imgs_words/en/word_1.png" --rec_model_dir="./inference/rec_sar/" --rec_image_shape="3, 48, 48, 160" --rec_algorithm="SAR" --rec_char_dict_path="ppocr/utils/dict90.txt" --max_text_length=30 --use_space_char=False
 ```
 
 <a name="4-2"></a>

doc/doc_ch/dataset/table_datasets.md

@@ -3,6 +3,7 @@
 - [数据集汇总](#数据集汇总)
 - [1. PubTabNet数据集](#1-pubtabnet数据集)
 - [2. 好未来表格识别竞赛数据集](#2-好未来表格识别竞赛数据集)
+- [3. 好未来表格识别竞赛数据集](#2-WTW中文场景表格数据集)
 
 这里整理了常用表格识别数据集，持续更新中，欢迎各位小伙伴贡献数据集～
 
@@ -12,6 +13,7 @@
 |---|---|---|
 | PubTabNet |https://github.com/ibm-aur-nlp/PubTabNet| jsonl格式，可直接用[pubtab_dataset.py](../../../ppocr/data/pubtab_dataset.py)加载 |
 | 好未来表格识别竞赛数据集 |https://ai.100tal.com/dataset| jsonl格式，可直接用[pubtab_dataset.py](../../../ppocr/data/pubtab_dataset.py)加载 |
+| WTW中文场景表格数据集 |https://github.com/wangwen-whu/WTW-Dataset| 需要进行转换后才能用[pubtab_dataset.py](../../../ppocr/data/pubtab_dataset.py)加载 |
 
 ## 1. PubTabNet数据集
 - **数据简介**：PubTabNet数据集的训练集合中包含50万张图像，验证集合中包含0.9万张图像。部分图像可视化如下所示。
@@ -31,3 +33,12 @@
     <img src="../../datasets/table_tal_demo/1.jpg" width="500">
     <img src="../../datasets/table_tal_demo/2.jpg" width="500">
 </div>
+
+## 3. WTW中文场景表格数据集
+- **数据简介**：WTW中文场景表格数据集包含表格检测和表格数据两部分数据，数据集中同时包含扫描和拍照两张场景的图像。
+
+https://github.com/wangwen-whu/WTW-Dataset/blob/main/demo/20210816_210413.gif
+
+<div align="center">
+    <img src="https://github.com/wangwen-whu/WTW-Dataset/blob/main/demo/20210816_210413.gif" width="500">
+</div>

doc/doc_ch/table_recognition.md

+# 表格识别
+
+本文提供了PaddleOCR表格识别模型的全流程指南，包括数据准备、模型训练、调优、评估、预测，各个阶段的详细说明：
+
+- [1. 数据准备](#1-数据准备)
+  - [1.1. 数据集格式](#11-数据集格式)
+  - [1.2. 数据下载](#12-数据下载)
+  - [1.3. 数据集生成](#13-数据集生成)
+- [2. 开始训练](#2-开始训练)
+  - [2.1. 启动训练](#21-启动训练)
+  - [2.2. 断点训练](#22-断点训练)
+  - [2.3. 更换Backbone 训练](#23-更换backbone-训练)
+  - [2.4. 混合精度训练](#24-混合精度训练)
+  - [2.5. 分布式训练](#25-分布式训练)
+  - [2.6. 其他训练环境](#26-其他训练环境)
+  - [2.7. 模型微调](#27-模型微调)
+- [3. 模型评估与预测](#3-模型评估与预测)
+  - [3.1. 指标评估](#31-指标评估)
+  - [3.2. 测试表格结构识别效果](#32-测试表格结构识别效果)
+- [4. 模型导出与预测](#4-模型导出与预测)
+  - [4.1 模型导出](#41-模型导出)
+  - [4.2 模型预测](#42-模型预测)
+- [5. FAQ](#5-faq)
+
+# 1. 数据准备
+
+## 1.1. 数据集格式
+
+PaddleOCR 表格识别模型数据集格式如下:
+```txt
+img_label # 每张图片标注经过json.dumps()之后的字符串
+...
+img_label
+```
+
+每一行的json格式为:
+```txt
+{
+   'filename': PMC5755158_010_01.png,							# 图像名
+   'split': ’train‘, 									# 图像属于训练集还是验证集
+   'imgid': 0,								 		# 图像的index
+   'html': {
+     'structure': {'tokens': ['<thead>', '<tr>', '<td>', ...]}, 			# 表格的HTML字符串
+     'cell': [
+       {
+         'tokens': ['P', 'a', 'd', 'd', 'l', 'e', 'P', 'a', 'd', 'd', 'l', 'e'], 	# 表格中的单个文本
+         'bbox': [x0, y0, x1, y1]  							# 表格中的单个文本的坐标
+       }
+     ]
+   }
+}
+```
+
+训练数据的默认存储路径是 `PaddleOCR/train_data`,如果您的磁盘上已有数据集，只需创建软链接至数据集目录：
+
+```
+# linux and mac os
+ln -sf <path/to/dataset> <path/to/paddle_ocr>/train_data/dataset
+# windows
+mklink /d <path/to/paddle_ocr>/train_data/dataset <path/to/dataset>
+```
+
+## 1.2. 数据下载
+
+公开数据集下载可参考 [table_datasets](dataset/table_datasets.md)。
+
+## 1.3. 数据集生成
+
+使用[TableGeneration](https://github.com/WenmuZhou/TableGeneration)可进行扫描表格图像的生成。
+
+TableGeneration是一个开源表格数据集生成工具，其通过浏览器渲染的方式对html字符串进行渲染后获得表格图像。部分样张如下：
+
+|类型|样例|
+|---|---|
+|简单表格|![](https://raw.githubusercontent.com/WenmuZhou/TableGeneration/main/imgs/simple.jpg)|
+|彩色表格|![](https://raw.githubusercontent.com/WenmuZhou/TableGeneration/main/imgs/color.jpg)|
+
+# 2. 开始训练
+
+PaddleOCR提供了训练脚本、评估脚本和预测脚本，本节将以 [SLANet](../../configs/table/SLANet.yml) 模型训练PubTabNet英文数据集为例：
+
+## 2.1. 启动训练
+
+*如果您安装的是cpu版本，请将配置文件中的 `use_gpu` 字段修改为false*
+
+```
+# GPU训练 支持单卡，多卡训练
+# 训练日志会自动保存为 "{save_model_dir}" 下的train.log
+
+#单卡训练（训练周期长，不建议）
+python3 tools/train.py -c configs/table/SLANet.yml
+
+#多卡训练，通过--gpus参数指定卡号
+python3 -m paddle.distributed.launch --gpus '0,1,2,3'  tools/train.py -c configs/table/SLANet.yml
+```
+
+正常启动训练后，会看到以下log输出：
+
+```
+[2022/08/16 03:07:33] ppocr INFO: epoch: [1/400], global_step: 20, lr: 0.000100, acc: 0.000000, loss: 3.915012, structure_loss: 3.229450, loc_loss: 0.670590, avg_reader_cost: 2.63382 s, avg_batch_cost: 6.32390 s, avg_samples: 48.0, ips: 7.59025 samples/s, eta: 9 days, 2:29:27
+[2022/08/16 03:08:41] ppocr INFO: epoch: [1/400], global_step: 40, lr: 0.000100, acc: 0.000000, loss: 1.750859, structure_loss: 1.082116, loc_loss: 0.652822, avg_reader_cost: 0.02533 s, avg_batch_cost: 3.37251 s, avg_samples: 48.0, ips: 14.23271 samples/s, eta: 6 days, 23:28:43
+[2022/08/16 03:09:46] ppocr INFO: epoch: [1/400], global_step: 60, lr: 0.000100, acc: 0.000000, loss: 1.395154, structure_loss: 0.776803, loc_loss: 0.625030, avg_reader_cost: 0.02550 s, avg_batch_cost: 3.26261 s, avg_samples: 48.0, ips: 14.71214 samples/s, eta: 6 days, 5:11:48
+```
+
+log 中自动打印如下信息：
+
+|  字段   |   含义   |  
+| :----: | :------: |
+|  epoch | 当前迭代轮次 |
+|  global_step  | 当前迭代次数 |
+|  lr    | 当前学习率 |
+|  acc   | 当前batch的准确率 |
+|  loss  | 当前损失函数 |
+|  structure_loss | 表格结构损失值 |
+|  loc_loss | 单元格坐标损失值 |
+|  avg_reader_cost | 当前 batch 数据处理耗时 |
+|  avg_batch_cost | 当前 batch 总耗时 |
+|  avg_samples  | 当前 batch 内的样本数 |
+|  ips  | 每秒处理图片的数量 |
+
+
+PaddleOCR支持训练和评估交替进行, 可以在 `configs/table/SLANet.yml` 中修改 `eval_batch_step` 设置评估频率，默认每1000个iter评估一次。评估过程中默认将最佳acc模型，保存为 `output/SLANet/best_accuracy` 。
+
+如果验证集很大，测试将会比较耗时，建议减少评估次数，或训练完再进行评估。
+
+**提示：** 可通过 -c 参数选择 `configs/table/` 路径下的多种模型配置进行训练，PaddleOCR支持的表格识别算法可以参考[前沿算法列表](https://github.com/PaddlePaddle/PaddleOCR/blob/dygraph/doc/doc_ch/algorithm_overview.md#3-%E8%A1%A8%E6%A0%BC%E8%AF%86%E5%88%AB%E7%AE%97%E6%B3%95)：
+
+**注意，预测/评估时的配置文件请务必与训练一致。**
+
+## 2.2. 断点训练
+
+如果训练程序中断，如果希望加载训练中断的模型从而恢复训练，可以通过指定Global.checkpoints指定要加载的模型路径：
+```shell
+python3 tools/train.py -c configs/table/SLANet.yml -o Global.checkpoints=./your/trained/model
+```
+
+**注意**：`Global.checkpoints`的优先级高于`Global.pretrained_model`的优先级，即同时指定两个参数时，优先加载`Global.checkpoints`指定的模型，如果`Global.checkpoints`指定的模型路径有误，会加载`Global.pretrained_model`指定的模型。
+
+## 2.3. 更换Backbone 训练
+
+PaddleOCR将网络划分为四部分，分别在[ppocr/modeling](../../ppocr/modeling)下。 进入网络的数据将按照顺序(transforms->backbones->necks->heads)依次通过这四个部分。
+
+```bash
+├── architectures # 网络的组网代码
+├── transforms    # 网络的图像变换模块
+├── backbones     # 网络的特征提取模块
+├── necks         # 网络的特征增强模块
+└── heads         # 网络的输出模块
+```
+如果要更换的Backbone 在PaddleOCR中有对应实现，直接修改配置yml文件中`Backbone`部分的参数即可。
+
+如果要使用新的Backbone，更换backbones的例子如下:
+
+1. 在 [ppocr/modeling/backbones](../../ppocr/modeling/backbones) 文件夹下新建文件，如my_backbone.py。
+2. 在 my_backbone.py 文件内添加相关代码，示例代码如下:
+
+```python
+import paddle
+import paddle.nn as nn
+import paddle.nn.functional as F
+
+
+class MyBackbone(nn.Layer):
+    def __init__(self, *args, **kwargs):
+        super(MyBackbone, self).__init__()
+        # your init code
+        self.conv = nn.xxxx
+
+    def forward(self, inputs):
+        # your network forward
+        y = self.conv(inputs)
+        return y
+```
+
+3. 在 [ppocr/modeling/backbones/\__init\__.py](../../ppocr/modeling/backbones/__init__.py)文件内导入添加的`MyBackbone`模块，然后修改配置文件中Backbone进行配置即可使用，格式如下:
+
+```yaml
+Backbone:
+name: MyBackbone
+args1: args1
+```
+
+**注意**：如果要更换网络的其他模块，可以参考[文档](./add_new_algorithm.md)。
+
+## 2.4. 混合精度训练
+
+如果您想进一步加快训练速度，可以使用[自动混合精度训练](https://www.paddlepaddle.org.cn/documentation/docs/zh/guides/01_paddle2.0_introduction/basic_concept/amp_cn.html)， 以单机单卡为例，命令如下：
+
+```shell
+python3 tools/train.py -c configs/table/SLANet.yml \
+     -o Global.pretrained_model=./pretrain_models/SLANet/best_accuracy \
+     Global.use_amp=True Global.scale_loss=1024.0 Global.use_dynamic_loss_scaling=True
+ ```
+
+## 2.5. 分布式训练
+
+多机多卡训练时，通过 `--ips` 参数设置使用的机器IP地址，通过 `--gpus` 参数设置使用的GPU ID：
+
+```bash
+python3 -m paddle.distributed.launch --ips="xx.xx.xx.xx,xx.xx.xx.xx" --gpus '0,1,2,3' tools/train.py -c configs/table/SLANet.yml \
+     -o Global.pretrained_model=./pretrain_models/SLANet/best_accuracy
+```
+
+**注意:** （1）采用多机多卡训练时，需要替换上面命令中的ips值为您机器的地址，机器之间需要能够相互ping通；（2）训练时需要在多个机器上分别启动命令。查看机器ip地址的命令为`ifconfig`；（3）更多关于分布式训练的性能优势等信息，请参考：[分布式训练教程](./distributed_training.md)。
+
+
+## 2.6. 其他训练环境
+
+- Windows GPU/CPU
+在Windows平台上与Linux平台略有不同:
+Windows平台只支持`单卡`的训练与预测，指定GPU进行训练`set CUDA_VISIBLE_DEVICES=0`
+在Windows平台，DataLoader只支持单进程模式，因此需要设置 `num_workers` 为0;
+
+- macOS
+不支持GPU模式，需要在配置文件中设置`use_gpu`为False，其余训练评估预测命令与Linux GPU完全相同。
+
+- Linux DCU
+DCU设备上运行需要设置环境变量 `export HIP_VISIBLE_DEVICES=0,1,2,3`，其余训练评估预测命令与Linux GPU完全相同。
+
+## 2.7. 模型微调
+
+实际使用过程中，建议加载官方提供的预训练模型，在自己的数据集中进行微调，关于模型的微调方法，请参考：[模型微调教程](./finetune.md)。
+
+
+# 3. 模型评估与预测
+
+## 3.1. 指标评估
+
+训练中模型参数默认保存在`Global.save_model_dir`目录下。在评估指标时，需要设置`Global.checkpoints`指向保存的参数文件。评估数据集可以通过 `configs/table/SLANet.yml`  修改Eval中的 `label_file_list` 设置。
+
+
+```
+# GPU 评估， Global.checkpoints 为待测权重
+python3 -m paddle.distributed.launch --gpus '0' tools/eval.py -c configs/table/SLANet.yml -o Global.checkpoints={path/to/weights}/best_accuracy
+```
+
+运行完成后，会输出模型的acc指标，如对英文表格识别模型进行评估，会见到如下输出。
+```bash
+[2022/08/16 07:59:55] ppocr INFO: acc:0.7622245132160782
+[2022/08/16 07:59:55] ppocr INFO: fps:30.991640622573044
+```
+
+## 3.2. 测试表格结构识别效果
+
+使用 PaddleOCR 训练好的模型，可以通过以下脚本进行快速预测。
+
+默认预测图片存储在 `infer_img` 里，通过 `-o Global.checkpoints` 加载训练好的参数文件：
+
+根据配置文件中设置的 `save_model_dir` 和 `save_epoch_step` 字段，会有以下几种参数被保存下来：
+
+```
+output/SLANet/
+├── best_accuracy.pdopt  
+├── best_accuracy.pdparams  
+├── best_accuracy.states  
+├── config.yml  
+├── latest.pdopt  
+├── latest.pdparams  
+├── latest.states  
+└── train.log
+```
+其中 best_accuracy.* 是评估集上的最优模型；latest.* 是最后一个epoch的模型。
+
+```
+# 预测表格图像
+python3 tools/infer_table.py -c configs/table/SLANet.yml -o Global.pretrained_model={path/to/weights}/best_accuracy  Global.infer_img=ppstructure/docs/table/table.jpg
+```
+
+预测图片：
+
+![](../../ppstructure/docs/table/table.jpg)
+
+得到输入图像的预测结果：
+
+```
+['<html>', '<body>', '<table>', '<thead>', '<tr>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '</tr>', '</thead>', '<tbody>', '<tr>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '</tr>', '<tr>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '</tr>', '<tr>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '</tr>', '<tr>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '</tr>', '<tr>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '</tr>', '<tr>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '</tr>', '<tr>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '</tr>', '<tr>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '</tr>', '<tr>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '</tr>', '<tr>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '</tr>', '<tr>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '</tr>', '<tr>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '</tr>', '<tr>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '</tr>', '<tr>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '</tr>', '<tr>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '</tr>', '</tbody>', '</table>', '</body>', '</html>'],[[320.0562438964844, 197.83375549316406, 350.0928955078125, 214.4309539794922], ... , [318.959228515625, 271.0166931152344, 353.7394104003906, 286.4538269042969]]
+```
+
+单元格坐标可视化结果为
+
+![](../../ppstructure/docs/imgs/slanet_result.jpg)
+
+# 4. 模型导出与预测
+
+## 4.1 模型导出
+
+inference 模型（`paddle.jit.save`保存的模型）
+一般是模型训练，把模型结构和模型参数保存在文件中的固化模型，多用于预测部署场景。
+训练过程中保存的模型是checkpoints模型，保存的只有模型的参数，多用于恢复训练等。
+与checkpoints模型相比，inference 模型会额外保存模型的结构信息，在预测部署、加速推理上性能优越，灵活方便，适合于实际系统集成。
+
+表格识别模型转inference模型与文字检测识别的方式相同，如下：
+
+```
+# -c 后面设置训练算法的yml配置文件
+# -o 配置可选参数
+# Global.pretrained_model 参数设置待转换的训练模型地址，不用添加文件后缀 .pdmodel，.pdopt或.pdparams。
+# Global.save_inference_dir参数设置转换的模型将保存的地址。
+
+python3 tools/export_model.py -c configs/table/SLANet.yml -o Global.pretrained_model=./pretrain_models/SLANet/best_accuracy  Global.save_inference_dir=./inference/SLANet/
+```
+
+转换成功后，在目录下有三个文件：
+
+```
+inference/SLANet/
+    ├── inference.pdiparams         # inference模型的参数文件
+    ├── inference.pdiparams.info    # inference模型的参数信息，可忽略
+    └── inference.pdmodel           # inference模型的program文件
+```
+
+## 4.2 模型预测
+
+模型导出后，使用如下命令即可完成inference模型的预测
+
+```python
+python3.7 table/predict_structure.py \
+    --table_model_dir={path/to/inference model} \
+    --table_char_dict_path=../ppocr/utils/dict/table_structure_dict_ch.txt \
+    --image_dir=docs/table/table.jpg \
+    --output=../output/table
+```
+
+预测图片：
+
+![](../../ppstructure/docs/table/table.jpg)
+
+得到输入图像的预测结果：
+
+```
+['<html>', '<body>', '<table>', '<thead>', '<tr>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '</tr>', '</thead>', '<tbody>', '<tr>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '</tr>', '<tr>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '</tr>', '<tr>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '</tr>', '<tr>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '</tr>', '<tr>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '</tr>', '<tr>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '</tr>', '<tr>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '</tr>', '<tr>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '</tr>', '<tr>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '</tr>', '<tr>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '</tr>', '<tr>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '</tr>', '<tr>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '</tr>', '<tr>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '</tr>', '<tr>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '</tr>', '<tr>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '</tr>', '</tbody>', '</table>', '</body>', '</html>'],[[320.0562438964844, 197.83375549316406, 350.0928955078125, 214.4309539794922], ... , [318.959228515625, 271.0166931152344, 353.7394104003906, 286.4538269042969]]
+```
+
+单元格坐标可视化结果为
+
+![](../../ppstructure/docs/imgs/slanet_result.jpg)
+
+
+# 5. FAQ
+
+Q1: 训练模型转inference 模型之后预测效果不一致？
+
+**A**：此类问题出现较多，问题多是trained model预测时候的预处理、后处理参数和inference model预测的时候的预处理、后处理参数不一致导致的。可以对比训练使用的配置文件中的预处理、后处理和预测时是否存在差异。

doc/doc_en/algorithm_rec_sar_en.md

@@ -79,7 +79,7 @@ python3 tools/export_model.py -c configs/rec/rec_r31_sar.yml -o Global.pretraine
 For SAR text recognition model inference, the following commands can be executed:
 
 ```
-python3 tools/infer/predict_rec.py --image_dir="./doc/imgs_words/en/word_1.png" --rec_model_dir="./inference/rec_sar/" --rec_image_shape="3, 48, 48, 160" --rec_char_type="ch" --rec_algorithm="SAR" --rec_char_dict_path="ppocr/utils/dict90.txt" --max_text_length=30 --use_space_char=False
+python3 tools/infer/predict_rec.py --image_dir="./doc/imgs_words/en/word_1.png" --rec_model_dir="./inference/rec_sar/" --rec_image_shape="3, 48, 48, 160" --rec_algorithm="SAR" --rec_char_dict_path="ppocr/utils/dict90.txt" --max_text_length=30 --use_space_char=False
 ```
 
 <a name="4-2"></a>

doc/doc_en/dataset/table_datasets_en.md

@@ -3,6 +3,7 @@
 - [Dataset Summary](#dataset-summary)
 - [1. PubTabNet](#1-pubtabnet)
 - [2. TAL Table Recognition Competition Dataset](#2-tal-table-recognition-competition-dataset)
+- [3. WTW Chinese scene table dataset](#3-wtw-chinese-scene-table-dataset)
 
 Here are the commonly used table recognition datasets, which are being updated continuously. Welcome to contribute datasets~
 
@@ -12,6 +13,7 @@ Here are the commonly used table recognition datasets, which are being updated c
 |---|---|---|
 | PubTabNet |https://github.com/ibm-aur-nlp/PubTabNet| jsonl format, which can be loaded directly with [pubtab_dataset.py](../../../ppocr/data/pubtab_dataset.py) |
 | TAL Table Recognition Competition Dataset |https://ai.100tal.com/dataset| jsonl format, which can be loaded directly with [pubtab_dataset.py](../../../ppocr/data/pubtab_dataset.py) |
+| WTW Chinese scene table dataset |https://github.com/wangwen-whu/WTW-Dataset| Conversion is required to load with [pubtab_dataset.py](../../../ppocr/data/pubtab_dataset.py)|
 
 ## 1. PubTabNet
 - **Data Introduction**：The training set of the PubTabNet dataset contains 500,000 images and the validation set contains 9000 images. Part of the image visualization is shown below.
@@ -30,3 +32,11 @@ Here are the commonly used table recognition datasets, which are being updated c
     <img src="../../datasets/table_tal_demo/1.jpg" width="500">
     <img src="../../datasets/table_tal_demo/2.jpg" width="500">
 </div>
+
+## 3. WTW Chinese scene table dataset
+- **Data Introduction**：The WTW Chinese scene table dataset consists of two parts: table detection and table data. The dataset contains images of two scenes, scanned and photographed.
+https://github.com/wangwen-whu/WTW-Dataset/blob/main/demo/20210816_210413.gif
+
+<div align="center">
+    <img src="https://github.com/wangwen-whu/WTW-Dataset/blob/main/demo/20210816_210413.gif" width="500">
+</div>

doc/doc_en/quickstart_en.md

@@ -3,14 +3,14 @@
 **Note:** This tutorial mainly introduces the usage of PP-OCR series models, please refer to [PP-Structure Quick Start](../../ppstructure/docs/quickstart_en.md) for the quick use of document analysis related functions.
 
 - [1. Installation](#1-installation)
-    - [1.1 Install PaddlePaddle](#11-install-paddlepaddle)
-    - [1.2 Install PaddleOCR Whl Package](#12-install-paddleocr-whl-package)
+  - [1.1 Install PaddlePaddle](#11-install-paddlepaddle)
+  - [1.2 Install PaddleOCR Whl Package](#12-install-paddleocr-whl-package)
 - [2. Easy-to-Use](#2-easy-to-use)
-    - [2.1 Use by Command Line](#21-use-by-command-line)
-      - [2.1.1 Chinese and English Model](#211-chinese-and-english-model)
-      - [2.1.2 Multi-language Model](#212-multi-language-model)
-    - [2.2 Use by Code](#22-use-by-code)
-      - [2.2.1 Chinese & English Model and Multilingual Model](#221-chinese--english-model-and-multilingual-model)
+  - [2.1 Use by Command Line](#21-use-by-command-line)
+    - [2.1.1 Chinese and English Model](#211-chinese-and-english-model)
+    - [2.1.2 Multi-language Model](#212-multi-language-model)
+  - [2.2 Use by Code](#22-use-by-code)
+    - [2.2.1 Chinese & English Model and Multilingual Model](#221-chinese--english-model-and-multilingual-model)
 - [3. Summary](#3-summary)
 
 
@@ -51,12 +51,6 @@ pip install "paddleocr>=2.0.1" # Recommend to use version 2.0.1+
 
   Reference: [Solve shapely installation on windows](https://stackoverflow.com/questions/44398265/install-shapely-oserror-winerror-126-the-specified-module-could-not-be-found)
 
-- **For layout analysis users**, run the following command to install **Layout-Parser**
-
-  ```bash
-  pip3 install -U https://paddleocr.bj.bcebos.com/whl/layoutparser-0.0.0-py3-none-any.whl
-  ```
-
 <a name="2-easy-to-use"></a>
 
 ## 2. Easy-to-Use

doc/doc_en/table_recognition_en.md

+# Table Recognition
+
+This article provides a full-process guide for the PaddleOCR table recognition model, including data preparation, model training, tuning, evaluation, prediction, and detailed descriptions of each stage:
+
+- [1. Data Preparation](#1-data-preparation)
+  - [1.1. DataSet Format](#11-dataset-format)
+  - [1.2. Data Download](#12-data-download)
+  - [1.3. Dataset Generation](#13-dataset-generation)
+- [2. Training](#2-training)
+  - [2.1. Start Training](#21-start-training)
+  - [2.2. Resume Training](#22-resume-training)
+  - [2.3. Training with New Backbone](#23-training-with-new-backbone)
+  - [2.4. Mixed Precision Training](#24-mixed-precision-training)
+  - [2.5. Distributed Training](#25-distributed-training)
+  - [2.6. Training on other platform(Windows/macOS/Linux DCU)](#26-training-on-other-platformwindowsmacoslinux-dcu)
+  - [2.7. Fine-tuning](#27-fine-tuning)
+- [3. Evaluation and Test](#3-evaluation-and-test)
+  - [3.1. Evaluation](#31-evaluation)
+  - [3.2. Test table structure recognition effect](#32-test-table-structure-recognition-effect)
+- [4. Model export and prediction](#4-model-export-and-prediction)
+  - [4.1 Model export](#41-model-export)
+  - [4.2 Prediction](#42-prediction)
+- [5. FAQ](#5-faq)
+
+# 1. Data Preparation
+
+## 1.1. DataSet Format
+
+The format of the PaddleOCR table recognition model dataset is as follows:
+```txt
+img_label # Each image is marked with a string after json.dumps()
+...
+img_label
+```
+
+The json format of each line is:
+```txt
+{
+   'filename': PMC5755158_010_01.png,# image name
+   'split': ’train‘, # whether the image belongs to the training set or the validation set
+   'imgid': 0,# index of image
+   'html': {
+     'structure': {'tokens': ['<thead>', '<tr>', '<td>', ...]}, # HTML string of the table
+     'cell': [
+       {
+         'tokens': ['P', 'a', 'd', 'd', 'l', 'e', 'P', 'a', 'd', 'd', 'l', 'e'], # text in cell
+         'bbox': [x0, y0, x1, y1] # bbox of cell
+       }
+     ]
+   }
+}
+```
+
+The default storage path for training data is `PaddleOCR/train_data`, if you already have a dataset on disk, just create a soft link to the dataset directory:
+
+```
+# linux and mac os
+ln -sf <path/to/dataset> <path/to/paddle_ocr>/train_data/dataset
+# windows
+mklink /d <path/to/paddle_ocr>/train_data/dataset <path/to/dataset>
+```
+
+## 1.2. Data Download
+
+Download the public dataset reference [table_datasets](dataset/table_datasets_en.md)。
+
+## 1.3. Dataset Generation
+
+Use [TableGeneration](https://github.com/WenmuZhou/TableGeneration) to generate scanned table images.
+
+TableGeneration is an open source table dataset generation tool, which renders html strings through browser rendering to obtain table images.
+
+Some samples are as follows:
+
+|Type|Sample|
+|---|---|
+|Simple Table|![](https://raw.githubusercontent.com/WenmuZhou/TableGeneration/main/imgs/simple.jpg)|
+|Simple Color Table|![](https://raw.githubusercontent.com/WenmuZhou/TableGeneration/main/imgs/color.jpg)|
+
+# 2. Training
+
+PaddleOCR provides training scripts, evaluation scripts, and prediction scripts. In this section, the [SLANet](../../configs/table/SLANet.yml) model will be used as an example:
+
+## 2.1. Start Training
+
+*If you are installing the cpu version, please modify the `use_gpu` field in the configuration file to false*
+
+```
+# GPU training Support single card and multi-card training
+# The training log will be automatically saved as train.log under "{save_model_dir}"
+
+# specify the single card training(Long training time, not recommended)
+python3 tools/train.py -c configs/table/SLANet.yml
+
+# specify the card number through --gpus
+python3 -m paddle.distributed.launch --gpus '0,1,2,3'  tools/train.py -c configs/table/SLANet.yml
+```
+
+After starting training normally, you will see the following log output:
+
+```
+[2022/08/16 03:07:33] ppocr INFO: epoch: [1/400], global_step: 20, lr: 0.000100, acc: 0.000000, loss: 3.915012, structure_loss: 3.229450, loc_loss: 0.670590, avg_reader_cost: 2.63382 s, avg_batch_cost: 6.32390 s, avg_samples: 48.0, ips: 7.59025 samples/s, eta: 9 days, 2:29:27
+[2022/08/16 03:08:41] ppocr INFO: epoch: [1/400], global_step: 40, lr: 0.000100, acc: 0.000000, loss: 1.750859, structure_loss: 1.082116, loc_loss: 0.652822, avg_reader_cost: 0.02533 s, avg_batch_cost: 3.37251 s, avg_samples: 48.0, ips: 14.23271 samples/s, eta: 6 days, 23:28:43
+[2022/08/16 03:09:46] ppocr INFO: epoch: [1/400], global_step: 60, lr: 0.000100, acc: 0.000000, loss: 1.395154, structure_loss: 0.776803, loc_loss: 0.625030, avg_reader_cost: 0.02550 s, avg_batch_cost: 3.26261 s, avg_samples: 48.0, ips: 14.71214 samples/s, eta: 6 days, 5:11:48
+```
+
+The following information is automatically printed in the log:
+
+|  Field   |   Meaning   |  
+| :----: | :------: |
+|  epoch | current iteration round |
+|  global_step  | current iteration count |
+|  lr    | current learning rate |
+|  acc   | The accuracy of the current batch |
+|  loss  | current loss function |
+|  structure_loss | Table Structure Loss Values |
+|  loc_loss | Cell Coordinate Loss Value |
+|  avg_reader_cost | Current batch data processing time |
+|  avg_batch_cost | The total time spent in the current batch |
+|  avg_samples  | The number of samples in the current batch |
+|  ips  | Number of images processed per second |
+
+
+PaddleOCR supports alternating training and evaluation. You can modify `eval_batch_step` in `configs/table/SLANet.yml` to set the evaluation frequency. By default, it is evaluated once every 1000 iters. During the evaluation process, the best acc model is saved as `output/SLANet/best_accuracy` by default.
+
+If the validation set is large, the test will be time-consuming. It is recommended to reduce the number of evaluations, or perform evaluation after training.
+
+**Tips:** You can use the -c parameter to select various model configurations under the `configs/table/` path for training. For the table recognition algorithms supported by PaddleOCR, please refer to [Table Algorithms List](https://github.com/PaddlePaddle/PaddleOCR/blob/dygraph/doc/doc_en/algorithm_overview_en.md#3):
+
+**Note that the configuration file for prediction/evaluation must be the same as training. **
+
+## 2.2. Resume Training
+
+If the training program is interrupted, if you want to load the interrupted model to resume training, you can specify the path of the model to be loaded by specifying Global.checkpoints:
+
+```shell
+python3 tools/train.py -c configs/table/SLANet.yml -o Global.checkpoints=./your/trained/model
+```
+**Note**: The priority of `Global.checkpoints` is higher than that of `Global.pretrained_model`, that is, when two parameters are specified at the same time, the model specified by `Global.checkpoints` will be loaded first. If `Global.checkpoints` The specified model path is incorrect, and the model specified by `Global.pretrained_model` will be loaded.
+
+## 2.3. Training with New Backbone
+
+The network part completes the construction of the network, and PaddleOCR divides the network into four parts, which are under [ppocr/modeling](../../ppocr/modeling). The data entering the network will pass through these four parts in sequence(transforms->backbones->
+necks->heads).
+
+```bash
+├── architectures # Code for building network
+├── transforms    # Image Transformation Module
+├── backbones     # Feature extraction module
+├── necks         # Feature enhancement module
+└── heads         # Output module
+```
+
+If the Backbone to be replaced has a corresponding implementation in PaddleOCR, you can directly modify the parameters in the `Backbone` part of the configuration yml file.
+
+However, if you want to use a new Backbone, an example of replacing the backbones is as follows:
+
+1. Create a new file under the [ppocr/modeling/backbones](../../ppocr/modeling/backbones) folder, such as my_backbone.py.
+2. Add code in the my_backbone.py file, the sample code is as follows:
+
+```python
+import paddle
+import paddle.nn as nn
+import paddle.nn.functional as F
+
+
+class MyBackbone(nn.Layer):
+    def __init__(self, *args, **kwargs):
+        super(MyBackbone, self).__init__()
+        # your init code
+        self.conv = nn.xxxx
+
+    def forward(self, inputs):
+        # your network forward
+        y = self.conv(inputs)
+        return y
+```
+
+3. Import the added module in the [ppocr/modeling/backbones/\__init\__.py](../../ppocr/modeling/backbones/__init__.py) file.
+
+After adding the four-part modules of the network, you only need to configure them in the configuration file to use, such as:
+
+```yaml
+  Backbone:
+    name: MyBackbone
+    args1: args1
+```
+
+**NOTE**: More details about replace Backbone and other mudule can be found in [doc](add_new_algorithm_en.md).
+
+## 2.4. Mixed Precision Training
+
+If you want to speed up your training further, you can use [Auto Mixed Precision Training](https://www.paddlepaddle.org.cn/documentation/docs/zh/guides/01_paddle2.0_introduction/basic_concept/amp_cn.html), taking a single machine and a single gpu as an example, the commands are as follows:
+
+```shell
+python3 tools/train.py -c configs/table/SLANet.yml \
+     -o Global.pretrained_model=./pretrain_models/SLANet/best_accuracy \
+     Global.use_amp=True Global.scale_loss=1024.0 Global.use_dynamic_loss_scaling=True
+ ```
+
+## 2.5. Distributed Training
+
+During multi-machine multi-gpu training, use the `--ips` parameter to set the used machine IP address, and the `--gpus` parameter to set the used GPU ID:
+
+```bash
+python3 -m paddle.distributed.launch --ips="xx.xx.xx.xx,xx.xx.xx.xx" --gpus '0,1,2,3' tools/train.py -c configs/table/SLANet.yml \
+     -o Global.pretrained_model=./pretrain_models/SLANet/best_accuracy
+```
+
+
+**Note:** (1) When using multi-machine and multi-gpu training, you need to replace the ips value in the above command with the address of your machine, and the machines need to be able to ping each other. (2) Training needs to be launched separately on multiple machines. The command to view the ip address of the machine is `ifconfig`. (3) For more details about the distributed training speedup ratio, please refer to [Distributed Training Tutorial](./distributed_training_en.md).
+
+## 2.6. Training on other platform(Windows/macOS/Linux DCU)
+
+- Windows GPU/CPU
+The Windows platform is slightly different from the Linux platform:
+Windows platform only supports `single gpu` training and inference, specify GPU for training `set CUDA_VISIBLE_DEVICES=0`
+On the Windows platform, DataLoader only supports single-process mode, so you need to set `num_workers` to 0;
+
+- macOS
+GPU mode is not supported, you need to set `use_gpu` to False in the configuration file, and the rest of the training evaluation prediction commands are exactly the same as Linux GPU.
+
+- Linux DCU
+Running on a DCU device requires setting the environment variable `export HIP_VISIBLE_DEVICES=0,1,2,3`, and the rest of the training and evaluation prediction commands are exactly the same as the Linux GPU.
+
+
+## 2.7. Fine-tuning
+
+In the actual use process, it is recommended to load the officially provided pre-training model and fine-tune it in your own data set. For the fine-tuning method of the table recognition model, please refer to: [Model fine-tuning tutorial](./finetune.md).
+
+
+# 3. Evaluation and Test
+
+## 3.1. Evaluation
+
+The model parameters during training are saved in the `Global.save_model_dir` directory by default. When evaluating metrics, you need to set `Global.checkpoints` to point to the saved parameter file. Evaluation datasets can be modified via the `label_file_list` setting in Eval via `configs/table/SLANet.yml`.
+
+```
+# GPU evaluation, Global.checkpoints is the weight to be tested
+python3 -m paddle.distributed.launch --gpus '0' tools/eval.py -c configs/table/SLANet.yml -o Global.checkpoints={path/to/weights}/best_accuracy
+```
+
+After the operation is completed, the acc indicator of the model will be output. If you evaluate the English table recognition model, you will see the following output.
+
+```bash
+[2022/08/16 07:59:55] ppocr INFO: acc:0.7622245132160782
+[2022/08/16 07:59:55] ppocr INFO: fps:30.991640622573044
+```
+
+## 3.2. Test table structure recognition effect
+
+Using the model trained by PaddleOCR, you can quickly get prediction through the following script.
+
+The default prediction picture is stored in `infer_img`, and the trained weight is specified via `-o Global.checkpoints`:
+
+
+According to the `save_model_dir` and `save_epoch_step` fields set in the configuration file, the following parameters will be saved:
+
+
+```
+output/SLANet/
+├── best_accuracy.pdopt  
+├── best_accuracy.pdparams  
+├── best_accuracy.states  
+├── config.yml  
+├── latest.pdopt  
+├── latest.pdparams  
+├── latest.states  
+└── train.log
+```
+Among them, best_accuracy.* is the best model on the evaluation set; latest.* is the model of the last epoch.
+
+```
+# Predict table image
+python3 tools/infer_table.py -c configs/table/SLANet.yml -o Global.pretrained_model={path/to/weights}/best_accuracy  Global.infer_img=ppstructure/docs/table/table.jpg
+```
+
+Input image:
+
+![](../../ppstructure/docs/table/table.jpg)
+
+Get the prediction result of the input image:
+
+```
+['<html>', '<body>', '<table>', '<thead>', '<tr>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '</tr>', '</thead>', '<tbody>', '<tr>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '</tr>', '<tr>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '</tr>', '<tr>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '</tr>', '<tr>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '</tr>', '<tr>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '</tr>', '<tr>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '</tr>', '<tr>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '</tr>', '<tr>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '</tr>', '<tr>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '</tr>', '<tr>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '</tr>', '<tr>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '</tr>', '<tr>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '</tr>', '<tr>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '</tr>', '<tr>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '</tr>', '<tr>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '</tr>', '</tbody>', '</table>', '</body>', '</html>'],[[320.0562438964844, 197.83375549316406, 350.0928955078125, 214.4309539794922], ... , [318.959228515625, 271.0166931152344, 353.7394104003906, 286.4538269042969]]
+```
+
+The cell coordinates are visualized as
+
+![](../../ppstructure/docs/imgs/slanet_result.jpg)
+
+# 4. Model export and prediction
+
+## 4.1 Model export
+
+inference model (model saved by `paddle.jit.save`)
+Generally, it is model training, a solidified model that saves the model structure and model parameters in a file, and is mostly used to predict deployment scenarios.
+The model saved during the training process is the checkpoints model, and only the parameters of the model are saved, which are mostly used to resume training.
+Compared with the checkpoints model, the inference model will additionally save the structural information of the model. It has superior performance in predicting deployment and accelerating reasoning, and is flexible and convenient, and is suitable for actual system integration.
+
+The way to convert the form recognition model to the inference model is the same as the text detection and recognition, as follows:
+
+```
+# -c Set the training algorithm yml configuration file
+# -o Set optional parameters
+# Global.pretrained_model parameter Set the training model address to be converted without adding the file suffix .pdmodel, .pdopt or .pdparams.
+# Global.save_inference_dir Set the address where the converted model will be saved.
+
+python3 tools/export_model.py -c configs/table/SLANet.yml -o Global.pretrained_model=./pretrain_models/SLANet/best_accuracy  Global.save_inference_dir=./inference/SLANet/
+```
+
+After the conversion is successful, there are three files in the model save directory:
+
+
+```
+inference/SLANet/
+    ├── inference.pdiparams         # The parameter file of inference model
+    ├── inference.pdiparams.info    # The parameter information of inference model, which can be ignored
+    └── inference.pdmodel           # The program file of model
+```
+
+## 4.2 Prediction
+
+After the model is exported, use the following command to complete the prediction of the inference model
+
+```python
+python3.7 table/predict_structure.py \
+    --table_model_dir={path/to/inference model} \
+    --table_char_dict_path=../ppocr/utils/dict/table_structure_dict_ch.txt \
+    --image_dir=docs/table/table.jpg \
+    --output=../output/table
+```
+
+Input image:
+
+![](../../ppstructure/docs/table/table.jpg)
+
+Get the prediction result of the input image:
+
+```
+['<html>', '<body>', '<table>', '<thead>', '<tr>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '</tr>', '</thead>', '<tbody>', '<tr>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '</tr>', '<tr>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '</tr>', '<tr>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '</tr>', '<tr>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '</tr>', '<tr>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '</tr>', '<tr>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '</tr>', '<tr>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '</tr>', '<tr>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '</tr>', '<tr>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '</tr>', '<tr>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '</tr>', '<tr>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '</tr>', '<tr>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '</tr>', '<tr>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '</tr>', '<tr>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '</tr>', '<tr>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '<td></td>', '</tr>', '</tbody>', '</table>', '</body>', '</html>'],[[320.0562438964844, 197.83375549316406, 350.0928955078125, 214.4309539794922], ... , [318.959228515625, 271.0166931152344, 353.7394104003906, 286.4538269042969]]
+```
+
+The cell coordinates are visualized as
+
+![](../../ppstructure/docs/imgs/slanet_result.jpg)
+
+
+
+# 5. FAQ
+
+Q1: After the training model is transferred to the inference model, the prediction effect is inconsistent?
+
+**A**: There are many such problems, and the problems are mostly caused by inconsistent preprocessing and postprocessing parameters when the trained model predicts and the preprocessing and postprocessing parameters when the inference model predicts. You can compare whether there are differences in preprocessing, postprocessing, and prediction in the configuration files used for training.

paddleocr.py

@@ -47,14 +47,14 @@ __all__ = [
 ]
 
 SUPPORT_DET_MODEL = ['DB']
-VERSION = '2.5.0.3'
+VERSION = '2.6'
 SUPPORT_REC_MODEL = ['CRNN', 'SVTR_LCNet']
 BASE_DIR = os.path.expanduser("~/.paddleocr/")
 
 DEFAULT_OCR_MODEL_VERSION = 'PP-OCRv3'
 SUPPORT_OCR_MODEL_VERSION = ['PP-OCR', 'PP-OCRv2', 'PP-OCRv3']
-DEFAULT_STRUCTURE_MODEL_VERSION = 'PP-STRUCTURE'
-SUPPORT_STRUCTURE_MODEL_VERSION = ['PP-STRUCTURE']
+DEFAULT_STRUCTURE_MODEL_VERSION = 'PP-Structurev2'
+SUPPORT_STRUCTURE_MODEL_VERSION = ['PP-Structure', 'PP-Structurev2']
 MODEL_URLS = {
     'OCR': {
         'PP-OCRv3': {
@@ -263,7 +263,7 @@ MODEL_URLS = {
         }
     },
     'STRUCTURE': {
-        'PP-STRUCTURE': {
+        'PP-Structure': {
             'table': {
                 'en': {
                     'url':
@@ -271,6 +271,27 @@ MODEL_URLS = {
                     'dict_path': 'ppocr/utils/dict/table_structure_dict.txt'
                 }
             }
+        },
+        'PP-Structurev2': {
+            'table': {
+                'en': {
+                    'url':
+                    'https://paddleocr.bj.bcebos.com/ppstructure/models/slanet/en_ppstructure_mobile_v2.0_SLANet_infer.tar',
+                    'dict_path': 'ppocr/utils/dict/table_structure_dict.txt'
+                },
+                'ch': {
+                    'url':
+                    'https://paddleocr.bj.bcebos.com/ppstructure/models/slanet/ch_ppstructure_mobile_v2.0_SLANet_infer.tar',
+                    'dict_path': 'ppocr/utils/dict/table_structure_dict_ch.txt'
+                }
+            },
+            'layout': {
+                'ch': {
+                    'url':
+                    'https://paddleocr.bj.bcebos.com/ppstructure/models/layout/picodet_lcnet_x1_0_layout_infer.tar',
+                    'dict_path': 'ppocr/utils/dict/layout_publaynet_dict.txt'
+                }
+            }
         }
     }
 }
@@ -298,12 +319,15 @@ def parse_args(mMain=True):
         "--structure_version",
         type=str,
         choices=SUPPORT_STRUCTURE_MODEL_VERSION,
-        default='PP-STRUCTURE',
+        default='PP-Structurev2',
         help='Model version, the current model support list is as follows:'
-        ' 1. STRUCTURE Support en table structure model.')
+        ' 1. PP-Structure Support en table structure model.'
+        ' 2. PP-Structurev2 Support ch and en table structure model.')
 
     for action in parser._actions:
-        if action.dest in ['rec_char_dict_path', 'table_char_dict_path']:
+        if action.dest in [
+                'rec_char_dict_path', 'table_char_dict_path', 'layout_dict_path'
+        ]:
             action.default = None
     if mMain:
         return parser.parse_args()
@@ -477,7 +501,7 @@ class PaddleOCR(predict_system.TextSystem):
         if isinstance(img, np.ndarray) and len(img.shape) == 2:
             img = cv2.cvtColor(img, cv2.COLOR_GRAY2BGR)
         if det and rec:
-            dt_boxes, rec_res = self.__call__(img, cls)
+            dt_boxes, rec_res, _ = self.__call__(img, cls)
             return [[box.tolist(), res] for box, res in zip(dt_boxes, rec_res)]
         elif det and not rec:
             dt_boxes, elapse = self.text_detector(img)
@@ -506,6 +530,12 @@ class PPStructure(StructureSystem):
         if not params.show_log:
             logger.setLevel(logging.INFO)
         lang, det_lang = parse_lang(params.lang)
+        if lang == 'ch':
+            table_lang = 'ch'
+        else:
+            table_lang = 'en'
+        if params.structure_version == 'PP-Structure':
+            params.merge_no_span_structure = False
 
         # init model dir
         det_model_config = get_model_config('OCR', params.ocr_version, 'det',
@@ -520,14 +550,20 @@ class PPStructure(StructureSystem):
             params.rec_model_dir,
             os.path.join(BASE_DIR, 'whl', 'rec', lang), rec_model_config['url'])
         table_model_config = get_model_config(
-            'STRUCTURE', params.structure_version, 'table', 'en')
+            'STRUCTURE', params.structure_version, 'table', table_lang)
         params.table_model_dir, table_url = confirm_model_dir_url(
             params.table_model_dir,
             os.path.join(BASE_DIR, 'whl', 'table'), table_model_config['url'])
+        layout_model_config = get_model_config(
+            'STRUCTURE', params.structure_version, 'layout', 'ch')
+        params.layout_model_dir, layout_url = confirm_model_dir_url(
+            params.layout_model_dir,
+            os.path.join(BASE_DIR, 'whl', 'layout'), layout_model_config['url'])
         # download model
         maybe_download(params.det_model_dir, det_url)
         maybe_download(params.rec_model_dir, rec_url)
         maybe_download(params.table_model_dir, table_url)
+        maybe_download(params.layout_model_dir, layout_url)
 
         if params.rec_char_dict_path is None:
             params.rec_char_dict_path = str(
@@ -535,7 +571,9 @@ class PPStructure(StructureSystem):
         if params.table_char_dict_path is None:
             params.table_char_dict_path = str(
                 Path(__file__).parent / table_model_config['dict_path'])
-
+        if params.layout_dict_path is None:
+            params.layout_dict_path = str(
+                Path(__file__).parent / layout_model_config['dict_path'])
         logger.debug(params)
         super().__init__(params)
 
@@ -557,7 +595,7 @@ class PPStructure(StructureSystem):
         if isinstance(img, np.ndarray) and len(img.shape) == 2:
             img = cv2.cvtColor(img, cv2.COLOR_GRAY2BGR)
 
-        res = super().__call__(img, return_ocr_result_in_table)
+        res, _ = super().__call__(img, return_ocr_result_in_table)
         return res
 
 

ppocr/data/imaug/label_ops.py

@@ -575,7 +575,7 @@ class TableLabelEncode(AttnLabelEncode):
                  replace_empty_cell_token=False,
                  merge_no_span_structure=False,
                  learn_empty_box=False,
-                 point_num=2,
+                 loc_reg_num=4,
                  **kwargs):
         self.max_text_len = max_text_length
         self.lower = False
@@ -590,6 +590,12 @@ class TableLabelEncode(AttnLabelEncode):
                 line = line.decode('utf-8').strip("\n").strip("\r\n")
                 dict_character.append(line)
 
+        if self.merge_no_span_structure:
+            if "<td></td>" not in dict_character:
+                dict_character.append("<td></td>")
+            if "<td>" in dict_character:
+                dict_character.remove("<td>")
+
         dict_character = self.add_special_char(dict_character)
         self.dict = {}
         for i, char in enumerate(dict_character):
@@ -597,7 +603,7 @@ class TableLabelEncode(AttnLabelEncode):
         self.idx2char = {v: k for k, v in self.dict.items()}
 
         self.character = dict_character
-        self.point_num = point_num
+        self.loc_reg_num = loc_reg_num
         self.pad_idx = self.dict[self.beg_str]
         self.start_idx = self.dict[self.beg_str]
         self.end_idx = self.dict[self.end_str]
@@ -653,7 +659,7 @@ class TableLabelEncode(AttnLabelEncode):
 
         # encode box
         bboxes = np.zeros(
-            (self._max_text_len, self.point_num * 2), dtype=np.float32)
+            (self._max_text_len, self.loc_reg_num), dtype=np.float32)
         bbox_masks = np.zeros((self._max_text_len, 1), dtype=np.float32)
 
         bbox_idx = 0
@@ -718,11 +724,11 @@ class TableMasterLabelEncode(TableLabelEncode):
                  replace_empty_cell_token=False,
                  merge_no_span_structure=False,
                  learn_empty_box=False,
-                 point_num=2,
+                 loc_reg_num=4,
                  **kwargs):
         super(TableMasterLabelEncode, self).__init__(
             max_text_length, character_dict_path, replace_empty_cell_token,
-            merge_no_span_structure, learn_empty_box, point_num, **kwargs)
+            merge_no_span_structure, learn_empty_box, loc_reg_num, **kwargs)
         self.pad_idx = self.dict[self.pad_str]
         self.unknown_idx = self.dict[self.unknown_str]
 
@@ -743,27 +749,35 @@ class TableMasterLabelEncode(TableLabelEncode):
 
 
 class TableBoxEncode(object):
-    def __init__(self, use_xywh=False, **kwargs):
-        self.use_xywh = use_xywh
+    def __init__(self, in_box_format='xyxy', out_box_format='xyxy', **kwargs):
+        assert out_box_format in ['xywh', 'xyxy', 'xyxyxyxy']
+        self.in_box_format = in_box_format
+        self.out_box_format = out_box_format
 
     def __call__(self, data):
         img_height, img_width = data['image'].shape[:2]
         bboxes = data['bboxes']
-        if self.use_xywh and bboxes.shape[1] == 4:
-            bboxes = self.xyxy2xywh(bboxes)
+        if self.in_box_format != self.out_box_format:
+            if self.out_box_format == 'xywh':
+                if self.in_box_format == 'xyxyxyxy':
+                    bboxes = self.xyxyxyxy2xywh(bboxes)
+                elif self.in_box_format == 'xyxy':
+                    bboxes = self.xyxy2xywh(bboxes)
+
         bboxes[:, 0::2] /= img_width
         bboxes[:, 1::2] /= img_height
         data['bboxes'] = bboxes
         return data
 
+    def xyxyxyxy2xywh(self, boxes):
+        new_bboxes = np.zeros([len(bboxes), 4])
+        new_bboxes[:, 0] = bboxes[:, 0::2].min()  # x1
+        new_bboxes[:, 1] = bboxes[:, 1::2].min()  # y1
+        new_bboxes[:, 2] = bboxes[:, 0::2].max() - new_bboxes[:, 0]  # w
+        new_bboxes[:, 3] = bboxes[:, 1::2].max() - new_bboxes[:, 1]  # h
+        return new_bboxes
+
     def xyxy2xywh(self, bboxes):
-        """
-        Convert coord (x1,y1,x2,y2) to (x,y,w,h).
-        where (x1,y1) is top-left, (x2,y2) is bottom-right.
-        (x,y) is bbox center and (w,h) is width and height.
-        :param bboxes: (x1, y1, x2, y2)
-        :return:
-        """
         new_bboxes = np.empty_like(bboxes)
         new_bboxes[:, 0] = (bboxes[:, 0] + bboxes[:, 2]) / 2  # x center
         new_bboxes[:, 1] = (bboxes[:, 1] + bboxes[:, 3]) / 2  # y center

ppocr/data/imaug/table_ops.py

@@ -206,7 +206,7 @@ class ResizeTableImage(object):
             data['bboxes'] = data['bboxes'] * ratio
         data['image'] = resize_img
         data['src_img'] = img
-        data['shape'] = np.array([resize_h, resize_w, ratio, ratio])
+        data['shape'] = np.array([height, width, ratio, ratio])
         data['max_len'] = self.max_len
         return data
 

ppocr/losses/__init__.py

@@ -52,7 +52,7 @@ from .basic_loss import DistanceLoss
 from .combined_loss import CombinedLoss
 
 # table loss
-from .table_att_loss import TableAttentionLoss
+from .table_att_loss import TableAttentionLoss, SLALoss
 from .table_master_loss import TableMasterLoss
 # vqa token loss
 from .vqa_token_layoutlm_loss import VQASerTokenLayoutLMLoss
@@ -67,7 +67,8 @@ def build_loss(config):
         'ClsLoss', 'AttentionLoss', 'SRNLoss', 'PGLoss', 'CombinedLoss',
         'CELoss', 'TableAttentionLoss', 'SARLoss', 'AsterLoss', 'SDMGRLoss',
         'VQASerTokenLayoutLMLoss', 'LossFromOutput', 'PRENLoss', 'MultiLoss',
-        'TableMasterLoss', 'SPINAttentionLoss', 'VLLoss', 'StrokeFocusLoss'
+        'TableMasterLoss', 'SPINAttentionLoss', 'VLLoss', 'StrokeFocusLoss',
+        'SLALoss'
     ]
     config = copy.deepcopy(config)
     module_name = config.pop('name')

ppocr/losses/table_att_loss.py

@@ -22,65 +22,11 @@ from paddle.nn import functional as F
 
 
 class TableAttentionLoss(nn.Layer):
-    def __init__(self,
-                 structure_weight,
-                 loc_weight,
-                 use_giou=False,
-                 giou_weight=1.0,
-                 **kwargs):
+    def __init__(self, structure_weight, loc_weight, **kwargs):
         super(TableAttentionLoss, self).__init__()
         self.loss_func = nn.CrossEntropyLoss(weight=None, reduction='none')
         self.structure_weight = structure_weight
         self.loc_weight = loc_weight
-        self.use_giou = use_giou
-        self.giou_weight = giou_weight
-
-    def giou_loss(self, preds, bbox, eps=1e-7, reduction='mean'):
-        '''
-        :param preds:[[x1,y1,x2,y2], [x1,y1,x2,y2],,,]
-        :param bbox:[[x1,y1,x2,y2], [x1,y1,x2,y2],,,]
-        :return: loss
-        '''
-        ix1 = paddle.maximum(preds[:, 0], bbox[:, 0])
-        iy1 = paddle.maximum(preds[:, 1], bbox[:, 1])
-        ix2 = paddle.minimum(preds[:, 2], bbox[:, 2])
-        iy2 = paddle.minimum(preds[:, 3], bbox[:, 3])
-
-        iw = paddle.clip(ix2 - ix1 + 1e-3, 0., 1e10)
-        ih = paddle.clip(iy2 - iy1 + 1e-3, 0., 1e10)
-
-        # overlap
-        inters = iw * ih
-
-        # union
-        uni = (preds[:, 2] - preds[:, 0] + 1e-3) * (
-            preds[:, 3] - preds[:, 1] + 1e-3) + (bbox[:, 2] - bbox[:, 0] + 1e-3
-                                                 ) * (bbox[:, 3] - bbox[:, 1] +
-                                                      1e-3) - inters + eps
-
-        # ious
-        ious = inters / uni
-
-        ex1 = paddle.minimum(preds[:, 0], bbox[:, 0])
-        ey1 = paddle.minimum(preds[:, 1], bbox[:, 1])
-        ex2 = paddle.maximum(preds[:, 2], bbox[:, 2])
-        ey2 = paddle.maximum(preds[:, 3], bbox[:, 3])
-        ew = paddle.clip(ex2 - ex1 + 1e-3, 0., 1e10)
-        eh = paddle.clip(ey2 - ey1 + 1e-3, 0., 1e10)
-
-        # enclose erea
-        enclose = ew * eh + eps
-        giou = ious - (enclose - uni) / enclose
-
-        loss = 1 - giou
-
-        if reduction == 'mean':
-            loss = paddle.mean(loss)
-        elif reduction == 'sum':
-            loss = paddle.sum(loss)
-        else:
-            raise NotImplementedError
-        return loss
 
     def forward(self, predicts, batch):
         structure_probs = predicts['structure_probs']
@@ -100,20 +46,48 @@ class TableAttentionLoss(nn.Layer):
         loc_targets_mask = loc_targets_mask[:, 1:, :]
         loc_loss = F.mse_loss(loc_preds * loc_targets_mask,
                               loc_targets) * self.loc_weight
-        if self.use_giou:
-            loc_loss_giou = self.giou_loss(loc_preds * loc_targets_mask,
-                                           loc_targets) * self.giou_weight
-            total_loss = structure_loss + loc_loss + loc_loss_giou
-            return {
-                'loss': total_loss,
-                "structure_loss": structure_loss,
-                "loc_loss": loc_loss,
-                "loc_loss_giou": loc_loss_giou
-            }
-        else:
-            total_loss = structure_loss + loc_loss
-            return {
-                'loss': total_loss,
-                "structure_loss": structure_loss,
-                "loc_loss": loc_loss
-            }
+
+        total_loss = structure_loss + loc_loss
+        return {
+            'loss': total_loss,
+            "structure_loss": structure_loss,
+            "loc_loss": loc_loss
+        }
+
+
+class SLALoss(nn.Layer):
+    def __init__(self, structure_weight, loc_weight, loc_loss='mse', **kwargs):
+        super(SLALoss, self).__init__()
+        self.loss_func = nn.CrossEntropyLoss(weight=None, reduction='mean')
+        self.structure_weight = structure_weight
+        self.loc_weight = loc_weight
+        self.loc_loss = loc_loss
+        self.eps = 1e-12
+
+    def forward(self, predicts, batch):
+        structure_probs = predicts['structure_probs']
+        structure_targets = batch[1].astype("int64")
+        structure_targets = structure_targets[:, 1:]
+
+        structure_loss = self.loss_func(structure_probs, structure_targets)
+
+        structure_loss = paddle.mean(structure_loss) * self.structure_weight
+
+        loc_preds = predicts['loc_preds']
+        loc_targets = batch[2].astype("float32")
+        loc_targets_mask = batch[3].astype("float32")
+        loc_targets = loc_targets[:, 1:, :]
+        loc_targets_mask = loc_targets_mask[:, 1:, :]
+
+        loc_loss = F.smooth_l1_loss(
+            loc_preds * loc_targets_mask,
+            loc_targets * loc_targets_mask,
+            reduction='sum') * self.loc_weight
+
+        loc_loss = loc_loss / (loc_targets_mask.sum() + self.eps)
+        total_loss = structure_loss + loc_loss
+        return {
+            'loss': total_loss,
+            "structure_loss": structure_loss,
+            "loc_loss": loc_loss
+        }

ppocr/metrics/table_metric.py

@@ -16,9 +16,14 @@ from ppocr.metrics.det_metric import DetMetric
 
 
 class TableStructureMetric(object):
-    def __init__(self, main_indicator='acc', eps=1e-6, **kwargs):
+    def __init__(self,
+                 main_indicator='acc',
+                 eps=1e-6,
+                 del_thead_tbody=False,
+                 **kwargs):
         self.main_indicator = main_indicator
         self.eps = eps
+        self.del_thead_tbody = del_thead_tbody
         self.reset()
 
     def __call__(self, pred_label, batch=None, *args, **kwargs):
@@ -31,6 +36,13 @@ class TableStructureMetric(object):
                                              gt_structure_batch_list):
             pred_str = ''.join(pred)
             target_str = ''.join(target)
+            if self.del_thead_tbody:
+                pred_str = pred_str.replace('<thead>', '').replace(
+                    '</thead>', '').replace('<tbody>', '').replace('</tbody>',
+                                                                   '')
+                target_str = target_str.replace('<thead>', '').replace(
+                    '</thead>', '').replace('<tbody>', '').replace('</tbody>',
+                                                                   '')
             if pred_str == target_str:
                 correct_num += 1
             all_num += 1
@@ -59,7 +71,8 @@ class TableMetric(object):
     def __init__(self,
                  main_indicator='acc',
                  compute_bbox_metric=False,
-                 point_num=2,
+                 box_format='xyxy',
+                 del_thead_tbody=False,
                  **kwargs):
         """
 
@@ -67,10 +80,11 @@ class TableMetric(object):
         @param main_matric: main_matric for save best_model
         @param kwargs:
         """
-        self.structure_metric = TableStructureMetric()
+        self.structure_metric = TableStructureMetric(
+            del_thead_tbody=del_thead_tbody)
         self.bbox_metric = DetMetric() if compute_bbox_metric else None
         self.main_indicator = main_indicator
-        self.point_num = point_num
+        self.box_format = box_format
         self.reset()
 
     def __call__(self, pred_label, batch=None, *args, **kwargs):
@@ -129,10 +143,14 @@ class TableMetric(object):
             self.bbox_metric.reset()
 
     def format_box(self, box):
-        if self.point_num == 2:
+        if self.box_format == 'xyxy':
             x1, y1, x2, y2 = box
             box = [[x1, y1], [x2, y1], [x2, y2], [x1, y2]]
-        elif self.point_num == 4:
+        elif self.box_format == 'xywh':
+            x, y, w, h = box
+            x1, y1, x2, y2 = x - w // 2, y - h // 2, x + w // 2, y + h // 2
+            box = [[x1, y1], [x2, y1], [x2, y2], [x1, y2]]
+        elif self.box_format == 'xyxyxyxy':
             x1, y1, x2, y2, x3, y3, x4, y4 = box
             box = [[x1, y1], [x2, y2], [x3, y3], [x4, y4]]
         return box

ppocr/modeling/backbones/__init__.py

@@ -21,7 +21,10 @@ def build_backbone(config, model_type):
         from .det_resnet import ResNet
         from .det_resnet_vd import ResNet_vd
         from .det_resnet_vd_sast import ResNet_SAST
-        support_dict = ["MobileNetV3", "ResNet", "ResNet_vd", "ResNet_SAST"]
+        from .det_pp_lcnet import PPLCNet
+        support_dict = [
+            "MobileNetV3", "ResNet", "ResNet_vd", "ResNet_SAST", "PPLCNet"
+        ]
         if model_type == "table":
             from .table_master_resnet import TableResNetExtra
             support_dict.append('TableResNetExtra')

ppocr/modeling/backbones/det_pp_lcnet.py

+# copyright (c) 2021 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import absolute_import, division, print_function
+
+import os
+import paddle
+import paddle.nn as nn
+from paddle import ParamAttr
+from paddle.nn import AdaptiveAvgPool2D, BatchNorm, Conv2D, Dropout, Linear
+from paddle.regularizer import L2Decay
+from paddle.nn.initializer import KaimingNormal
+from paddle.utils.download import get_path_from_url
+
+MODEL_URLS = {
+    "PPLCNet_x0.25":
+    "https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/legendary_models/PPLCNet_x0_25_pretrained.pdparams",
+    "PPLCNet_x0.35":
+    "https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/legendary_models/PPLCNet_x0_35_pretrained.pdparams",
+    "PPLCNet_x0.5":
+    "https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/legendary_models/PPLCNet_x0_5_pretrained.pdparams",
+    "PPLCNet_x0.75":
+    "https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/legendary_models/PPLCNet_x0_75_pretrained.pdparams",
+    "PPLCNet_x1.0":
+    "https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/legendary_models/PPLCNet_x1_0_pretrained.pdparams",
+    "PPLCNet_x1.5":
+    "https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/legendary_models/PPLCNet_x1_5_pretrained.pdparams",
+    "PPLCNet_x2.0":
+    "https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/legendary_models/PPLCNet_x2_0_pretrained.pdparams",
+    "PPLCNet_x2.5":
+    "https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/legendary_models/PPLCNet_x2_5_pretrained.pdparams"
+}
+
+MODEL_STAGES_PATTERN = {
+    "PPLCNet": ["blocks2", "blocks3", "blocks4", "blocks5", "blocks6"]
+}
+
+__all__ = list(MODEL_URLS.keys())
+
+# Each element(list) represents a depthwise block, which is composed of k, in_c, out_c, s, use_se.
+# k: kernel_size
+# in_c: input channel number in depthwise block
+# out_c: output channel number in depthwise block
+# s: stride in depthwise block
+# use_se: whether to use SE block
+
+NET_CONFIG = {
+    "blocks2":
+    # k, in_c, out_c, s, use_se
+    [[3, 16, 32, 1, False]],
+    "blocks3": [[3, 32, 64, 2, False], [3, 64, 64, 1, False]],
+    "blocks4": [[3, 64, 128, 2, False], [3, 128, 128, 1, False]],
+    "blocks5":
+    [[3, 128, 256, 2, False], [5, 256, 256, 1, False], [5, 256, 256, 1, False],
+     [5, 256, 256, 1, False], [5, 256, 256, 1, False], [5, 256, 256, 1, False]],
+    "blocks6": [[5, 256, 512, 2, True], [5, 512, 512, 1, True]]
+}
+
+
+def make_divisible(v, divisor=8, min_value=None):
+    if min_value is None:
+        min_value = divisor
+    new_v = max(min_value, int(v + divisor / 2) // divisor * divisor)
+    if new_v < 0.9 * v:
+        new_v += divisor
+    return new_v
+
+
+class ConvBNLayer(nn.Layer):
+    def __init__(self,
+                 num_channels,
+                 filter_size,
+                 num_filters,
+                 stride,
+                 num_groups=1):
+        super().__init__()
+
+        self.conv = Conv2D(
+            in_channels=num_channels,
+            out_channels=num_filters,
+            kernel_size=filter_size,
+            stride=stride,
+            padding=(filter_size - 1) // 2,
+            groups=num_groups,
+            weight_attr=ParamAttr(initializer=KaimingNormal()),
+            bias_attr=False)
+
+        self.bn = BatchNorm(
+            num_filters,
+            param_attr=ParamAttr(regularizer=L2Decay(0.0)),
+            bias_attr=ParamAttr(regularizer=L2Decay(0.0)))
+        self.hardswish = nn.Hardswish()
+
+    def forward(self, x):
+        x = self.conv(x)
+        x = self.bn(x)
+        x = self.hardswish(x)
+        return x
+
+
+class DepthwiseSeparable(nn.Layer):
+    def __init__(self,
+                 num_channels,
+                 num_filters,
+                 stride,
+                 dw_size=3,
+                 use_se=False):
+        super().__init__()
+        self.use_se = use_se
+        self.dw_conv = ConvBNLayer(
+            num_channels=num_channels,
+            num_filters=num_channels,
+            filter_size=dw_size,
+            stride=stride,
+            num_groups=num_channels)
+        if use_se:
+            self.se = SEModule(num_channels)
+        self.pw_conv = ConvBNLayer(
+            num_channels=num_channels,
+            filter_size=1,
+            num_filters=num_filters,
+            stride=1)
+
+    def forward(self, x):
+        x = self.dw_conv(x)
+        if self.use_se:
+            x = self.se(x)
+        x = self.pw_conv(x)
+        return x
+
+
+class SEModule(nn.Layer):
+    def __init__(self, channel, reduction=4):
+        super().__init__()
+        self.avg_pool = AdaptiveAvgPool2D(1)
+        self.conv1 = Conv2D(
+            in_channels=channel,
+            out_channels=channel // reduction,
+            kernel_size=1,
+            stride=1,
+            padding=0)
+        self.relu = nn.ReLU()
+        self.conv2 = Conv2D(
+            in_channels=channel // reduction,
+            out_channels=channel,
+            kernel_size=1,
+            stride=1,
+            padding=0)
+        self.hardsigmoid = nn.Hardsigmoid()
+
+    def forward(self, x):
+        identity = x
+        x = self.avg_pool(x)
+        x = self.conv1(x)
+        x = self.relu(x)
+        x = self.conv2(x)
+        x = self.hardsigmoid(x)
+        x = paddle.multiply(x=identity, y=x)
+        return x
+
+
+class PPLCNet(nn.Layer):
+    def __init__(self,
+                 in_channels=3,
+                 scale=1.0,
+                 pretrained=False,
+                 use_ssld=False):
+        super().__init__()
+        self.out_channels = [
+            int(NET_CONFIG["blocks3"][-1][2] * scale),
+            int(NET_CONFIG["blocks4"][-1][2] * scale),
+            int(NET_CONFIG["blocks5"][-1][2] * scale),
+            int(NET_CONFIG["blocks6"][-1][2] * scale)
+        ]
+        self.scale = scale
+
+        self.conv1 = ConvBNLayer(
+            num_channels=in_channels,
+            filter_size=3,
+            num_filters=make_divisible(16 * scale),
+            stride=2)
+
+        self.blocks2 = nn.Sequential(* [
+            DepthwiseSeparable(
+                num_channels=make_divisible(in_c * scale),
+                num_filters=make_divisible(out_c * scale),
+                dw_size=k,
+                stride=s,
+                use_se=se)
+            for i, (k, in_c, out_c, s, se) in enumerate(NET_CONFIG["blocks2"])
+        ])
+
+        self.blocks3 = nn.Sequential(* [
+            DepthwiseSeparable(
+                num_channels=make_divisible(in_c * scale),
+                num_filters=make_divisible(out_c * scale),
+                dw_size=k,
+                stride=s,
+                use_se=se)
+            for i, (k, in_c, out_c, s, se) in enumerate(NET_CONFIG["blocks3"])
+        ])
+
+        self.blocks4 = nn.Sequential(* [
+            DepthwiseSeparable(
+                num_channels=make_divisible(in_c * scale),
+                num_filters=make_divisible(out_c * scale),
+                dw_size=k,
+                stride=s,
+                use_se=se)
+            for i, (k, in_c, out_c, s, se) in enumerate(NET_CONFIG["blocks4"])
+        ])
+
+        self.blocks5 = nn.Sequential(* [
+            DepthwiseSeparable(
+                num_channels=make_divisible(in_c * scale),
+                num_filters=make_divisible(out_c * scale),
+                dw_size=k,
+                stride=s,
+                use_se=se)
+            for i, (k, in_c, out_c, s, se) in enumerate(NET_CONFIG["blocks5"])
+        ])
+
+        self.blocks6 = nn.Sequential(* [
+            DepthwiseSeparable(
+                num_channels=make_divisible(in_c * scale),
+                num_filters=make_divisible(out_c * scale),
+                dw_size=k,
+                stride=s,
+                use_se=se)
+            for i, (k, in_c, out_c, s, se) in enumerate(NET_CONFIG["blocks6"])
+        ])
+
+        if pretrained:
+            self._load_pretrained(
+                MODEL_URLS['PPLCNet_x{}'.format(scale)], use_ssld=use_ssld)
+
+    def forward(self, x):
+        outs = []
+        x = self.conv1(x)
+        x = self.blocks2(x)
+        x = self.blocks3(x)
+        outs.append(x)
+        x = self.blocks4(x)
+        outs.append(x)
+        x = self.blocks5(x)
+        outs.append(x)
+        x = self.blocks6(x)
+        outs.append(x)
+        return outs
+
+    def _load_pretrained(self, pretrained_url, use_ssld=False):
+        if use_ssld:
+            pretrained_url = pretrained_url.replace("_pretrained",
+                                                    "_ssld_pretrained")
+        print(pretrained_url)
+        local_weight_path = get_path_from_url(
+            pretrained_url, os.path.expanduser("~/.paddleclas/weights"))
+        param_state_dict = paddle.load(local_weight_path)
+        self.set_dict(param_state_dict)
+        return

ppocr/modeling/heads/__init__.py

@@ -44,7 +44,7 @@ def build_head(config):
     #kie head
     from .kie_sdmgr_head import SDMGRHead
 
-    from .table_att_head import TableAttentionHead
+    from .table_att_head import TableAttentionHead, SLAHead
     from .table_master_head import TableMasterHead
 
     support_dict = [
@@ -52,7 +52,7 @@ def build_head(config):
         'ClsHead', 'AttentionHead', 'SRNHead', 'PGHead', 'Transformer',
         'TableAttentionHead', 'SARHead', 'AsterHead', 'SDMGRHead', 'PRENHead',
         'MultiHead', 'ABINetHead', 'TableMasterHead', 'SPINAttentionHead',
-        'VLHead', 'RobustScannerHead'
+        'VLHead', 'SLAHead', 'RobustScannerHead'
     ]
 
     #table head

ppocr/modeling/heads/table_att_head.py

@@ -18,12 +18,26 @@ from __future__ import print_function
 
 import paddle
 import paddle.nn as nn
+from paddle import ParamAttr
 import paddle.nn.functional as F
 import numpy as np
 
 from .rec_att_head import AttentionGRUCell
 
 
+def get_para_bias_attr(l2_decay, k):
+    if l2_decay > 0:
+        regularizer = paddle.regularizer.L2Decay(l2_decay)
+        stdv = 1.0 / math.sqrt(k * 1.0)
+        initializer = nn.initializer.Uniform(-stdv, stdv)
+    else:
+        regularizer = None
+        initializer = None
+    weight_attr = ParamAttr(regularizer=regularizer, initializer=initializer)
+    bias_attr = ParamAttr(regularizer=regularizer, initializer=initializer)
+    return [weight_attr, bias_attr]
+
+
 class TableAttentionHead(nn.Layer):
     def __init__(self,
                  in_channels,
@@ -32,7 +46,7 @@ class TableAttentionHead(nn.Layer):
                  in_max_len=488,
                  max_text_length=800,
                  out_channels=30,
-                 point_num=2,
+                 loc_reg_num=4,
                  **kwargs):
         super(TableAttentionHead, self).__init__()
         self.input_size = in_channels[-1]
@@ -56,7 +70,7 @@ class TableAttentionHead(nn.Layer):
             else:
                 self.loc_fea_trans = nn.Linear(256, self.max_text_length + 1)
             self.loc_generator = nn.Linear(self.input_size + hidden_size,
-                                           point_num * 2)
+                                           loc_reg_num)
 
     def _char_to_onehot(self, input_char, onehot_dim):
         input_ont_hot = F.one_hot(input_char, onehot_dim)
@@ -129,3 +143,121 @@ class TableAttentionHead(nn.Layer):
                 loc_preds = self.loc_generator(loc_concat)
                 loc_preds = F.sigmoid(loc_preds)
         return {'structure_probs': structure_probs, 'loc_preds': loc_preds}
+
+
+class SLAHead(nn.Layer):
+    def __init__(self,
+                 in_channels,
+                 hidden_size,
+                 out_channels=30,
+                 max_text_length=500,
+                 loc_reg_num=4,
+                 fc_decay=0.0,
+                 **kwargs):
+        """
+        @param in_channels: input shape
+        @param hidden_size: hidden_size for RNN and Embedding
+        @param out_channels: num_classes to rec
+        @param max_text_length: max text pred
+        """
+        super().__init__()
+        in_channels = in_channels[-1]
+        self.hidden_size = hidden_size
+        self.max_text_length = max_text_length
+        self.emb = self._char_to_onehot
+        self.num_embeddings = out_channels
+
+        # structure
+        self.structure_attention_cell = AttentionGRUCell(
+            in_channels, hidden_size, self.num_embeddings)
+        weight_attr, bias_attr = get_para_bias_attr(
+            l2_decay=fc_decay, k=hidden_size)
+        weight_attr1_1, bias_attr1_1 = get_para_bias_attr(
+            l2_decay=fc_decay, k=hidden_size)
+        weight_attr1_2, bias_attr1_2 = get_para_bias_attr(
+            l2_decay=fc_decay, k=hidden_size)
+        self.structure_generator = nn.Sequential(
+            nn.Linear(
+                self.hidden_size,
+                self.hidden_size,
+                weight_attr=weight_attr1_2,
+                bias_attr=bias_attr1_2),
+            nn.Linear(
+                hidden_size,
+                out_channels,
+                weight_attr=weight_attr,
+                bias_attr=bias_attr))
+        # loc
+        weight_attr1, bias_attr1 = get_para_bias_attr(
+            l2_decay=fc_decay, k=self.hidden_size)
+        weight_attr2, bias_attr2 = get_para_bias_attr(
+            l2_decay=fc_decay, k=self.hidden_size)
+        self.loc_generator = nn.Sequential(
+            nn.Linear(
+                self.hidden_size,
+                self.hidden_size,
+                weight_attr=weight_attr1,
+                bias_attr=bias_attr1),
+            nn.Linear(
+                self.hidden_size,
+                loc_reg_num,
+                weight_attr=weight_attr2,
+                bias_attr=bias_attr2),
+            nn.Sigmoid())
+
+    def forward(self, inputs, targets=None):
+        fea = inputs[-1]
+        batch_size = fea.shape[0]
+        # reshape
+        fea = paddle.reshape(fea, [fea.shape[0], fea.shape[1], -1])
+        fea = fea.transpose([0, 2, 1])  # (NTC)(batch, width, channels)
+
+        hidden = paddle.zeros((batch_size, self.hidden_size))
+        structure_preds = []
+        loc_preds = []
+        if self.training and targets is not None:
+            structure = targets[0]
+            for i in range(self.max_text_length + 1):
+                hidden, structure_step, loc_step = self._decode(structure[:, i],
+                                                                fea, hidden)
+                structure_preds.append(structure_step)
+                loc_preds.append(loc_step)
+        else:
+            pre_chars = paddle.zeros(shape=[batch_size], dtype="int32")
+            max_text_length = paddle.to_tensor(self.max_text_length)
+            # for export
+            loc_step, structure_step = None, None
+            for i in range(max_text_length + 1):
+                hidden, structure_step, loc_step = self._decode(pre_chars, fea,
+                                                                hidden)
+                pre_chars = structure_step.argmax(axis=1, dtype="int32")
+                structure_preds.append(structure_step)
+                loc_preds.append(loc_step)
+        structure_preds = paddle.stack(structure_preds, axis=1)
+        loc_preds = paddle.stack(loc_preds, axis=1)
+        if not self.training:
+            structure_preds = F.softmax(structure_preds)
+        return {'structure_probs': structure_preds, 'loc_preds': loc_preds}
+
+    def _decode(self, pre_chars, features, hidden):
+        """
+        Predict table label and coordinates for each step
+        @param pre_chars: Table label in previous step
+        @param features:
+        @param hidden: hidden status in previous step
+        @return:
+        """
+        emb_feature = self.emb(pre_chars)
+        # output shape is b * self.hidden_size
+        (output, hidden), alpha = self.structure_attention_cell(
+            hidden, features, emb_feature)
+
+        # structure
+        structure_step = self.structure_generator(output)
+        # loc
+        loc_step = self.loc_generator(output)
+        return hidden, structure_step, loc_step
+
+    def _char_to_onehot(self, input_char):
+        input_ont_hot = F.one_hot(input_char, self.num_embeddings)
+        return input_ont_hot

ppocr/modeling/heads/table_master_head.py

@@ -37,7 +37,7 @@ class TableMasterHead(nn.Layer):
                  d_ff=2048,
                  dropout=0,
                  max_text_length=500,
-                 point_num=2,
+                 loc_reg_num=4,
                  **kwargs):
         super(TableMasterHead, self).__init__()
         hidden_size = in_channels[-1]
@@ -50,7 +50,7 @@ class TableMasterHead(nn.Layer):
         self.cls_fc = nn.Linear(hidden_size, out_channels)
         self.bbox_fc = nn.Sequential(
             # nn.Linear(hidden_size, hidden_size),
-            nn.Linear(hidden_size, point_num * 2),
+            nn.Linear(hidden_size, loc_reg_num),
             nn.Sigmoid())
         self.norm = nn.LayerNorm(hidden_size)
         self.embedding = Embeddings(d_model=hidden_size, vocab=out_channels)
@@ -59,7 +59,7 @@ class TableMasterHead(nn.Layer):
         self.SOS = out_channels - 3
         self.PAD = out_channels - 1
         self.out_channels = out_channels
-        self.point_num = point_num
+        self.loc_reg_num = loc_reg_num
         self.max_text_length = max_text_length
 
     def make_mask(self, tgt):
@@ -105,7 +105,7 @@ class TableMasterHead(nn.Layer):
         output = paddle.zeros(
             [input.shape[0], self.max_text_length + 1, self.out_channels])
         bbox_output = paddle.zeros(
-            [input.shape[0], self.max_text_length + 1, self.point_num * 2])
+            [input.shape[0], self.max_text_length + 1, self.loc_reg_num])
         max_text_length = paddle.to_tensor(self.max_text_length)
         for i in range(max_text_length + 1):
             target_mask = self.make_mask(input)

ppocr/modeling/necks/__init__.py

@@ -25,9 +25,10 @@ def build_neck(config):
     from .fpn import FPN
     from .fce_fpn import FCEFPN
     from .pren_fpn import PRENFPN
+    from .csp_pan import CSPPAN
     support_dict = [
         'FPN', 'FCEFPN', 'LKPAN', 'DBFPN', 'RSEFPN', 'EASTFPN', 'SASTFPN',
-        'SequenceEncoder', 'PGFPN', 'TableFPN', 'PRENFPN'
+        'SequenceEncoder', 'PGFPN', 'TableFPN', 'PRENFPN', 'CSPPAN'
     ]
 
     module_name = config.pop('name')

ppocr/modeling/necks/csp_pan.py

+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# The code is based on:
+# https://github.com/PaddlePaddle/PaddleDetection/blob/release%2F2.3/ppdet/modeling/necks/csp_pan.py
+
+import paddle
+import paddle.nn as nn
+import paddle.nn.functional as F
+from paddle import ParamAttr
+
+__all__ = ['CSPPAN']
+
+
+class ConvBNLayer(nn.Layer):
+    def __init__(self,
+                 in_channel=96,
+                 out_channel=96,
+                 kernel_size=3,
+                 stride=1,
+                 groups=1,
+                 act='leaky_relu'):
+        super(ConvBNLayer, self).__init__()
+        initializer = nn.initializer.KaimingUniform()
+        self.act = act
+        assert self.act in ['leaky_relu', "hard_swish"]
+        self.conv = nn.Conv2D(
+            in_channels=in_channel,
+            out_channels=out_channel,
+            kernel_size=kernel_size,
+            groups=groups,
+            padding=(kernel_size - 1) // 2,
+            stride=stride,
+            weight_attr=ParamAttr(initializer=initializer),
+            bias_attr=False)
+        self.bn = nn.BatchNorm2D(out_channel)
+
+    def forward(self, x):
+        x = self.bn(self.conv(x))
+        if self.act == "leaky_relu":
+            x = F.leaky_relu(x)
+        elif self.act == "hard_swish":
+            x = F.hardswish(x)
+        return x
+
+
+class DPModule(nn.Layer):
+    """
+    Depth-wise and point-wise module.
+     Args:
+        in_channel (int): The input channels of this Module.
+        out_channel (int): The output channels of this Module.
+        kernel_size (int): The conv2d kernel size of this Module.
+        stride (int): The conv2d's stride of this Module.
+        act (str): The activation function of this Module,
+                   Now support `leaky_relu` and `hard_swish`.
+    """
+
+    def __init__(self,
+                 in_channel=96,
+                 out_channel=96,
+                 kernel_size=3,
+                 stride=1,
+                 act='leaky_relu'):
+        super(DPModule, self).__init__()
+        initializer = nn.initializer.KaimingUniform()
+        self.act = act
+        self.dwconv = nn.Conv2D(
+            in_channels=in_channel,
+            out_channels=out_channel,
+            kernel_size=kernel_size,
+            groups=out_channel,
+            padding=(kernel_size - 1) // 2,
+            stride=stride,
+            weight_attr=ParamAttr(initializer=initializer),
+            bias_attr=False)
+        self.bn1 = nn.BatchNorm2D(out_channel)
+        self.pwconv = nn.Conv2D(
+            in_channels=out_channel,
+            out_channels=out_channel,
+            kernel_size=1,
+            groups=1,
+            padding=0,
+            weight_attr=ParamAttr(initializer=initializer),
+            bias_attr=False)
+        self.bn2 = nn.BatchNorm2D(out_channel)
+
+    def act_func(self, x):
+        if self.act == "leaky_relu":
+            x = F.leaky_relu(x)
+        elif self.act == "hard_swish":
+            x = F.hardswish(x)
+        return x
+
+    def forward(self, x):
+        x = self.act_func(self.bn1(self.dwconv(x)))
+        x = self.act_func(self.bn2(self.pwconv(x)))
+        return x
+
+
+class DarknetBottleneck(nn.Layer):
+    """The basic bottleneck block used in Darknet.
+    Each Block consists of two ConvModules and the input is added to the
+    final output. Each ConvModule is composed of Conv, BN, and act.
+    The first convLayer has filter size of 1x1 and the second one has the
+    filter size of 3x3.
+    Args:
+        in_channels (int): The input channels of this Module.
+        out_channels (int): The output channels of this Module.
+        expansion (int): The kernel size of the convolution. Default: 0.5
+        add_identity (bool): Whether to add identity to the out.
+            Default: True
+        use_depthwise (bool): Whether to use depthwise separable convolution.
+            Default: False
+    """
+
+    def __init__(self,
+                 in_channels,
+                 out_channels,
+                 kernel_size=3,
+                 expansion=0.5,
+                 add_identity=True,
+                 use_depthwise=False,
+                 act="leaky_relu"):
+        super(DarknetBottleneck, self).__init__()
+        hidden_channels = int(out_channels * expansion)
+        conv_func = DPModule if use_depthwise else ConvBNLayer
+        self.conv1 = ConvBNLayer(
+            in_channel=in_channels,
+            out_channel=hidden_channels,
+            kernel_size=1,
+            act=act)
+        self.conv2 = conv_func(
+            in_channel=hidden_channels,
+            out_channel=out_channels,
+            kernel_size=kernel_size,
+            stride=1,
+            act=act)
+        self.add_identity = \
+            add_identity and in_channels == out_channels
+
+    def forward(self, x):
+        identity = x
+        out = self.conv1(x)
+        out = self.conv2(out)
+
+        if self.add_identity:
+            return out + identity
+        else:
+            return out
+
+
+class CSPLayer(nn.Layer):
+    """Cross Stage Partial Layer.
+    Args:
+        in_channels (int): The input channels of the CSP layer.
+        out_channels (int): The output channels of the CSP layer.
+        expand_ratio (float): Ratio to adjust the number of channels of the
+            hidden layer. Default: 0.5
+        num_blocks (int): Number of blocks. Default: 1
+        add_identity (bool): Whether to add identity in blocks.
+            Default: True
+        use_depthwise (bool): Whether to depthwise separable convolution in
+            blocks. Default: False
+    """
+
+    def __init__(self,
+                 in_channels,
+                 out_channels,
+                 kernel_size=3,
+                 expand_ratio=0.5,
+                 num_blocks=1,
+                 add_identity=True,
+                 use_depthwise=False,
+                 act="leaky_relu"):
+        super().__init__()
+        mid_channels = int(out_channels * expand_ratio)
+        self.main_conv = ConvBNLayer(in_channels, mid_channels, 1, act=act)
+        self.short_conv = ConvBNLayer(in_channels, mid_channels, 1, act=act)
+        self.final_conv = ConvBNLayer(
+            2 * mid_channels, out_channels, 1, act=act)
+
+        self.blocks = nn.Sequential(* [
+            DarknetBottleneck(
+                mid_channels,
+                mid_channels,
+                kernel_size,
+                1.0,
+                add_identity,
+                use_depthwise,
+                act=act) for _ in range(num_blocks)
+        ])
+
+    def forward(self, x):
+        x_short = self.short_conv(x)
+
+        x_main = self.main_conv(x)
+        x_main = self.blocks(x_main)
+
+        x_final = paddle.concat((x_main, x_short), axis=1)
+        return self.final_conv(x_final)
+
+
+class Channel_T(nn.Layer):
+    def __init__(self,
+                 in_channels=[116, 232, 464],
+                 out_channels=96,
+                 act="leaky_relu"):
+        super(Channel_T, self).__init__()
+        self.convs = nn.LayerList()
+        for i in range(len(in_channels)):
+            self.convs.append(
+                ConvBNLayer(
+                    in_channels[i], out_channels, 1, act=act))
+
+    def forward(self, x):
+        outs = [self.convs[i](x[i]) for i in range(len(x))]
+        return outs
+
+
+class CSPPAN(nn.Layer):
+    """Path Aggregation Network with CSP module.
+    Args:
+        in_channels (List[int]): Number of input channels per scale.
+        out_channels (int): Number of output channels (used at each scale)
+        kernel_size (int): The conv2d kernel size of this Module.
+        num_csp_blocks (int): Number of bottlenecks in CSPLayer. Default: 1
+        use_depthwise (bool): Whether to depthwise separable convolution in
+            blocks. Default: True
+    """
+
+    def __init__(self,
+                 in_channels,
+                 out_channels,
+                 kernel_size=5,
+                 num_csp_blocks=1,
+                 use_depthwise=True,
+                 act='hard_swish'):
+        super(CSPPAN, self).__init__()
+        self.in_channels = in_channels
+        self.out_channels = [out_channels] * len(in_channels)
+        conv_func = DPModule if use_depthwise else ConvBNLayer
+
+        self.conv_t = Channel_T(in_channels, out_channels, act=act)
+
+        # build top-down blocks
+        self.upsample = nn.Upsample(scale_factor=2, mode='nearest')
+        self.top_down_blocks = nn.LayerList()
+        for idx in range(len(in_channels) - 1, 0, -1):
+            self.top_down_blocks.append(
+                CSPLayer(
+                    out_channels * 2,
+                    out_channels,
+                    kernel_size=kernel_size,
+                    num_blocks=num_csp_blocks,
+                    add_identity=False,
+                    use_depthwise=use_depthwise,
+                    act=act))
+
+        # build bottom-up blocks
+        self.downsamples = nn.LayerList()
+        self.bottom_up_blocks = nn.LayerList()
+        for idx in range(len(in_channels) - 1):
+            self.downsamples.append(
+                conv_func(
+                    out_channels,
+                    out_channels,
+                    kernel_size=kernel_size,
+                    stride=2,
+                    act=act))
+            self.bottom_up_blocks.append(
+                CSPLayer(
+                    out_channels * 2,
+                    out_channels,
+                    kernel_size=kernel_size,
+                    num_blocks=num_csp_blocks,
+                    add_identity=False,
+                    use_depthwise=use_depthwise,
+                    act=act))
+
+    def forward(self, inputs):
+        """
+        Args:
+            inputs (tuple[Tensor]): input features.
+        Returns:
+            tuple[Tensor]: CSPPAN features.
+        """
+        assert len(inputs) == len(self.in_channels)
+        inputs = self.conv_t(inputs)
+
+        # top-down path
+        inner_outs = [inputs[-1]]
+        for idx in range(len(self.in_channels) - 1, 0, -1):
+            feat_heigh = inner_outs[0]
+            feat_low = inputs[idx - 1]
+            upsample_feat = F.upsample(
+                feat_heigh, size=paddle.shape(feat_low)[2:4], mode="nearest")
+
+            inner_out = self.top_down_blocks[len(self.in_channels) - 1 - idx](
+                paddle.concat([upsample_feat, feat_low], 1))
+            inner_outs.insert(0, inner_out)
+
+        # bottom-up path
+        outs = [inner_outs[0]]
+        for idx in range(len(self.in_channels) - 1):
+            feat_low = outs[-1]
+            feat_height = inner_outs[idx + 1]
+            downsample_feat = self.downsamples[idx](feat_low)
+            out = self.bottom_up_blocks[idx](paddle.concat(
+                [downsample_feat, feat_height], 1))
+            outs.append(out)
+
+        return tuple(outs)

ppocr/postprocess/table_postprocess.py

@@ -21,9 +21,29 @@ from .rec_postprocess import AttnLabelDecode
 class TableLabelDecode(AttnLabelDecode):
     """  """
 
-    def __init__(self, character_dict_path, **kwargs):
-        super(TableLabelDecode, self).__init__(character_dict_path)
-        self.td_token = ['<td>', '<td', '<eb></eb>', '<td></td>']
+    def __init__(self,
+                 character_dict_path,
+                 merge_no_span_structure=False,
+                 **kwargs):
+        dict_character = []
+        with open(character_dict_path, "rb") as fin:
+            lines = fin.readlines()
+            for line in lines:
+                line = line.decode('utf-8').strip("\n").strip("\r\n")
+                dict_character.append(line)
+
+        if merge_no_span_structure:
+            if "<td></td>" not in dict_character:
+                dict_character.append("<td></td>")
+            if "<td>" in dict_character:
+                dict_character.remove("<td>")
+
+        dict_character = self.add_special_char(dict_character)
+        self.dict = {}
+        for i, char in enumerate(dict_character):
+            self.dict[char] = i
+        self.character = dict_character
+        self.td_token = ['<td>', '<td', '<td></td>']
 
     def __call__(self, preds, batch=None):
         structure_probs = preds['structure_probs']
@@ -114,18 +134,21 @@ class TableLabelDecode(AttnLabelDecode):
 
     def _bbox_decode(self, bbox, shape):
         h, w, ratio_h, ratio_w, pad_h, pad_w = shape
-        src_h = h / ratio_h
-        src_w = w / ratio_w
-        bbox[0::2] *= src_w
-        bbox[1::2] *= src_h
+        bbox[0::2] *= w
+        bbox[1::2] *= h
         return bbox
 
 
 class TableMasterLabelDecode(TableLabelDecode):
     """  """
 
-    def __init__(self, character_dict_path, box_shape='ori', **kwargs):
-        super(TableMasterLabelDecode, self).__init__(character_dict_path)
+    def __init__(self,
+                 character_dict_path,
+                 box_shape='ori',
+                 merge_no_span_structure=True,
+                 **kwargs):
+        super(TableMasterLabelDecode, self).__init__(character_dict_path,
+                                                     merge_no_span_structure)
         self.box_shape = box_shape
         assert box_shape in [
             'ori', 'pad'
@@ -157,4 +180,7 @@ class TableMasterLabelDecode(TableLabelDecode):
         bbox[1::2] *= h
         bbox[0::2] /= ratio_w
         bbox[1::2] /= ratio_h
+        x, y, w, h = bbox
+        x1, y1, x2, y2 = x - w // 2, y - h // 2, x + w // 2, y + h // 2
+        bbox = np.array([x1, y1, x2, y2])
         return bbox

ppocr/utils/dict/table_structure_dict_ch.txt

+<thead>
+</thead>
+<tbody>
+</tbody>
+<tr>
+</tr>
+<td>
+<td
+>
+</td>
+ colspan="2"
+ colspan="3"
+ colspan="4"
+ colspan="5"
+ colspan="6"
+ colspan="7"
+ colspan="8"
+ colspan="9"
+ colspan="10"
+ colspan="11"
+ colspan="12"
+ colspan="13"
+ colspan="14"
+ colspan="15"
+ colspan="16"
+ colspan="17"
+ colspan="18"
+ colspan="19"
+ colspan="20"
+ rowspan="2"
+ rowspan="3"
+ rowspan="4"
+ rowspan="5"
+ rowspan="6"
+ rowspan="7"
+ rowspan="8"
+ rowspan="9"
+ rowspan="10"
+ rowspan="11"
+ rowspan="12"
+ rowspan="13"
+ rowspan="14"
+ rowspan="15"
+ rowspan="16"
+ rowspan="17"
+ rowspan="18"
+ rowspan="19"
+ rowspan="20"

ppocr/utils/network.py

@@ -41,9 +41,7 @@ def download_with_progressbar(url, save_path):
 
 def maybe_download(model_storage_directory, url):
     # using custom model
-    tar_file_name_list = [
-        'inference.pdiparams', 'inference.pdiparams.info', 'inference.pdmodel'
-    ]
+    tar_file_name_list = ['.pdiparams', '.pdiparams.info', '.pdmodel']
     if not os.path.exists(
             os.path.join(model_storage_directory, 'inference.pdiparams')
     ) or not os.path.exists(
@@ -57,8 +55,8 @@ def maybe_download(model_storage_directory, url):
             for member in tarObj.getmembers():
                 filename = None
                 for tar_file_name in tar_file_name_list:
-                    if tar_file_name in member.name:
-                        filename = tar_file_name
+                    if member.name.endswith(tar_file_name):
+                        filename = 'inference' + tar_file_name
                 if filename is None:
                     continue
                 file = tarObj.extractfile(member)

ppocr/utils/visual.py

@@ -113,14 +113,11 @@ def draw_re_results(image,
     return np.array(img_new)
 
 
-def draw_rectangle(img_path, boxes, use_xywh=False):
+def draw_rectangle(img_path, boxes):
+    boxes = np.array(boxes)
     img = cv2.imread(img_path)
     img_show = img.copy()
     for box in boxes.astype(int):
-        if use_xywh:
-            x, y, w, h = box
-            x1, y1, x2, y2 = x - w // 2, y - h // 2, x + w // 2, y + h // 2
-        else:
-            x1, y1, x2, y2 = box
+        x1, y1, x2, y2 = box
         cv2.rectangle(img_show, (x1, y1), (x2, y2), (255, 0, 0), 2)
     return img_show
\ No newline at end of file

ppstructure/README.md

@@ -106,9 +106,9 @@ PP-Structure Series Model List (Updating)
 
 |model name|description|model size|download|
 | --- | --- | --- | --- |
-|ch_PP-OCRv2_det_slim|[New] Slim quantization with distillation lightweight model, supporting Chinese, English, multilingual text detection| 3M |[inference model](https://paddleocr.bj.bcebos.com/PP-OCRv2/chinese/ch_PP-OCRv2_det_slim_quant_infer.tar)|
-|ch_PP-OCRv2_rec_slim|[New] Slim qunatization with distillation lightweight model, supporting Chinese, English, multilingual text recognition| 9M |[inference model](https://paddleocr.bj.bcebos.com/PP-OCRv2/chinese/ch_PP-OCRv2_rec_slim_quant_infer.tar) / [trained model](https://paddleocr.bj.bcebos.com/PP-OCRv2/chinese/ch_PP-OCRv2_rec_slim_quant_train.tar) |
-|en_ppocr_mobile_v2.0_table_structure|Table structure prediction of English table scene trained on PubLayNet dataset| 18.6M |[inference model](https://paddleocr.bj.bcebos.com/dygraph_v2.0/table/en_ppocr_mobile_v2.0_table_structure_infer.tar) / [trained model](https://paddleocr.bj.bcebos.com/dygraph_v2.1/table/en_ppocr_mobile_v2.0_table_structure_train.tar) |
+|ch_PP-OCRv3_det_slim|[New] slim quantization with distillation lightweight model, supporting Chinese, English, multilingual text detection| 1.1M |[inference model](https://paddleocr.bj.bcebos.com/PP-OCRv3/chinese/ch_PP-OCRv3_det_slim_infer.tar) / [trained model](https://paddleocr.bj.bcebos.com/PP-OCRv3/chinese/ch_PP-OCRv3_det_slim_distill_train.tar)|
+|ch_PP-OCRv3_rec_slim |[New] Slim qunatization with distillation lightweight model, supporting Chinese, English text recognition| 4.9M |[inference model](https://paddleocr.bj.bcebos.com/PP-OCRv3/chinese/ch_PP-OCRv3_rec_slim_infer.tar) / [trained model](https://paddleocr.bj.bcebos.com/PP-OCRv3/chinese/ch_PP-OCRv3_rec_slim_train.tar) |
+|ch_ppstructure_mobile_v2.0_SLANet|Chinese table recognition model trained on PubTabNet dataset based on SLANet|9.3M|[inference model](https://paddleocr.bj.bcebos.com/ppstructure/models/slanet/ch_ppstructure_mobile_v2.0_SLANet_infer.tar) / [trained model](https://paddleocr.bj.bcebos.com/ppstructure/models/slanet/ch_ppstructure_mobile_v2.0_SLANet_train.tar) |
 
 ### 7.3 DOC-VQA model
 

ppstructure/README_ch.md

@@ -120,9 +120,10 @@ PP-Structure系列模型列表（更新中）
 
 |模型名称|模型简介|模型大小|下载地址|
 | --- | --- | --- | --- |
-|ch_PP-OCRv2_det_slim|【最新】slim量化+蒸馏版超轻量模型，支持中英文、多语种文本检测| 3M |[推理模型](https://paddleocr.bj.bcebos.com/PP-OCRv2/chinese/ch_PP-OCRv2_det_slim_quant_infer.tar)|
-|ch_PP-OCRv2_rec_slim|【最新】slim量化版超轻量模型，支持中英文、数字识别| 9M |[推理模型](https://paddleocr.bj.bcebos.com/PP-OCRv2/chinese/ch_PP-OCRv2_rec_slim_quant_infer.tar) / [训练模型](https://paddleocr.bj.bcebos.com/PP-OCRv2/chinese/ch_PP-OCRv2_rec_slim_quant_train.tar) |
-|en_ppocr_mobile_v2.0_table_structure|PubLayNet数据集训练的英文表格场景的表格结构预测|18.6M|[推理模型](https://paddleocr.bj.bcebos.com/dygraph_v2.0/table/en_ppocr_mobile_v2.0_table_structure_infer.tar) / [训练模型](https://paddleocr.bj.bcebos.com/dygraph_v2.1/table/en_ppocr_mobile_v2.0_table_structure_train.tar) |
+|ch_PP-OCRv3_det_slim|【最新】slim量化+蒸馏版超轻量模型，支持中英文、多语种文本检测| 1.1M |[推理模型](https://paddleocr.bj.bcebos.com/PP-OCRv3/chinese/ch_PP-OCRv3_det_slim_infer.tar) / [训练模型](https://paddleocr.bj.bcebos.com/PP-OCRv3/chinese/ch_PP-OCRv3_det_slim_distill_train.tar)|
+|ch_PP-OCRv3_rec_slim |【最新】slim量化版超轻量模型，支持中英文、数字识别| 4.9M |[推理模型](https://paddleocr.bj.bcebos.com/PP-OCRv3/chinese/ch_PP-OCRv3_rec_slim_infer.tar) / [训练模型](https://paddleocr.bj.bcebos.com/PP-OCRv3/chinese/ch_PP-OCRv3_rec_slim_train.tar) |
+|ch_ppstructure_mobile_v2.0_SLANet|基于SLANet在PubTabNet数据集上训练的中文表格识别模型|9.3M|[推理模型](https://paddleocr.bj.bcebos.com/ppstructure/models/slanet/ch_ppstructure_mobile_v2.0_SLANet_infer.tar) / [训练模型](https://paddleocr.bj.bcebos.com/ppstructure/models/slanet/ch_ppstructure_mobile_v2.0_SLANet_train.tar) |
+
 
 <a name="73"></a>
 ### 7.3 DocVQA 模型

ppstructure/docs/installation.md

 - [快速安装](#快速安装)
   - [1. PaddlePaddle 和 PaddleOCR](#1-paddlepaddle-和-paddleocr)
   - [2. 安装其他依赖](#2-安装其他依赖)
-    - [2.1 版面分析所需  Layout-Parser](#21-版面分析所需--layout-parser)
-    - [2.2  VQA所需依赖](#22--vqa所需依赖)
+    - [2.1  VQA所需依赖](#21--vqa所需依赖)
 
 # 快速安装
 
@@ -12,14 +11,7 @@
 
 ## 2. 安装其他依赖
 
-### 2.1 版面分析所需  Layout-Parser
-
-Layout-Parser 可通过如下命令安装
-
-```bash
-pip3 install -U https://paddleocr.bj.bcebos.com/whl/layoutparser-0.0.0-py3-none-any.whl
-```
-### 2.2  VQA所需依赖
+### 2.1  VQA所需依赖
 * paddleocr
 
 ```bash

ppstructure/docs/installation_en.md

+# Quick installation
+
+- [1. PaddlePaddle 和 PaddleOCR](#1)
+- [2. Install other dependencies](#2)
+  - [2.1  VQA](#21)
+
+
+<a name="1"></a>
+## 1. PaddlePaddle and PaddleOCR
+
+Please refer to [PaddleOCR installation documentation](../../doc/doc_en/installation_en.md)
+
+<a name="2"></a>
+## 2. Install other dependencies
+
+<a name="21"></a>
+### 2.1  VQA
+
+* paddleocr
+
+```bash
+pip3 install paddleocr
+```
+
+* PaddleNLP
+```bash
+git clone https://github.com/PaddlePaddle/PaddleNLP -b develop
+cd PaddleNLP
+pip3 install -e .
+```

ppstructure/docs/models_list.md

@@ -34,7 +34,9 @@
 
 |模型名称|模型简介|推理模型大小|下载地址|
 | --- | --- | --- | --- |
-|en_ppocr_mobile_v2.0_table_structure|PubTabNet数据集训练的英文表格场景的表格结构预测|18.6M|[推理模型](https://paddleocr.bj.bcebos.com/dygraph_v2.0/table/en_ppocr_mobile_v2.0_table_structure_infer.tar) / [训练模型](https://paddleocr.bj.bcebos.com/dygraph_v2.1/table/en_ppocr_mobile_v2.0_table_structure_train.tar) |
+|en_ppocr_mobile_v2.0_table_structure|基于TableRec-RARE在PubTabNet数据集上训练的英文表格识别模型|18.6M|[推理模型](https://paddleocr.bj.bcebos.com/dygraph_v2.0/table/en_ppocr_mobile_v2.0_table_structure_infer.tar) / [训练模型](https://paddleocr.bj.bcebos.com/dygraph_v2.1/table/en_ppocr_mobile_v2.0_table_structure_train.tar) |
+|en_ppstructure_mobile_v2.0_SLANet|基于SLANet在PubTabNet数据集上训练的英文表格识别模型|9M|[推理模型](https://paddleocr.bj.bcebos.com/ppstructure/models/slanet/en_ppstructure_mobile_v2.0_SLANet_infer.tar) / [训练模型](https://paddleocr.bj.bcebos.com/ppstructure/models/slanet/en_ppstructure_mobile_v2.0_SLANet_train.tar) |
+|ch_ppstructure_mobile_v2.0_SLANet|基于SLANet在PubTabNet数据集上训练的中文表格识别模型|9.3M|[推理模型](https://paddleocr.bj.bcebos.com/ppstructure/models/slanet/ch_ppstructure_mobile_v2.0_SLANet_infer.tar) / [训练模型](https://paddleocr.bj.bcebos.com/ppstructure/models/slanet/ch_ppstructure_mobile_v2.0_SLANet_train.tar) |
 
 <a name="3"></a>
 

ppstructure/docs/models_list_en.md

@@ -35,7 +35,9 @@ If you need to use other OCR models, you can download the model in [PP-OCR model
 
 |model| description                                                                 |inference model size|download|
 | --- |-----------------------------------------------------------------------------| --- | --- |
-|en_ppocr_mobile_v2.0_table_structure| Table structure model for English table scenes trained on PubTabNet dataset |18.6M|[inference model](https://paddleocr.bj.bcebos.com/dygraph_v2.0/table/en_ppocr_mobile_v2.0_table_structure_infer.tar) / [trained model](https://paddleocr.bj.bcebos.com/dygraph_v2.1/table/en_ppocr_mobile_v2.0_table_structure_train.tar) |
+|en_ppocr_mobile_v2.0_table_structure| English table recognition model trained on PubTabNet dataset based on TableRec-RARE |18.6M|[inference model](https://paddleocr.bj.bcebos.com/dygraph_v2.0/table/en_ppocr_mobile_v2.0_table_structure_infer.tar) / [trained model](https://paddleocr.bj.bcebos.com/dygraph_v2.1/table/en_ppocr_mobile_v2.0_table_structure_train.tar) |
+|en_ppstructure_mobile_v2.0_SLANet|English table recognition model trained on PubTabNet dataset based on SLANet|9M|[inference model](https://paddleocr.bj.bcebos.com/ppstructure/models/slanet/en_ppstructure_mobile_v2.0_SLANet_infer.tar) / [trained model](https://paddleocr.bj.bcebos.com/ppstructure/models/slanet/en_ppstructure_mobile_v2.0_SLANet_train.tar) |
+|ch_ppstructure_mobile_v2.0_SLANet|Chinese table recognition model trained on PubTabNet dataset based on SLANet|9.3M|[inference model](https://paddleocr.bj.bcebos.com/ppstructure/models/slanet/ch_ppstructure_mobile_v2.0_SLANet_infer.tar) / [trained model](https://paddleocr.bj.bcebos.com/ppstructure/models/slanet/ch_ppstructure_mobile_v2.0_SLANet_train.tar) |
 
 <a name="3"></a>
 ## 3. VQA

ppstructure/docs/quickstart.md

 # PP-Structure 快速开始
 
-- [1. 安装依赖包](#1)
-- [2. 便捷使用](#2)
-    - [2.1 命令行使用](#21)
-        - [2.1.1 版面分析+表格识别](#211)
-        - [2.1.2 版面分析](#212)
-        - [2.1.3 表格识别](#213)
-        - [2.1.4 DocVQA](#214)
-    - [2.2 代码使用](#22)
-        - [2.2.1 版面分析+表格识别](#221)
-        - [2.2.2 版面分析](#222)
-        - [2.2.3 表格识别](#223)
-        - [2.2.4 DocVQA](#224)
-    - [2.3 返回结果说明](#23)
-        - [2.3.1 版面分析+表格识别](#231)
-        - [2.3.2 DocVQA](#232)
-    - [2.4 参数说明](#24)
+- [1. 安装依赖包](#1-安装依赖包)
+- [2. 便捷使用](#2-便捷使用)
+  - [2.1 命令行使用](#21-命令行使用)
+    - [2.1.1 图像方向分类+版面分析+表格识别](#211-图像方向分类版面分析表格识别)
+    - [2.1.2 版面分析+表格识别](#212-版面分析表格识别)
+    - [2.1.3 版面分析](#213-版面分析)
+    - [2.1.4 表格识别](#214-表格识别)
+    - [2.1.5 DocVQA](#215-docvqa)
+  - [2.2 代码使用](#22-代码使用)
+    - [2.2.1 图像方向分类版面分析表格识别](#221-图像方向分类版面分析表格识别)
+    - [2.2.2 版面分析+表格识别](#222-版面分析表格识别)
+    - [2.2.3 版面分析](#223-版面分析)
+    - [2.2.4 表格识别](#224-表格识别)
+    - [2.2.5 DocVQA](#225-docvqa)
+  - [2.3 返回结果说明](#23-返回结果说明)
+    - [2.3.1 版面分析+表格识别](#231-版面分析表格识别)
+    - [2.3.2 DocVQA](#232-docvqa)
+  - [2.4 参数说明](#24-参数说明)
 
 
 <a name="1"></a>
@@ -24,8 +26,6 @@
 ```bash
 # 安装 paddleocr，推荐使用2.5+版本
 pip3 install "paddleocr>=2.5"
-# 安装 版面分析依赖包layoutparser（如不需要版面分析功能，可跳过）
-pip3 install -U https://paddleocr.bj.bcebos.com/whl/layoutparser-0.0.0-py3-none-any.whl
 # 安装 DocVQA依赖包paddlenlp（如不需要DocVQA功能，可跳过）
 pip install paddlenlp
 
@@ -38,25 +38,31 @@ pip install paddlenlp
 ### 2.1 命令行使用  
 
 <a name="211"></a>
-#### 2.1.1 版面分析+表格识别
+#### 2.1.1 图像方向分类+版面分析+表格识别
 ```bash
-paddleocr --image_dir=PaddleOCR/ppstructure/docs/table/1.png --type=structure
+paddleocr --image_dir=PaddleOCR/ppstructure/docs/table/1.png --type=structure --image_orientation=true
 ```
 
 <a name="212"></a>
-#### 2.1.2 版面分析
+#### 2.1.2 版面分析+表格识别
 ```bash
-paddleocr --image_dir=PaddleOCR/ppstructure/docs/table/1.png --type=structure --table=false --ocr=false
+paddleocr --image_dir=PaddleOCR/ppstructure/docs/table/1.png --type=structure
 ```
 
 <a name="213"></a>
-#### 2.1.3 表格识别
+#### 2.1.3 版面分析
 ```bash
-paddleocr --image_dir=PaddleOCR/ppstructure/docs/table/table.jpg --type=structure --layout=false
+paddleocr --image_dir=PaddleOCR/ppstructure/docs/table/1.png --type=structure --table=false --ocr=false
 ```
 
 <a name="214"></a>
-#### 2.1.4 DocVQA
+#### 2.1.4 表格识别
+```bash
+paddleocr --image_dir=PaddleOCR/ppstructure/docs/table/table.jpg --type=structure --layout=false
+```
+
+<a name="215"></a>
+#### 2.1.5 DocVQA
 
 请参考：[文档视觉问答](../vqa/README.md)。
 
@@ -64,14 +70,14 @@ paddleocr --image_dir=PaddleOCR/ppstructure/docs/table/table.jpg --type=structur
 ### 2.2 代码使用
 
 <a name="221"></a>
-#### 2.2.1 版面分析+表格识别
+#### 2.2.1 图像方向分类版面分析表格识别
 
 ```python
 import os
 import cv2
 from paddleocr import PPStructure,draw_structure_result,save_structure_res
 
-table_engine = PPStructure(show_log=True)
+table_engine = PPStructure(show_log=True, image_orientation=True)
 
 save_folder = './output'
 img_path = 'PaddleOCR/ppstructure/docs/table/1.png'
@@ -93,7 +99,36 @@ im_show.save('result.jpg')
 ```
 
 <a name="222"></a>
-#### 2.2.2 版面分析
+#### 2.2.2 版面分析+表格识别
+
+```python
+import os
+import cv2
+from paddleocr import PPStructure,draw_structure_result,save_structure_res
+
+table_engine = PPStructure(show_log=True)
+
+save_folder = './output'
+img_path = 'PaddleOCR/ppstructure/docs/table/1.png'
+img = cv2.imread(img_path)
+result = table_engine(img)
+save_structure_res(result, save_folder,os.path.basename(img_path).split('.')[0])
+
+for line in result:
+    line.pop('img')
+    print(line)
+
+from PIL import Image
+
+font_path = 'PaddleOCR/doc/fonts/simfang.ttf' # PaddleOCR下提供字体包
+image = Image.open(img_path).convert('RGB')
+im_show = draw_structure_result(image, result,font_path=font_path)
+im_show = Image.fromarray(im_show)
+im_show.save('result.jpg')
+```
+
+<a name="223"></a>
+#### 2.2.3 版面分析
 
 ```python
 import os
@@ -113,8 +148,8 @@ for line in result:
     print(line)
 ```
 
-<a name="223"></a>
-#### 2.2.3 表格识别
+<a name="224"></a>
+#### 2.2.4 表格识别
 
 ```python
 import os
@@ -134,8 +169,8 @@ for line in result:
     print(line)
 ```
 
-<a name="224"></a>
-#### 2.2.4 DocVQA
+<a name="225"></a>
+#### 2.2.5 DocVQA
 
 请参考：[文档视觉问答](../vqa/README.md)。
 
@@ -156,10 +191,10 @@ PP-Structure的返回结果为一个dict组成的list，示例如下
 ```
 dict 里各个字段说明如下
 
-| 字段            | 说明                                                                                                                                                                                                                                                                                                                                                                                            |
-| --------------- |-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-|type| 图片区域的类型                                                                                                                                                                                                                                                                                                                                                                                       |
-|bbox| 图片区域的在原图的坐标，分别[左上角x，左上角y，右下角x，右下角y]                                                                                                                                                                                                                                                                                                                                                           |
+| 字段   | 说明|
+| --- |---|
+|type| 图片区域的类型 |
+|bbox| 图片区域的在原图的坐标，分别[左上角x，左上角y，右下角x，右下角y]|
 |res| 图片区域的OCR或表格识别结果。<br> 表格: 一个dict，字段说明如下<br>&emsp;&emsp;&emsp;&emsp;&emsp;&emsp;&emsp; `html`: 表格的HTML字符串<br>&emsp;&emsp;&emsp;&emsp;&emsp;&emsp;&emsp; 在代码使用模式下，前向传入return_ocr_result_in_table=True可以拿到表格中每个文本的检测识别结果，对应为如下字段: <br>&emsp;&emsp;&emsp;&emsp;&emsp;&emsp;&emsp; `boxes`: 文本检测坐标<br>&emsp;&emsp;&emsp;&emsp;&emsp;&emsp;&emsp; `rec_res`: 文本识别结果。<br> OCR: 一个包含各个单行文字的检测坐标和识别结果的元组 |
 
 运行完成后，每张图片会在`output`字段指定的目录下有一个同名目录，图片里的每个表格会存储为一个excel，图片区域会被裁剪之后保存下来，excel文件和图片名为表格在图片里的坐标。
@@ -180,20 +215,26 @@ dict 里各个字段说明如下
 <a name="24"></a>
 ### 2.4 参数说明
 
-| 字段                   | 说明                                                                                                                                                 | 默认值                                                     |
-|----------------------|----------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------|
-| output               | excel和识别结果保存的地址                                                                                                                                    | ./output/table                                          |
-| table_max_len        | 表格结构模型预测时，图像的长边resize尺度                                                                                                                            | 488                                                     |
-| table_model_dir      | 表格结构模型 inference 模型地址                                                                                                                              | None                                                    |
-| table_char_dict_path | 表格结构模型所用字典地址                                                                                                                                       | ../ppocr/utils/dict/table_structure_dict.txt            |
-| layout_path_model    | 版面分析模型模型地址，可以为在线地址或者本地地址，当为本地地址时，需要指定 layout_label_map, 命令行模式下可通过--layout_label_map='{0: "Text", 1: "Title", 2: "List", 3:"Table", 4:"Figure"}' 指定 | lp://PubLayNet/ppyolov2_r50vd_dcn_365e_publaynet/config |
-| layout_label_map     | 版面分析模型模型label映射字典                                                                                                                                  | None                                                    |
-| model_name_or_path   | VQA SER模型地址                                                                                                                                        | None                                                    |
-| max_seq_length       | VQA SER模型最大支持token长度                                                                                                                               | 512                                                     |
-| label_map_path       | VQA SER 标签文件地址                                                                                                                                     | ./vqa/labels/labels_ser.txt                             |
-| layout               | 前向中是否执行版面分析                                                                                                                                        | True                                                    |
-| table                | 前向中是否执行表格识别                                                                                                                                        | True                                                    |
-| ocr                  | 对于版面分析中的非表格区域，是否执行ocr。当layout为False时会被自动设置为False                                                                                                  | True                                                    |
-| structure_version                  |      表格结构化模型版本，可选 PP-STRUCTURE。PP-STRUCTURE支持表格结构化模型                                              | PP-STRUCTURE                                                    |
+| 字段 | 说明  | 默认值  |
+|---|---|---|
+| output | 结果保存地址 | ./output/table |
+| table_max_len | 表格结构模型预测时，图像的长边resize尺度 | 488 |
+| table_model_dir | 表格结构模型 inference 模型地址| None |
+| table_char_dict_path | 表格结构模型所用字典地址 | ../ppocr/utils/dict/table_structure_dict.txt  |
+| merge_no_span_structure | 表格识别模型中，是否对'\<td>'和'\</td>' 进行合并 | False |
+| layout_model_dir  | 版面分析模型 inference 模型地址 | None |
+| layout_dict_path  | 版面分析模型字典| ../ppocr/utils/dict/layout_publaynet_dict.txt |
+| layout_score_threshold  | 版面分析模型检测框阈值| 0.5|
+| layout_nms_threshold  | 版面分析模型nms阈值| 0.5|
+| vqa_algorithm  | vqa模型算法| LayoutXLM|
+| ser_model_dir  | ser模型  inference 模型地址| None|
+| ser_dict_path  | ser模型字典| ../train_data/XFUND/class_list_xfun.txt|
+| mode | structure or vqa  | structure   |
+| image_orientation | 前向中是否执行图像方向分类  | False   |
+| layout | 前向中是否执行版面分析  | True   |
+| table  | 前向中是否执行表格识别  | True   |
+| ocr    | 对于版面分析中的非表格区域，是否执行ocr。当layout为False时会被自动设置为False| True |
+| recovery    | 前向中是否执行版面恢复| False |
+| structure_version |  模型版本，可选 PP-structure和PP-structurev2  | PP-structure |
 
 大部分参数和PaddleOCR whl包保持一致，见 [whl包文档](../../doc/doc_ch/whl.md)

ppstructure/docs/quickstart_en.md

 # PP-Structure Quick Start
 
-- [1. Install package](#1)
-- [2. Use](#2)
-    - [2.1 Use by command line](#21)
-        - [2.1.1 layout analysis + table recognition](#211)
-        - [2.1.2 layout analysis](#212)
-        - [2.1.3 table recognition](#213)
-        - [2.1.4 DocVQA](#214)
-    - [2.2 Use by code](#22)
-        - [2.2.1 layout analysis + table recognition](#221)
-        - [2.2.2 layout analysis](#222)
-        - [2.2.3 table recognition](#223)
-        - [2.2.4 DocVQA](#224)
-    - [2.3 Result description](#23)
-        - [2.3.1 layout analysis + table recognition](#231)
-        - [2.3.2 DocVQA](#232)
-    - [2.4 Parameter Description](#24)
+- [1. Install package](#1-install-package)
+- [2. Use](#2-use)
+  - [2.1 Use by command line](#21-use-by-command-line)
+    - [2.1.1 image orientation + layout analysis + table recognition](#211-image-orientation--layout-analysis--table-recognition)
+    - [2.1.2 layout analysis + table recognition](#212-layout-analysis--table-recognition)
+    - [2.1.3 layout analysis](#213-layout-analysis)
+    - [2.1.4 table recognition](#214-table-recognition)
+    - [2.1.5 DocVQA](#215-docvqa)
+  - [2.2 Use by code](#22-use-by-code)
+    - [2.2.1 image orientation + layout analysis + table recognition](#221-image-orientation--layout-analysis--table-recognition)
+    - [2.2.2 layout analysis + table recognition](#222-layout-analysis--table-recognition)
+    - [2.2.3 layout analysis](#223-layout-analysis)
+    - [2.2.4 table recognition](#224-table-recognition)
+    - [2.2.5 DocVQA](#225-docvqa)
+  - [2.3 Result description](#23-result-description)
+    - [2.3.1 layout analysis + table recognition](#231-layout-analysis--table-recognition)
+    - [2.3.2 DocVQA](#232-docvqa)
+  - [2.4 Parameter Description](#24-parameter-description)
 
 
 <a name="1"></a>
@@ -24,8 +26,6 @@
 ```bash
 # Install paddleocr, version 2.5+ is recommended
 pip3 install "paddleocr>=2.5"
-# Install layoutparser (if you do not use the layout analysis, you can skip it)
-pip3 install -U https://paddleocr.bj.bcebos.com/whl/layoutparser-0.0.0-py3-none-any.whl
 # Install the DocVQA dependency package paddlenlp (if you do not use the DocVQA, you can skip it)
 pip install paddlenlp
 
@@ -38,25 +38,31 @@ pip install paddlenlp
 ### 2.1 Use by command line
 
 <a name="211"></a>
-#### 2.1.1 layout analysis + table recognition
+#### 2.1.1 image orientation + layout analysis + table recognition
 ```bash
-paddleocr --image_dir=PaddleOCR/ppstructure/docs/table/1.png --type=structure
+paddleocr --image_dir=PaddleOCR/ppstructure/docs/table/1.png --type=structure --image_orientation=true
 ```
 
 <a name="212"></a>
-#### 2.1.2 layout analysis
+#### 2.1.2 layout analysis + table recognition
 ```bash
-paddleocr --image_dir=PaddleOCR/ppstructure/docs/table/1.png --type=structure --table=false --ocr=false
+paddleocr --image_dir=PaddleOCR/ppstructure/docs/table/1.png --type=structure
 ```
 
 <a name="213"></a>
-#### 2.1.3 table recognition
+#### 2.1.3 layout analysis
 ```bash
-paddleocr --image_dir=PaddleOCR/ppstructure/docs/table/table.jpg --type=structure --layout=false
+paddleocr --image_dir=PaddleOCR/ppstructure/docs/table/1.png --type=structure --table=false --ocr=false
 ```
 
 <a name="214"></a>
-#### 2.1.4 DocVQA
+#### 2.1.4 table recognition
+```bash
+paddleocr --image_dir=PaddleOCR/ppstructure/docs/table/table.jpg --type=structure --layout=false
+```
+
+<a name="215"></a>
+#### 2.1.5 DocVQA
 
 Please refer to: [Documentation Visual Q&A](../vqa/README.md) .
 
@@ -64,14 +70,14 @@ Please refer to: [Documentation Visual Q&A](../vqa/README.md) .
 ### 2.2 Use by code
 
 <a name="221"></a>
-#### 2.2.1 layout analysis + table recognition
+#### 2.2.1 image orientation + layout analysis + table recognition
 
 ```python
 import os
 import cv2
 from paddleocr import PPStructure,draw_structure_result,save_structure_res
 
-table_engine = PPStructure(show_log=True)
+table_engine = PPStructure(show_log=True, image_orientation=True)
 
 save_folder = './output'
 img_path = 'PaddleOCR/ppstructure/docs/table/1.png'
@@ -93,7 +99,36 @@ im_show.save('result.jpg')
 ```
 
 <a name="222"></a>
-#### 2.2.2 layout analysis
+#### 2.2.2 layout analysis + table recognition
+
+```python
+import os
+import cv2
+from paddleocr import PPStructure,draw_structure_result,save_structure_res
+
+table_engine = PPStructure(show_log=True)
+
+save_folder = './output'
+img_path = 'PaddleOCR/ppstructure/docs/table/1.png'
+img = cv2.imread(img_path)
+result = table_engine(img)
+save_structure_res(result, save_folder,os.path.basename(img_path).split('.')[0])
+
+for line in result:
+    line.pop('img')
+    print(line)
+
+from PIL import Image
+
+font_path = 'PaddleOCR/doc/fonts/simfang.ttf' # PaddleOCR下提供字体包
+image = Image.open(img_path).convert('RGB')
+im_show = draw_structure_result(image, result,font_path=font_path)
+im_show = Image.fromarray(im_show)
+im_show.save('result.jpg')
+```
+
+<a name="223"></a>
+#### 2.2.3 layout analysis
 
 ```python
 import os
@@ -113,8 +148,8 @@ for line in result:
     print(line)
 ```
 
-<a name="223"></a>
-#### 2.2.3 table recognition
+<a name="224"></a>
+#### 2.2.4 table recognition
 
 ```python
 import os
@@ -134,8 +169,8 @@ for line in result:
     print(line)
 ```
 
-<a name="224"></a>
-#### 2.2.4 DocVQA
+<a name="225"></a>
+#### 2.2.5 DocVQA
 
 Please refer to: [Documentation Visual Q&A](../vqa/README.md) .
 
@@ -157,8 +192,8 @@ The return of PP-Structure is a list of dicts, the example is as follows:
 ```
 Each field in dict is described as follows:
 
-| field            | description                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                |
-| --------------- |--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| field | description  |
+| --- |---|
 |type| Type of image area.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        |
 |bbox| The coordinates of the image area in the original image, respectively [upper left corner x, upper left corner y, lower right corner x, lower right corner y].                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                              |
 |res| OCR or table recognition result of the image area. <br> table: a dict with field descriptions as follows: <br>&emsp;&emsp;&emsp;&emsp;&emsp;&emsp;&emsp; `html`: html str of table.<br>&emsp;&emsp;&emsp;&emsp;&emsp;&emsp;&emsp; In the code usage mode, set return_ocr_result_in_table=True whrn call can get the detection and recognition results of each text in the table area, corresponding to the following fields: <br>&emsp;&emsp;&emsp;&emsp;&emsp;&emsp;&emsp; `boxes`: text detection boxes.<br>&emsp;&emsp;&emsp;&emsp;&emsp;&emsp;&emsp; `rec_res`: text recognition results.<br> OCR: A tuple containing the detection boxes and recognition results of each single text. |
@@ -180,19 +215,26 @@ Please refer to: [Documentation Visual Q&A](../vqa/README.md) .
 <a name="24"></a>
 ### 2.4 Parameter Description
 
-| field                | description                                                                                                                                                                                                                                                      | default                                                 |
-|----------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------|
-| output               | The save path of result                                                                                                                                                                                                                                          | ./output/table                                          |
-| table_max_len        | When the table structure model predicts, the long side of the image                                                                                                                                                                                              | 488                                                     |
-| table_model_dir      | the path of table structure model                                                                                                                                                                                                                                | None                                                    |
-| table_char_dict_path | the dict path of table structure model                                                                                                                                                                                                                           | ../ppocr/utils/dict/table_structure_dict.txt            |
-| layout_path_model    | The model path of the layout analysis model, which can be an online address or a local path. When it is a local path, layout_label_map needs to be set. In command line mode, use --layout_label_map='{0: "Text", 1: "Title", 2: "List", 3:"Table", 4:"Figure"}' | lp://PubLayNet/ppyolov2_r50vd_dcn_365e_publaynet/config |
-| layout_label_map     | Layout analysis model model label mapping dictionary path                                                                                                                                                                                                        | None                                                    |
-| model_name_or_path   | the model path of VQA SER model                                                                                                                                                                                                                                  | None                                                    |
-| max_seq_length       | the max token length of VQA SER model                                                                                                                                                                                                                            | 512                                                     |
-| label_map_path       | the label path of VQA SER model                                                                                                                                                                                                                                  | ./vqa/labels/labels_ser.txt                             |
-| layout               | Whether to perform layout analysis in forward                                                                                                                                                                                                                    | True                                                    |
-| table                | Whether to perform table recognition in forward                                                                                                                                                                                                                  | True                                                    |
-| ocr                  | Whether to perform ocr for non-table areas in layout analysis. When layout is False, it will be automatically set to False                                                                                                                                                                                                                 | True                                                    |
-| structure_version                     | table structure Model version number, the current model support list is as follows: PP-STRUCTURE support english table structure model | PP-STRUCTURE                 |
+| field | description | default |
+|---|---|---|
+| output | result save path | ./output/table |
+| table_max_len | long side of the image resize in table structure model | 488 |
+| table_model_dir | Table structure model inference model path| None |
+| table_char_dict_path | The dictionary path of table structure model | ../ppocr/utils/dict/table_structure_dict.txt  |
+| merge_no_span_structure | In the table recognition model, whether to merge '\<td>' and '\</td>' | False |
+| layout_model_dir  | Layout analysis model inference model path| None |
+| layout_dict_path  | The dictionary path of layout analysis model| ../ppocr/utils/dict/layout_publaynet_dict.txt |
+| layout_score_threshold  | The box threshold path of layout analysis model| 0.5|
+| layout_nms_threshold  | The nms threshold path of layout analysis model| 0.5|
+| vqa_algorithm  | vqa model algorithm| LayoutXLM|
+| ser_model_dir  | Ser model inference model path| None|
+| ser_dict_path  | The dictionary path of Ser model| ../train_data/XFUND/class_list_xfun.txt|
+| mode | structure or vqa  | structure   |
+| image_orientation | Whether to perform image orientation classification in forward  | False   |
+| layout | Whether to perform layout analysis in forward  | True   |
+| table  | Whether to perform table recognition in forward  | True   |
+| ocr    | Whether to perform ocr for non-table areas in layout analysis. When layout is False, it will be automatically set to False| True |
+| recovery    | Whether to perform layout recovery in forward| False |
+| structure_version |  Structure version, optional PP-structure and PP-structurev2  | PP-structure |
+
 Most of the parameters are consistent with the PaddleOCR whl package, see [whl package documentation](../../doc/doc_en/whl.md)

ppstructure/predict_system.py

@@ -18,7 +18,7 @@ import subprocess
 
 __dir__ = os.path.dirname(os.path.abspath(__file__))
 sys.path.append(__dir__)
-sys.path.append(os.path.abspath(os.path.join(__dir__, '..')))
+sys.path.insert(0, os.path.abspath(os.path.join(__dir__, '../')))
 
 os.environ["FLAGS_allocator_strategy"] = 'auto_growth'
 import cv2
@@ -27,11 +27,11 @@ import numpy as np
 import time
 import logging
 from copy import deepcopy
-from attrdict import AttrDict
 
 from ppocr.utils.utility import get_image_file_list, check_and_read_gif
 from ppocr.utils.logging import get_logger
 from tools.infer.predict_system import TextSystem
+from ppstructure.layout.predict_layout import LayoutPredictor
 from ppstructure.table.predict_table import TableSystem, to_excel
 from ppstructure.utility import parse_args, draw_structure_result
 from ppstructure.recovery.recovery_to_doc import convert_info_docx
@@ -42,6 +42,14 @@ logger = get_logger()
 class StructureSystem(object):
     def __init__(self, args):
         self.mode = args.mode
+        self.recovery = args.recovery
+
+        self.image_orientation_predictor = None
+        if args.image_orientation:
+            import paddleclas
+            self.image_orientation_predictor = paddleclas.PaddleClas(
+                model_name="text_image_orientation")
+
         if self.mode == 'structure':
             if not args.show_log:
                 logger.setLevel(logging.INFO)
@@ -51,28 +59,14 @@ class StructureSystem(object):
                     "When args.layout is false, args.ocr is automatically set to false"
                 )
             args.drop_score = 0
-            # init layout and ocr model
+            # init model
+            self.layout_predictor = None
             self.text_system = None
+            self.table_system = None
             if args.layout:
-                import layoutparser as lp
-                config_path = None
-                model_path = None
-                if os.path.isdir(args.layout_path_model):
-                    model_path = args.layout_path_model
-                else:
-                    config_path = args.layout_path_model
-                self.table_layout = lp.PaddleDetectionLayoutModel(
-                    config_path=config_path,
-                    model_path=model_path,
-                    label_map=args.layout_label_map,
-                    threshold=0.5,
-                    enable_mkldnn=args.enable_mkldnn,
-                    enforce_cpu=not args.use_gpu,
-                    thread_num=args.cpu_threads)
+                self.layout_predictor = LayoutPredictor(args)
                 if args.ocr:
                     self.text_system = TextSystem(args)
-            else:
-                self.table_layout = None
             if args.table:
                 if self.text_system is not None:
                     self.table_system = TableSystem(
@@ -80,39 +74,78 @@ class StructureSystem(object):
                         self.text_system.text_recognizer)
                 else:
                     self.table_system = TableSystem(args)
-            else:
-                self.table_system = None
 
         elif self.mode == 'vqa':
             raise NotImplementedError
 
     def __call__(self, img, return_ocr_result_in_table=False):
+        time_dict = {
+            'image_orientation': 0,
+            'layout': 0,
+            'table': 0,
+            'table_match': 0,
+            'det': 0,
+            'rec': 0,
+            'vqa': 0,
+            'all': 0
+        }
+        start = time.time()
+        if self.image_orientation_predictor is not None:
+            tic = time.time()
+            cls_result = self.image_orientation_predictor.predict(
+                input_data=img)
+            cls_res = next(cls_result)
+            angle = cls_res[0]['label_names'][0]
+            cv_rotate_code = {
+                '90': cv2.ROTATE_90_COUNTERCLOCKWISE,
+                '180': cv2.ROTATE_180,
+                '270': cv2.ROTATE_90_CLOCKWISE
+            }
+            img = cv2.rotate(img, cv_rotate_code[angle])
+            toc = time.time()
+            time_dict['image_orientation'] = toc - tic
         if self.mode == 'structure':
             ori_im = img.copy()
-            if self.table_layout is not None:
-                layout_res = self.table_layout.detect(img[..., ::-1])
+            if self.layout_predictor is not None:
+                layout_res, elapse = self.layout_predictor(img)
+                time_dict['layout'] += elapse
             else:
                 h, w = ori_im.shape[:2]
-                layout_res = [AttrDict(coordinates=[0, 0, w, h], type='Table')]
+                layout_res = [dict(bbox=None, label='table')]
             res_list = []
             for region in layout_res:
                 res = ''
-                x1, y1, x2, y2 = region.coordinates
-                x1, y1, x2, y2 = int(x1), int(y1), int(x2), int(y2)
-                roi_img = ori_im[y1:y2, x1:x2, :]
-                if region.type == 'Table':
+                if region['bbox'] is not None:
+                    x1, y1, x2, y2 = region['bbox']
+                    x1, y1, x2, y2 = int(x1), int(y1), int(x2), int(y2)
+                    roi_img = ori_im[y1:y2, x1:x2, :]
+                else:
+                    x1, y1, x2, y2 = 0, 0, w, h
+                    roi_img = ori_im
+                if region['label'] == 'table':
                     if self.table_system is not None:
-                        res = self.table_system(roi_img,
-                                                return_ocr_result_in_table)
+                        res, table_time_dict = self.table_system(
+                            roi_img, return_ocr_result_in_table)
+                        time_dict['table'] += table_time_dict['table']
+                        time_dict['table_match'] += table_time_dict['match']
+                        time_dict['det'] += table_time_dict['det']
+                        time_dict['rec'] += table_time_dict['rec']
                 else:
                     if self.text_system is not None:
-                        if args.recovery:
+                        if self.recovery:
                             wht_im = np.ones(ori_im.shape, dtype=ori_im.dtype)
                             wht_im[y1:y2, x1:x2, :] = roi_img
-                            filter_boxes, filter_rec_res = self.text_system(wht_im)
+                            filter_boxes, filter_rec_res, ocr_time_dict = self.text_system(
+                                wht_im)
                         else:
-                            filter_boxes, filter_rec_res = self.text_system(roi_img)
-                        # remove style char
+                            filter_boxes, filter_rec_res, ocr_time_dict = self.text_system(
+                                roi_img)
+                        time_dict['det'] += ocr_time_dict['det']
+                        time_dict['rec'] += ocr_time_dict['rec']
+
+                        # remove style char, 
+                        # when using the recognition model trained on the PubtabNet dataset, 
+                        # it will recognize the text format in the table, such as <b>
                         style_token = [
                             '<strike>', '<strike>', '<sup>', '</sub>', '<b>',
                             '</b>', '<sub>', '</sup>', '<overline>',
@@ -125,7 +158,7 @@ class StructureSystem(object):
                             for token in style_token:
                                 if token in rec_str:
                                     rec_str = rec_str.replace(token, '')
-                            if not args.recovery:
+                            if not self.recovery:
                                 box += [x1, y1]
                             res.append({
                                 'text': rec_str,
@@ -133,15 +166,17 @@ class StructureSystem(object):
                                 'text_region': box.tolist()
                             })
                 res_list.append({
-                    'type': region.type,
+                    'type': region['label'].lower(),
                     'bbox': [x1, y1, x2, y2],
                     'img': roi_img,
                     'res': res
                 })
-            return res_list
+            end = time.time()
+            time_dict['all'] = end - start
+            return res_list, time_dict
         elif self.mode == 'vqa':
             raise NotImplementedError
-        return None
+        return None, None
 
 
 def save_structure_res(res, save_folder, img_name):
@@ -156,12 +191,12 @@ def save_structure_res(res, save_folder, img_name):
             roi_img = region.pop('img')
             f.write('{}\n'.format(json.dumps(region)))
 
-            if region['type'] == 'Table' and len(region[
+            if region['type'] == 'table' and len(region[
                     'res']) > 0 and 'html' in region['res']:
                 excel_path = os.path.join(excel_save_folder,
                                           '{}.xlsx'.format(region['bbox']))
                 to_excel(region['res']['html'], excel_path)
-            elif region['type'] == 'Figure':
+            elif region['type'] == 'figure':
                 img_path = os.path.join(excel_save_folder,
                                         '{}.jpg'.format(region['bbox']))
                 cv2.imwrite(img_path, roi_img)
@@ -187,8 +222,7 @@ def main(args):
         if img is None:
             logger.error("error in loading image:{}".format(image_file))
             continue
-        starttime = time.time()
-        res = structure_sys(img)
+        res, time_dict = structure_sys(img)
 
         if structure_sys.mode == 'structure':
             save_structure_res(res, save_folder, img_name)
@@ -201,9 +235,8 @@ def main(args):
         cv2.imwrite(img_save_path, draw_img)
         logger.info('result save to {}'.format(img_save_path))
         if args.recovery:
-            convert_info_docx(img, res, save_folder, img_name) 
-        elapse = time.time() - starttime
-        logger.info("Predict time : {:.3f}s".format(elapse))
+            convert_info_docx(img, res, save_folder, img_name)
+        logger.info("Predict time : {:.3f}s".format(time_dict['all']))
 
 
 if __name__ == "__main__":

ppstructure/table/README.md

-- [Table Recognition](#table-recognition)
-  - [1. pipeline](#1-pipeline)
-  - [2. Performance](#2-performance)
-  - [3. How to use](#3-how-to-use)
-    - [3.1 quick start](#31-quick-start)
-    - [3.2 Train](#32-train)
-    - [3.3 Eval](#33-eval)
-    - [3.4 Inference](#34-inference)
-
+English | [简体中文](README_ch.md)
 
 # Table Recognition
 
+- [1. pipeline](#1-pipeline)
+- [2. Performance](#2-performance)
+- [3. Result](#3-result)
+- [4. How to use](#4-how-to-use)
+  - [4.1 Quick start](#41-quick-start)
+  - [4.2 Train](#42-train)
+  - [4.3 Calculate TEDS](#43-calculate-teds)
+- [5. Reference](#5-reference)
+
+
 ## 1. pipeline
 The table recognition mainly contains three models
 1. Single line text detection-DB
 2. Single line text recognition-CRNN
-3. Table structure and cell coordinate prediction-RARE
+3. Table structure and cell coordinate prediction-SLANet
 
 The table recognition flow chart is as follows
 
 ![tableocr_pipeline](../docs/table/tableocr_pipeline_en.jpg)
 
 1. The coordinates of single-line text is detected by DB model, and then sends it to the recognition model to get the recognition result.
-2. The table structure and cell coordinates is predicted by RARE model.
+2. The table structure and cell coordinates is predicted by SLANet model.
 3. The recognition result of the cell is combined by the coordinates, recognition result of the single line and the coordinates of the cell.
 4. The cell recognition result and the table structure together construct the html string of the table.
 
 ## 2. Performance
 We evaluated the algorithm on the PubTabNet<sup>[1]</sup> eval dataset, and the performance is as follows:
 
+|Method|Acc|[TEDS(Tree-Edit-Distance-based Similarity)](https://github.com/ibm-aur-nlp/PubTabNet/tree/master/src)|Speed|
+| --- | --- | --- | ---|
+| EDD<sup>[2]</sup> |x| 88.3 |x|
+| TableRec-RARE(ours) |73.8%| 93.32 |1550ms|
+| SLANet(ours) | 76.2%| 94.98 |766ms|
+
+The performance indicators are explained as follows:
+- Acc: The accuracy of the table structure in each image, a wrong token is considered an error.
+- TEDS: The accuracy of the model's restoration of table information. This indicator evaluates not only the table structure, but also the text content in the table.
+- Speed: The inference speed of a single image when the model runs on the CPU machine and MKL is enabled.
+
+## 3. Result
+
+![](../docs/imgs/table_ch_result1.jpg)
+![](../docs/imgs/table_ch_result2.jpg)
+![](../docs/imgs/table_ch_result3.jpg)
 
-|Method|[TEDS(Tree-Edit-Distance-based Similarity)](https://github.com/ibm-aur-nlp/PubTabNet/tree/master/src)|
-| --- | --- |
-| EDD<sup>[2]</sup> | 88.3 |
-| Ours | 93.32 |
+## 4. How to use
 
-## 3. How to use
+### 4.1 Quick start
 
-### 3.1 quick start
+Use the following commands to quickly complete the identification of a table.
 
 ```python
 cd PaddleOCR/ppstructure
 
 # download model
 mkdir inference && cd inference
-# Download the detection model of the ultra-lightweight table English OCR model and unzip it
-wget https://paddleocr.bj.bcebos.com/dygraph_v2.0/table/en_ppocr_mobile_v2.0_table_det_infer.tar && tar xf en_ppocr_mobile_v2.0_table_det_infer.tar
-# Download the recognition model of the ultra-lightweight table English OCR model and unzip it
-wget https://paddleocr.bj.bcebos.com/dygraph_v2.0/table/en_ppocr_mobile_v2.0_table_rec_infer.tar && tar xf en_ppocr_mobile_v2.0_table_rec_infer.tar
-# Download the ultra-lightweight English table inch model and unzip it
-wget https://paddleocr.bj.bcebos.com/dygraph_v2.0/table/en_ppocr_mobile_v2.0_table_structure_infer.tar && tar xf en_ppocr_mobile_v2.0_table_structure_infer.tar
+# Download the PP-OCRv3 text detection model and unzip it
+wget https://paddleocr.bj.bcebos.com/PP-OCRv3/chinese/ch_PP-OCRv3_det_slim_infer.tar && tar xf ch_PP-OCRv3_det_slim_infer.tar
+# Download the PP-OCRv3 text recognition model and unzip it
+wget https://paddleocr.bj.bcebos.com/PP-OCRv3/chinese/ch_PP-OCRv3_rec_slim_infer.tar && tar xf ch_PP-OCRv3_rec_slim_infer.tar
+# Download the PP-Structurev2 form recognition model and unzip it
+wget https://paddleocr.bj.bcebos.com/ppstructure/models/slanet/ch_ppstructure_mobile_v2.0_SLANet_infer.tar && tar xf ch_ppstructure_mobile_v2.0_SLANet_infer.tar
 cd ..
 # run
-python3 table/predict_table.py --det_model_dir=inference/en_ppocr_mobile_v2.0_table_det_infer --rec_model_dir=inference/en_ppocr_mobile_v2.0_table_rec_infer --table_model_dir=inference/en_ppocr_mobile_v2.0_table_structure_infer --image_dir=./docs/table/table.jpg --rec_char_dict_path=../ppocr/utils/dict/table_dict.txt --table_char_dict_path=../ppocr/utils/dict/table_structure_dict.txt --det_limit_side_len=736 --det_limit_type=min --output ./output/table
-```
-Note: The above model is trained on the PubLayNet dataset and only supports English scanning scenarios. If you need to identify other scenarios, you need to train the model yourself and replace the three fields `det_model_dir`, `rec_model_dir`, `table_model_dir`.
-
-After running, the excel sheet of each picture will be saved in the directory specified by the output field
-
-### 3.2 Train
-
-In this chapter, we only introduce the training of the table structure model, For model training of [text detection](../../doc/doc_en/detection_en.md) and [text recognition](../../doc/doc_en/recognition_en.md), please refer to the corresponding documents
-
-* data preparation  
-The training data uses public data set [PubTabNet](https://arxiv.org/abs/1911.10683 ), Can be downloaded from the official [website](https://github.com/ibm-aur-nlp/PubTabNet) 。The PubTabNet data set contains about 500,000 images, as well as annotations in html format。
+python3.7 table/predict_table.py \
+    --det_model_dir=inference/ch_PP-OCRv3_det_slim_infer \
+    --rec_model_dir=inference/ch_PP-OCRv3_rec_slim_infer  \
+    --table_model_dir=inference/ch_ppstructure_mobile_v2.0_SLANet_infer \
+    --rec_char_dict_path=../ppocr/utils/ppocr_keys_v1.txt \
+    --table_char_dict_path=../ppocr/utils/dict/table_structure_dict_ch.txt \
+    --image_dir=docs/table/table.jpg \
+    --output=../output/table
 
-* Start training  
-*If you are installing the cpu version of paddle, please modify the `use_gpu` field in the configuration file to false*
-```shell
-# single GPU training
-python3 tools/train.py -c configs/table/table_mv3.yml
-# multi-GPU training
-# Set the GPU ID used by the '--gpus' parameter.
-python3 -m paddle.distributed.launch --gpus '0,1,2,3' tools/train.py -c configs/table/table_mv3.yml
 ```
 
-In the above instruction, use `-c` to select the training to use the `configs/table/table_mv3.yml` configuration file.
-For a detailed explanation of the configuration file, please refer to [config](../../doc/doc_en/config_en.md).
+After the operation is completed, the excel table of each image will be saved to the directory specified by the output field, and an html file will be produced in the directory to visually view the cell coordinates and the recognized table.
 
-* load trained model and continue training
+### 4.2 Train
 
-If you expect to load trained model and continue the training again, you can specify the parameter `Global.checkpoints` as the model path to be loaded.
+The training, evaluation and inference process of the text detection model can be referred to [detection](../../doc/doc_en/detection_en.md)
 
-```shell
-python3 tools/train.py -c configs/table/table_mv3.yml -o Global.checkpoints=./your/trained/model
-```
+The training, evaluation and inference process of the text recognition model can be referred to [recognition](../../doc/doc_en/recognition_en.md)
 
-**Note**: The priority of `Global.checkpoints` is higher than that of `Global.pretrain_weights`, that is, when two parameters are specified at the same time, the model specified by `Global.checkpoints` will be loaded first. If the model path specified by `Global.checkpoints` is wrong, the one specified by `Global.pretrain_weights` will be loaded.
+The training, evaluation and inference process of the table recognition model can be referred to [table_recognition](../../doc/doc_en/table_recognition_en.md)
 
-### 3.3 Eval
+### 4.3 Calculate TEDS
 
 The table uses [TEDS(Tree-Edit-Distance-based Similarity)](https://github.com/ibm-aur-nlp/PubTabNet/tree/master/src) as the evaluation metric of the model. Before the model evaluation, the three models in the pipeline need to be exported as inference models (we have provided them), and the gt for evaluation needs to be prepared. Examples of gt are as follows:
-```json
-{"PMC4289340_004_00.png": [
-  ["<html>", "<body>", "<table>", "<thead>", "<tr>", "<td>", "</td>", "<td>", "</td>", "<td>", "</td>", "</tr>", "</thead>", "<tbody>", "<tr>", "<td>", "</td>", "<td>", "</td>", "<td>", "</td>", "</tr>",  "</tbody>", "</table>", "</body>", "</html>"],
-  [[1, 4, 29, 13], [137, 4, 161, 13], [215, 4, 236, 13], [1, 17, 30, 27], [137, 17, 147, 27], [215, 17, 225, 27]],
-  [["<b>", "F", "e", "a", "t", "u", "r", "e", "</b>"], ["<b>", "G", "b", "3", " ", "+", "</b>"], ["<b>", "G", "b", "3", " ", "-", "</b>"], ["<b>", "P", "a", "t", "i", "e", "n", "t", "s", "</b>"], ["6", "2"], ["4", "5"]]
-]}
+```txt
+PMC5755158_010_01.png    <html><body><table><thead><tr><td></td><td><b>Weaning</b></td><td><b>Week 15</b></td><td><b>Off-test</b></td></tr></thead><tbody><tr><td>Weaning</td><td>–</td><td>–</td><td>–</td></tr><tr><td>Week 15</td><td>–</td><td>0.17 ± 0.08</td><td>0.16 ± 0.03</td></tr><tr><td>Off-test</td><td>–</td><td>0.80 ± 0.24</td><td>0.19 ± 0.09</td></tr></tbody></table></body></html>
+```
+Each line in gt consists of the file name and the html string of the table. The file name and the html string of the table are separated by `\t`.
+
+You can also use the following command to generate an evaluation gt file from the annotation file:
+```python
+python3 ppstructure/table/convert_label2html.py --ori_gt_path /path/to/your_label_file --save_path /path/to/save_file
 ```
-In gt json, the key is the image name, the value is the corresponding gt, and gt is a list composed of four items, and each item is
-1. HTML string list of table structure
-2. The coordinates of each cell (not including the empty text in the cell)
-3. The text information in each cell (not including the empty text in the cell)
 
 Use the following command to evaluate. After the evaluation is completed, the teds indicator will be output.
 ```python
-cd PaddleOCR/ppstructure
-python3 table/eval_table.py --det_model_dir=path/to/det_model_dir --rec_model_dir=path/to/rec_model_dir --table_model_dir=path/to/table_model_dir --image_dir=../doc/table/1.png --rec_char_dict_path=../ppocr/utils/dict/table_dict.txt --table_char_dict_path=../ppocr/utils/dict/table_structure_dict.txt --det_limit_side_len=736 --det_limit_type=min --gt_path=path/to/gt.json
+python3 table/eval_table.py \
+    --det_model_dir=path/to/det_model_dir \
+    --rec_model_dir=path/to/rec_model_dir \
+    --table_model_dir=path/to/table_model_dir \
+    --image_dir=../doc/table/1.png \
+    --rec_char_dict_path=../ppocr/utils/dict/table_dict.txt \
+    --table_char_dict_path=../ppocr/utils/dict/table_structure_dict.txt \
+    --det_limit_side_len=736 \
+    --det_limit_type=min \
+    --gt_path=path/to/gt.txt
 ```
 
 If the PubLatNet eval dataset is used, it will be output
 ```bash
-teds: 93.32
-```
-
-### 3.4 Inference
-
-```python
-cd PaddleOCR/ppstructure
-python3 table/predict_table.py --det_model_dir=path/to/det_model_dir --rec_model_dir=path/to/rec_model_dir --table_model_dir=path/to/table_model_dir --image_dir=../doc/table/1.png --rec_char_dict_path=../ppocr/utils/dict/table_dict.txt --table_char_dict_path=../ppocr/utils/dict/table_structure_dict.txt --det_limit_side_len=736 --det_limit_type=min --output ../output/table
+teds: 94.98
 ```
-After running, the excel sheet of each picture will be saved in the directory specified by the output field
 
-Reference
+## 5. Reference
 1. https://github.com/ibm-aur-nlp/PubTabNet
 2. https://arxiv.org/pdf/1911.10683

ppstructure/table/predict_structure.py

@@ -16,7 +16,7 @@ import sys
 
 __dir__ = os.path.dirname(os.path.abspath(__file__))
 sys.path.append(__dir__)
-sys.path.append(os.path.abspath(os.path.join(__dir__, '../..')))
+sys.path.insert(0, os.path.abspath(os.path.join(__dir__, '../..')))
 
 os.environ["FLAGS_allocator_strategy"] = 'auto_growth'
 
@@ -73,12 +73,14 @@ class TableStructurer(object):
             postprocess_params = {
                 'name': 'TableLabelDecode',
                 "character_dict_path": args.table_char_dict_path,
+                'merge_no_span_structure': args.merge_no_span_structure
             }
         else:
             postprocess_params = {
                 'name': 'TableMasterLabelDecode',
                 "character_dict_path": args.table_char_dict_path,
-                'box_shape': 'pad'
+                'box_shape': 'pad',
+                'merge_no_span_structure': args.merge_no_span_structure
             }
 
         self.preprocess_op = create_operators(pre_process_list)
@@ -87,6 +89,7 @@ class TableStructurer(object):
             utility.create_predictor(args, 'table', logger)
 
     def __call__(self, img):
+        starttime = time.time()
         ori_im = img.copy()
         data = {'image': img}
         data = transform(data, self.preprocess_op)
@@ -95,7 +98,6 @@ class TableStructurer(object):
             return None, 0
         img = np.expand_dims(img, axis=0)
         img = img.copy()
-        starttime = time.time()
 
         self.input_tensor.copy_from_cpu(img)
         self.predictor.run()
@@ -126,7 +128,6 @@ def main(args):
     table_structurer = TableStructurer(args)
     count = 0
     total_time = 0
-    use_xywh = args.table_algorithm in ['TableMaster']
     os.makedirs(args.output, exist_ok=True)
     with open(
             os.path.join(args.output, 'infer.txt'), mode='w',
@@ -146,7 +147,10 @@ def main(args):
             f_w.write("result: {}, {}\n".format(structure_str_list,
                                                 bbox_list_str))
 
-            img = draw_rectangle(image_file, bbox_list, use_xywh)
+            if len(bbox_list) > 0 and len(bbox_list[0]) == 4:
+                img = draw_rectangle(image_file, pred_res['cell_bbox'])
+            else:
+                img = utility.draw_boxes(img, bbox_list)
             img_save_path = os.path.join(args.output,
                                          os.path.basename(image_file))
             cv2.imwrite(img_save_path, img)

ppstructure/vqa/predict_vqa_token_ser.py

@@ -41,7 +41,11 @@ logger = get_logger()
 class SerPredictor(object):
     def __init__(self, args):
         self.ocr_engine = PaddleOCR(
-            use_angle_cls=False, show_log=False, use_gpu=args.use_gpu)
+            use_angle_cls=args.use_angle_cls,
+            det_model_dir=args.det_model_dir,
+            rec_model_dir=args.rec_model_dir,
+            show_log=False,
+            use_gpu=args.use_gpu)
 
         pre_process_list = [{
             'VQATokenLabelEncode': {

test_tipc/configs/en_table_structure/table_mv3.yml

@@ -19,8 +19,6 @@ Global:
   character_type: en
   max_text_length: 800
   infer_mode: False
-  process_total_num: 0
-  process_cut_num: 0
 
 Optimizer:
   name: Adam