Merge branch 'PaddlePaddle:dygraph' into dygraph

deploy/cpp_infer/docs/windows_vs2019_build.md

@@ -93,3 +93,5 @@ cd D:\projects\PaddleOCR\deploy\cpp_infer\out\build\x64-Release
 
 ### 注意
 * 在Windows下的终端中执行文件exe时，可能会发生乱码的现象，此时需要在终端中输入`CHCP 65001`，将终端的编码方式由GBK编码(默认)改为UTF-8编码，更加具体的解释可以参考这篇博客：[https://blog.csdn.net/qq_35038153/article/details/78430359](https://blog.csdn.net/qq_35038153/article/details/78430359)。
+
+* 编译时，如果报错`错误：C1083 无法打开包括文件:"dirent.h":No such file or directory`，可以参考该[文档](https://blog.csdn.net/Dora_blank/article/details/117740837#41_C1083_direnthNo_such_file_or_directory_54)，新建`dirent.h`文件，并添加到`VC++`的包含目录中。

deploy/cpp_infer/readme.md

@@ -18,6 +18,7 @@ PaddleOCR模型部署。
 * 首先需要从opencv官网上下载在Linux环境下源码编译的包，以opencv3.4.7为例，下载命令如下。
 
 ```
+cd deploy/cpp_infer
 wget https://github.com/opencv/opencv/archive/3.4.7.tar.gz
 tar -xf 3.4.7.tar.gz
 ```

deploy/cpp_infer/readme_en.md

@@ -18,6 +18,7 @@ PaddleOCR model deployment.
 * First of all, you need to download the source code compiled package in the Linux environment from the opencv official website. Taking opencv3.4.7 as an example, the download command is as follows.
 
 ```
+cd deploy/cpp_infer
 wget https://github.com/opencv/opencv/archive/3.4.7.tar.gz
 tar -xf 3.4.7.tar.gz
 ```

deploy/hubserving/readme.md

@@ -29,6 +29,7 @@ deploy/hubserving/ocr_system/
 ### 1. 准备环境
 ```shell
 # 安装paddlehub  
+# paddlehub 需要 python>3.6.2
 pip3 install paddlehub==2.1.0 --upgrade -i https://pypi.tuna.tsinghua.edu.cn/simple
 ```
 

deploy/hubserving/readme_en.md

@@ -30,6 +30,7 @@ The following steps take the 2-stage series service as an example. If only the d
 ### 1. Prepare the environment
 ```shell
 # Install paddlehub  
+# python>3.6.2 is required bt paddlehub
 pip3 install paddlehub==2.1.0 --upgrade -i https://pypi.tuna.tsinghua.edu.cn/simple
 ```
 

doc/doc_ch/detection.md

@@ -18,9 +18,9 @@ PaddleOCR 也提供了数据格式转换脚本，可以将官网 label 转换支
 
 ```
 # 将官网下载的标签文件转换为 train_icdar2015_label.txt
-python gen_label.py --mode="det" --root_path="icdar_c4_train_imgs/"  \
-                    --input_path="ch4_training_localization_transcription_gt" \
-                    --output_label="train_icdar2015_label.txt"
+python gen_label.py --mode="det" --root_path="/path/to/icdar_c4_train_imgs/"  \
+                    --input_path="/path/to/ch4_training_localization_transcription_gt" \
+                    --output_label="/path/to/train_icdar2015_label.txt"
 ```
 
 解压数据集和下载标注文件后，PaddleOCR/train_data/ 有两个文件夹和两个文件，分别是：

doc/doc_ch/inference.md

@@ -221,7 +221,7 @@ python3 tools/export_model.py -c configs/det/det_r50_vd_sast_totaltext.yml -o Gl
 
 ```
 
-**SAST文本检测模型推理，需要设置参数`--det_algorithm="SAST"`，同时，还需要增加参数`--det_sast_polygon=True`，**可以执行如下命令：
+SAST文本检测模型推理，需要设置参数`--det_algorithm="SAST"`，同时，还需要增加参数`--det_sast_polygon=True`，可以执行如下命令：
 ```
 python3 tools/infer/predict_det.py --det_algorithm="SAST" --image_dir="./doc/imgs_en/img623.jpg" --det_model_dir="./inference/det_sast_tt/" --det_sast_polygon=True
 ```

doc/doc_ch/recognition.md

@@ -330,6 +330,8 @@ PaddleOCR目前已支持80种（除中文外）语种识别，`configs/rec/multi
 
     ```
 
+意大利文由拉丁字母组成，因此执行完命令后会得到名为 rec_latin_lite_train.yml 的配置文件。
+
 2. 手动修改配置文件
 
    您也可以手动修改模版中的以下几个字段:

doc/doc_en/inference_en.md

@@ -230,7 +230,7 @@ First, convert the model saved in the SAST text detection training process into
 python3 tools/export_model.py -c configs/det/det_r50_vd_sast_totaltext.yml -o Global.pretrained_model=./det_r50_vd_sast_totaltext_v2.0_train/best_accuracy  Global.save_inference_dir=./inference/det_sast_tt
 ```
 
-**For SAST curved text detection model inference, you need to set the parameter `--det_algorithm="SAST"` and `--det_sast_polygon=True`**, run the following command:
+For SAST curved text detection model inference, you need to set the parameter `--det_algorithm="SAST"` and `--det_sast_polygon=True`, run the following command:
 
 ```
 python3 tools/infer/predict_det.py --det_algorithm="SAST" --image_dir="./doc/imgs_en/img623.jpg" --det_model_dir="./inference/det_sast_tt/" --det_sast_polygon=True

doc/doc_en/recognition_en.md

@@ -329,6 +329,7 @@ There are two ways to create the required configuration file:：
     ...
 
     ```
+Italian is made up of Latin letters, so after executing the command, you will get the rec_latin_lite_train.yml.
 
 2. Manually modify the configuration file
 

ppocr/data/simple_dataset.py

@@ -14,7 +14,6 @@
 import numpy as np
 import os
 import random
-import traceback
 from paddle.io import Dataset
 
 from .imaug import transform, create_operators
@@ -46,7 +45,6 @@ class SimpleDataSet(Dataset):
         self.seed = seed
         logger.info("Initialize indexs of datasets:%s" % label_file_list)
         self.data_lines = self.get_image_info_list(label_file_list, ratio_list)
-        self.check_data()
         self.data_idx_order_list = list(range(len(self.data_lines)))
         if self.mode == "train" and self.do_shuffle:
             self.shuffle_data_random()
@@ -103,18 +101,25 @@ class SimpleDataSet(Dataset):
 
     def __getitem__(self, idx):
         file_idx = self.data_idx_order_list[idx]
-        data = self.data_lines[file_idx]
+        data_line = self.data_lines[file_idx]
         try:
+            data_line = data_line.decode('utf-8')
+            substr = data_line.strip("\n").split(self.delimiter)
+            file_name = substr[0]
+            label = substr[1]
+            img_path = os.path.join(self.data_dir, file_name)
+            data = {'img_path': img_path, 'label': label}
+            if not os.path.exists(img_path):
+                raise Exception("{} does not exist!".format(img_path))
             with open(data['img_path'], 'rb') as f:
                 img = f.read()
                 data['image'] = img
             data['ext_data'] = self.get_ext_data()
             outs = transform(data, self.ops)
-        except:
-            error_meg = traceback.format_exc()
+        except Exception as e:
             self.logger.error(
-                "When parsing file {} and label {}, error happened with msg: {}".format(
-                    data['img_path'],data['label'], error_meg))
+                "When parsing line {}, error happened with msg: {}".format(
+                    data_line, e))
             outs = None
         if outs is None:
             # during evaluation, we should fix the idx to get same results for many times of evaluation.
@@ -125,17 +130,3 @@ class SimpleDataSet(Dataset):
 
     def __len__(self):
         return len(self.data_idx_order_list)
-
-    def check_data(self):
-        new_data_lines = []
-        for data_line in self.data_lines:
-            data_line = data_line.decode('utf-8')
-            substr = data_line.strip("\n").strip("\r").split(self.delimiter)
-            file_name = substr[0]
-            label = substr[1]
-            img_path = os.path.join(self.data_dir, file_name)
-            if os.path.exists(img_path):
-                new_data_lines.append({'img_path': img_path, 'label': label})
-            else:
-                self.logger.info("{} does not exist!".format(img_path))
-        self.data_lines = new_data_lines
\ No newline at end of file

ppocr/modeling/architectures/distillation_model.py

@@ -46,7 +46,7 @@ class DistillationModel(nn.Layer):
                 pretrained = model_config.pop("pretrained")
             model = BaseModel(model_config)
             if pretrained is not None:
-                model = load_pretrained_params(model, pretrained)
+                load_pretrained_params(model, pretrained)
             if freeze_params:
                 for param in model.parameters():
                     param.trainable = False

ppocr/utils/gen_label.py

-#copyright (c) 2020 PaddlePaddle Authors. All Rights Reserve.
+# copyright (c) 2020 PaddlePaddle Authors. All Rights Reserve.
 #
-#Licensed under the Apache License, Version 2.0 (the "License");
-#you may not use this file except in compliance with the License.
-#You may obtain a copy of the License at
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
 #
 #    http://www.apache.org/licenses/LICENSE-2.0
 #
-#Unless required by applicable law or agreed to in writing, software
-#distributed under the License is distributed on an "AS IS" BASIS,
-#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#See the License for the specific language governing permissions and
-#limitations under the License.
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 import os
 import argparse
 import json
@@ -31,7 +31,9 @@ def gen_det_label(root_path, input_dir, out_label):
         for label_file in os.listdir(input_dir):
             img_path = root_path + label_file[3:-4] + ".jpg"
             label = []
-            with open(os.path.join(input_dir, label_file), 'r') as f:
+            with open(
+                    os.path.join(input_dir, label_file), 'r',
+                    encoding='utf-8-sig') as f:
                 for line in f.readlines():
                     tmp = line.strip("\n\r").replace("\xef\xbb\xbf",
                                                      "").split(',')

test1/MANIFEST.in → ppstructure/MANIFEST.in

@@ -5,5 +5,5 @@ recursive-include ppocr/utils *.txt utility.py logging.py network.py
 recursive-include ppocr/data/ *.py
 recursive-include ppocr/postprocess *.py
 recursive-include tools/infer *.py
-recursive-include test1 *.py
+recursive-include ppstructure *.py
 

test1/api.md → ppstructure/README.md

@@ -10,7 +10,7 @@ pip3 install layoutparser-0.0.0-py3-none-any.whl
 
 PaddleStructure is a toolkit for complex layout text OCR, the process is as follows
 
-![pipeline](../doc/table/pipeline.png)
+![pipeline](../doc/table/pipeline.jpg)
 
 In PaddleStructure, the image will be analyzed by layoutparser first. In the layout analysis, the area in the image will be classified, and the OCR process will be carried out according to the category.
 

test1/api_ch.md → ppstructure/README_ch.md

@@ -9,7 +9,7 @@ pip3 install layoutparser-0.0.0-py3-none-any.whl
 ## 1. pipeline介绍
 
 PaddleStructure 是一个用于复杂板式文字OCR的工具包，流程如下
-![pipeline](../doc/table/pipeline.png)
+![pipeline](../doc/table/pipeline.jpg)
 
 在PaddleStructure中，图片会先经由layoutparser进行版面分析，在版面分析中，会对图片里的区域进行分类，根据根据类别进行对于的ocr流程。
 

test1/paddlestructure.py → ppstructure/paddlestructure.py

@@ -24,9 +24,9 @@ import numpy as np
 from pathlib import Path
 
 from ppocr.utils.logging import get_logger
-from test1.predict_system import OCRSystem, save_res
-from test1.table.predict_table import to_excel
-from test1.utility import init_args, draw_result
+from ppstructure.predict_system import OCRSystem, save_res
+from ppstructure.table.predict_table import to_excel
+from ppstructure.utility import init_args, draw_result
 
 logger = get_logger()
 from ppocr.utils.utility import check_and_read_gif, get_image_file_list

test1/predict_system.py → ppstructure/predict_system.py

@@ -31,8 +31,8 @@ import layoutparser as lp
 from ppocr.utils.utility import get_image_file_list, check_and_read_gif
 from ppocr.utils.logging import get_logger
 from tools.infer.predict_system import TextSystem
-from test1.table.predict_table import TableSystem, to_excel
-from test1.utility import parse_args, draw_result
+from ppstructure.table.predict_table import TableSystem, to_excel
+from ppstructure.utility import parse_args, draw_result
 
 logger = get_logger()
 

test1/setup.py → ppstructure/setup.py

@@ -23,14 +23,14 @@ with open('../requirements.txt', encoding="utf-8-sig") as f:
 
 
 def readme():
-    with open('api_ch.md', encoding="utf-8-sig") as f:
+    with open('README_ch.md', encoding="utf-8-sig") as f:
         README = f.read()
     return README
 
 
-shutil.copytree('./table', './test1/table')
-shutil.copyfile('./predict_system.py', './test1/predict_system.py')
-shutil.copyfile('./utility.py', './test1/utility.py')
+shutil.copytree('./table', './ppstructure/table')
+shutil.copyfile('./predict_system.py', './ppstructure/predict_system.py')
+shutil.copyfile('./utility.py', './ppstructure/utility.py')
 shutil.copytree('../ppocr', './ppocr')
 shutil.copytree('../tools', './tools')
 shutil.copyfile('../LICENSE', './LICENSE')
@@ -66,5 +66,5 @@ setup(
 
 shutil.rmtree('ppocr')
 shutil.rmtree('tools')
-shutil.rmtree('test1')
+shutil.rmtree('ppstructure')
 os.remove('LICENSE')

test1/table/eval_table.py → ppstructure/table/eval_table.py

@@ -20,9 +20,9 @@ sys.path.append(os.path.abspath(os.path.join(__dir__, '../..')))
 import cv2
 import json
 from tqdm import tqdm
-from test1.table.table_metric import TEDS
-from test1.table.predict_table import TableSystem
-from test1.utility import init_args
+from ppstructure.table.table_metric import TEDS
+from ppstructure.table.predict_table import TableSystem
+from ppstructure.utility import init_args
 from ppocr.utils.logging import get_logger
 
 logger = get_logger()

test1/table/predict_structure.py → ppstructure/table/predict_structure.py

@@ -22,17 +22,14 @@ os.environ["FLAGS_allocator_strategy"] = 'auto_growth'
 
 import cv2
 import numpy as np
-import math
 import time
-import traceback
-import paddle
 
 import tools.infer.utility as utility
 from ppocr.data import create_operators, transform
 from ppocr.postprocess import build_post_process
 from ppocr.utils.logging import get_logger
 from ppocr.utils.utility import get_image_file_list, check_and_read_gif
-from test1.utility import parse_args
+from ppstructure.utility import parse_args
 
 logger = get_logger()
 

test1/table/predict_table.py → ppstructure/table/predict_table.py

@@ -30,9 +30,9 @@ import tools.infer.predict_rec as predict_rec
 import tools.infer.predict_det as predict_det
 from ppocr.utils.utility import get_image_file_list, check_and_read_gif
 from ppocr.utils.logging import get_logger
-from test1.table.matcher import distance, compute_iou
-from test1.utility import parse_args
-import test1.table.predict_structure as predict_strture
+from ppstructure.table.matcher import distance, compute_iou
+from ppstructure.utility import parse_args
+import ppstructure.table.predict_structure as predict_strture
 
 logger = get_logger()