From f98cfee49e9ccf85da8d7b8d65e722ce993a03de Mon Sep 17 00:00:00 2001 From: wangguanzhong Date: Sat, 9 May 2020 12:27:37 +0800 Subject: [PATCH] optimize environment configuration (#521) * optimize python path --- dataset/coco/download_coco.py | 4 ++++ dataset/fruit/download_fruit.py | 4 ++++ dataset/voc/create_list.py | 4 ++++ dataset/voc/download_voc.py | 4 ++++ docs/advanced_tutorials/TRANSFER_LEARNING.md | 2 -- docs/advanced_tutorials/TRANSFER_LEARNING_cn.md | 2 -- docs/advanced_tutorials/inference/EXPORT_MODEL.md | 4 ++-- docs/featured_model/CONTRIB.md | 2 -- docs/featured_model/CONTRIB_cn.md | 2 -- docs/featured_model/FACE_DETECTION.md | 1 - docs/featured_model/FACE_DETECTION_en.md | 1 - docs/tutorials/GETTING_STARTED.md | 2 -- docs/tutorials/GETTING_STARTED_cn.md | 2 -- docs/tutorials/INSTALL.md | 12 ------------ docs/tutorials/INSTALL_cn.md | 12 ------------ docs/tutorials/QUICK_STARTED.md | 3 +-- docs/tutorials/QUICK_STARTED_cn.md | 3 +-- ppdet/data/tests/test_dataset.py | 4 ++++ ppdet/data/tests/test_loader.py | 5 +++++ ppdet/data/tests/test_loader_yaml.py | 5 +++++ ppdet/modeling/tests/test_architectures.py | 6 ++++++ tools/configure.py | 6 ++++++ tools/eval.py | 5 +++++ tools/export_model.py | 5 +++++ tools/face_eval.py | 5 +++++ tools/infer.py | 5 +++++ tools/train.py | 5 +++++ 27 files changed, 71 insertions(+), 44 deletions(-) diff --git a/dataset/coco/download_coco.py b/dataset/coco/download_coco.py index b3ad1e23b..47659fa76 100644 --- a/dataset/coco/download_coco.py +++ b/dataset/coco/download_coco.py @@ -15,6 +15,10 @@ import sys import os.path as osp import logging +# add python path of PadleDetection to sys.path +parent_path = osp.abspath(osp.join(__file__, *(['..'] * 3))) +if parent_path not in sys.path: + sys.path.append(parent_path) from ppdet.utils.download import download_dataset diff --git a/dataset/fruit/download_fruit.py b/dataset/fruit/download_fruit.py index 6aa41ba4c..2db2e2072 100644 --- a/dataset/fruit/download_fruit.py +++ b/dataset/fruit/download_fruit.py @@ -15,6 +15,10 @@ import sys import os.path as osp import logging +# add python path of PadleDetection to sys.path +parent_path = osp.abspath(osp.join(__file__, *(['..'] * 3))) +if parent_path not in sys.path: + sys.path.append(parent_path) from ppdet.utils.download import download_dataset diff --git a/dataset/voc/create_list.py b/dataset/voc/create_list.py index c2cef1159..5ab804222 100644 --- a/dataset/voc/create_list.py +++ b/dataset/voc/create_list.py @@ -15,6 +15,10 @@ import sys import os.path as osp import logging +# add python path of PadleDetection to sys.path +parent_path = osp.abspath(osp.join(__file__, *(['..'] * 3))) +if parent_path not in sys.path: + sys.path.append(parent_path) from ppdet.utils.download import create_voc_list diff --git a/dataset/voc/download_voc.py b/dataset/voc/download_voc.py index 31ed46f6c..e4c449ce2 100644 --- a/dataset/voc/download_voc.py +++ b/dataset/voc/download_voc.py @@ -15,6 +15,10 @@ import sys import os.path as osp import logging +# add python path of PadleDetection to sys.path +parent_path = osp.abspath(osp.join(__file__, *(['..'] * 3))) +if parent_path not in sys.path: + sys.path.append(parent_path) from ppdet.utils.download import download_dataset diff --git a/docs/advanced_tutorials/TRANSFER_LEARNING.md b/docs/advanced_tutorials/TRANSFER_LEARNING.md index 176c46ea6..975b85ccb 100644 --- a/docs/advanced_tutorials/TRANSFER_LEARNING.md +++ b/docs/advanced_tutorials/TRANSFER_LEARNING.md @@ -15,7 +15,6 @@ In transfer learning, it's needed to load pretrained model selectively. Two ways The parameters which have diffierent shape between model and pretrain\_weights are ignored automatically. For example: ```python -export PYTHONPATH=$PYTHONPATH:. export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 python -u tools/train.py -c configs/faster_rcnn_r50_1x.yml \ -o pretrain_weights=https://paddlemodels.bj.bcebos.com/object_detection/faster_rcnn_r50_1x.tar @@ -29,7 +28,6 @@ The parameters which need to ignore can be specified explicitly as well and arbi - Set `finetune_exclude_pretrained_params` in command line. For example: ```python -export PYTHONPATH=$PYTHONPATH:. export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 python -u tools/train.py -c configs/faster_rcnn_r50_1x.yml \ -o pretrain_weights=https://paddlemodels.bj.bcebos.com/object_detection/faster_rcnn_r50_1x.tar \ diff --git a/docs/advanced_tutorials/TRANSFER_LEARNING_cn.md b/docs/advanced_tutorials/TRANSFER_LEARNING_cn.md index b714bf00a..7db411ac4 100644 --- a/docs/advanced_tutorials/TRANSFER_LEARNING_cn.md +++ b/docs/advanced_tutorials/TRANSFER_LEARNING_cn.md @@ -13,7 +13,6 @@ 模型中和预训练模型中对应参数形状不同的参数将自动被忽略,例如: ```python -export PYTHONPATH=$PYTHONPATH:. export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 python -u tools/train.py -c configs/faster_rcnn_r50_1x.yml \ -o pretrain_weights=https://paddlemodels.bj.bcebos.com/object_detection/faster_rcnn_r50_1x.tar @@ -28,7 +27,6 @@ python -u tools/train.py -c configs/faster_rcnn_r50_1x.yml \ 2. 在 train.py的启动参数中设置`finetune_exclude_pretrained_params`。例如: ```python -export PYTHONPATH=$PYTHONPATH:. export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 python -u tools/train.py -c configs/faster_rcnn_r50_1x.yml \ -o pretrain_weights=https://paddlemodels.bj.bcebos.com/object_detection/faster_rcnn_r50_1x.tar \ diff --git a/docs/advanced_tutorials/inference/EXPORT_MODEL.md b/docs/advanced_tutorials/inference/EXPORT_MODEL.md index 70a5b8743..043e07ec5 100644 --- a/docs/advanced_tutorials/inference/EXPORT_MODEL.md +++ b/docs/advanced_tutorials/inference/EXPORT_MODEL.md @@ -4,7 +4,7 @@ **说明:** -- **输入部分:**导出模型输入为网络输入图像,即原始图片经过预处理后的图像,具体预处理方式可参考配置文件中TestReader部分。各类检测模型的输入格式分别为: +- **输入部分:** 导出模型输入为网络输入图像,即原始图片经过预处理后的图像,具体预处理方式可参考配置文件中TestReader部分。各类检测模型的输入格式分别为: | 模型系列名称 | 输入图像预处理方式 | 其他输入信息 | | :---------: | ----------- | ---------- | @@ -16,7 +16,7 @@ | Face | 归一化 | im\_shape: 格式为[origin\_H, origin\_W], origin为原始图像 | -- **输出部分:**导出模型输出统一为NMS的输出,形状为[N, 6], 其中N为预测框的个数,6为[class_id, score, x1, y1, x2, y2]。 +- **输出部分:** 导出模型输出统一为NMS的输出,形状为[N, 6], 其中N为预测框的个数,6为[class_id, score, x1, y1, x2, y2]。 - 模型导出不支持模型结构中包含```fluid.layers.py_func```的情况。 diff --git a/docs/featured_model/CONTRIB.md b/docs/featured_model/CONTRIB.md index 8121ea3b8..f5630d32a 100644 --- a/docs/featured_model/CONTRIB.md +++ b/docs/featured_model/CONTRIB.md @@ -43,7 +43,6 @@ Users can employ the model to conduct the inference: ``` export CUDA_VISIBLE_DEVICES=0 -export PYTHONPATH=$PYTHONPATH:. python -u tools/infer.py -c contrib/VehicleDetection/vehicle_yolov3_darknet.yml \ -o weights=https://paddlemodels.bj.bcebos.com/object_detection/vehicle_yolov3_darknet.tar \ --infer_dir contrib/VehicleDetection/demo \ @@ -90,7 +89,6 @@ Users can employ the model to conduct the inference: ``` export CUDA_VISIBLE_DEVICES=0 -export PYTHONPATH=$PYTHONPATH:. python -u tools/infer.py -c contrib/PedestrianDetection/pedestrian_yolov3_darknet.yml \ -o weights=https://paddlemodels.bj.bcebos.com/object_detection/pedestrian_yolov3_darknet.tar \ --infer_dir contrib/PedestrianDetection/demo \ diff --git a/docs/featured_model/CONTRIB_cn.md b/docs/featured_model/CONTRIB_cn.md index 90d48a31f..dbc9f1926 100644 --- a/docs/featured_model/CONTRIB_cn.md +++ b/docs/featured_model/CONTRIB_cn.md @@ -44,7 +44,6 @@ IOU=.5时的AP为 0.764。 ``` export CUDA_VISIBLE_DEVICES=0 -export PYTHONPATH=$PYTHONPATH:. python -u tools/infer.py -c contrib/VehicleDetection/vehicle_yolov3_darknet.yml \ -o weights=https://paddlemodels.bj.bcebos.com/object_detection/vehicle_yolov3_darknet.tar \ --infer_dir contrib/VehicleDetection/demo \ @@ -92,7 +91,6 @@ IOU=.5-.95时的AP为 0.518。 ``` export CUDA_VISIBLE_DEVICES=0 -export PYTHONPATH=$PYTHONPATH:. python -u tools/infer.py -c contrib/PedestrianDetection/pedestrian_yolov3_darknet.yml \ -o weights=https://paddlemodels.bj.bcebos.com/object_detection/pedestrian_yolov3_darknet.tar \ --infer_dir contrib/PedestrianDetection/demo \ diff --git a/docs/featured_model/FACE_DETECTION.md b/docs/featured_model/FACE_DETECTION.md index be51058a8..eed2c4439 100644 --- a/docs/featured_model/FACE_DETECTION.md +++ b/docs/featured_model/FACE_DETECTION.md @@ -155,7 +155,6 @@ cd dataset/wider_face && ./download.sh 评估并生成结果文件: ``` export CUDA_VISIBLE_DEVICES=0 -export PYTHONPATH=$PYTHONPATH:. python -u tools/face_eval.py -c configs/face_detection/blazeface.yml \ -o weights=output/blazeface/model_final \ --eval_mode=widerface diff --git a/docs/featured_model/FACE_DETECTION_en.md b/docs/featured_model/FACE_DETECTION_en.md index 3ef9f7fc1..0099ff8c1 100644 --- a/docs/featured_model/FACE_DETECTION_en.md +++ b/docs/featured_model/FACE_DETECTION_en.md @@ -170,7 +170,6 @@ Default is `False`, it will select `single-scale` evaluation. - Evaluate and generate results files: ``` export CUDA_VISIBLE_DEVICES=0 -export PYTHONPATH=$PYTHONPATH:. python -u tools/face_eval.py -c configs/face_detection/blazeface.yml \ -o weights=output/blazeface/model_final \ --eval_mode=widerface diff --git a/docs/tutorials/GETTING_STARTED.md b/docs/tutorials/GETTING_STARTED.md index 405f1eb0e..fb3105079 100644 --- a/docs/tutorials/GETTING_STARTED.md +++ b/docs/tutorials/GETTING_STARTED.md @@ -11,8 +11,6 @@ instructions](INSTALL.md). PaddleDetection provides scripots for training, evalution and inference with various features according to different configure. ```bash -# set PYTHONPATH -export PYTHONPATH=$PYTHONPATH:. # training in single-GPU and multi-GPU. specify different GPU numbers by CUDA_VISIBLE_DEVICES export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 python tools/train.py -c configs/faster_rcnn_r50_1x.yml diff --git a/docs/tutorials/GETTING_STARTED_cn.md b/docs/tutorials/GETTING_STARTED_cn.md index 47751bb49..6b44b0fcd 100644 --- a/docs/tutorials/GETTING_STARTED_cn.md +++ b/docs/tutorials/GETTING_STARTED_cn.md @@ -8,8 +8,6 @@ PaddleDetection提供了训练/评估/推断三个功能的使用脚本,支持通过不同可选参数实现特定功能 ```bash -# 设置PYTHONPATH路径 -export PYTHONPATH=$PYTHONPATH:. # GPU训练 支持单卡,多卡训练,通过CUDA_VISIBLE_DEVICES指定卡号 export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 python tools/train.py -c configs/faster_rcnn_r50_1x.yml diff --git a/docs/tutorials/INSTALL.md b/docs/tutorials/INSTALL.md index 697c7a96d..7835adc37 100644 --- a/docs/tutorials/INSTALL.md +++ b/docs/tutorials/INSTALL.md @@ -88,15 +88,6 @@ Required python packages are specified in [requirements.txt](https://github.com/ pip install -r requirements.txt ``` -**Specify the current Python path:** - -```shell -# In Linux/Mac -export PYTHONPATH=$PYTHONPATH:. -# In windows -set PYTHONPATH=%PYTHONPATH%;. -``` - **Make sure the tests pass:** ```shell @@ -121,7 +112,6 @@ ln -sf /dataset/voc For Pascal VOC dataset, you should create file list by: ``` -export PYTHONPATH=$PYTHONPATH:. python dataset/voc/create_list.py ``` @@ -132,7 +122,6 @@ On the other hand, to download the datasets, run the following commands: - COCO ``` -export PYTHONPATH=$PYTHONPATH:. python dataset/coco/download_coco.py ``` @@ -160,7 +149,6 @@ python dataset/coco/download_coco.py - Pascal VOC ``` -export PYTHONPATH=$PYTHONPATH:. python dataset/voc/download_voc.py python dataset/voc/create_list.py ``` diff --git a/docs/tutorials/INSTALL_cn.md b/docs/tutorials/INSTALL_cn.md index 75dbe0af2..3495d4dba 100644 --- a/docs/tutorials/INSTALL_cn.md +++ b/docs/tutorials/INSTALL_cn.md @@ -84,15 +84,6 @@ Python依赖库在[requirements.txt](https://github.com/PaddlePaddle/PaddleDetec pip install -r requirements.txt ``` -**指定当前Python路径:** - -```shell -# 在Linux/Mac系统下运行: -export PYTHONPATH=$PYTHONPATH:. -# 在windows系统下运行: -set PYTHONPATH=%PYTHONPATH%;. -``` - **确认测试通过:** ``` @@ -119,7 +110,6 @@ ln -sf /dataset/voc 对于Pascal VOC数据集,需通过如下命令创建文件列表: ``` -export PYTHONPATH=$PYTHONPATH:. python dataset/voc/create_list.py ``` @@ -130,7 +120,6 @@ python dataset/voc/create_list.py - COCO ``` -export PYTHONPATH=$PYTHONPATH:. python dataset/coco/download_coco.py ``` @@ -158,7 +147,6 @@ python dataset/coco/download_coco.py - Pascal VOC ``` -export PYTHONPATH=$PYTHONPATH:. python dataset/voc/download_voc.py python dataset/voc/create_list.py ``` diff --git a/docs/tutorials/QUICK_STARTED.md b/docs/tutorials/QUICK_STARTED.md index ac0c3abe9..e40d6869e 100644 --- a/docs/tutorials/QUICK_STARTED.md +++ b/docs/tutorials/QUICK_STARTED.md @@ -4,10 +4,9 @@ English | [简体中文](QUICK_STARTED_cn.md) This tutorial fine-tunes a tiny dataset by pretrained detection model for users to get a model and learn PaddleDetection quickly. The model can be trained in around 20min with good performance. -- **Note: before started, need to set PYTHONPATH and specifiy the GPU device as follows in Linux platform. For Windows users, also need to set PYTHONPATH correctly.** +- **Note: before started, need to specifiy the GPU device as follows.** ```bash -export PYTHONPATH=$PYTHONPATH:. export CUDA_VISIBLE_DEVICES=0 ``` diff --git a/docs/tutorials/QUICK_STARTED_cn.md b/docs/tutorials/QUICK_STARTED_cn.md index df0b318cc..407cb7f0b 100644 --- a/docs/tutorials/QUICK_STARTED_cn.md +++ b/docs/tutorials/QUICK_STARTED_cn.md @@ -4,10 +4,9 @@ 为了使得用户能够在很短的时间内快速产出模型,掌握PaddleDetection的使用方式,这篇教程通过一个预训练检测模型对小数据集进行finetune。在P40上单卡大约20min即可产出一个效果不错的模型。 -- **注:在开始前,对于Linux用户, 运行如下命令设置PYTHONPATH,如果有GPU设备,指定GPU设备号。对于Windows用户,同样需要将PaddleDetection的根目录设置到PYTHONPATH中。** +- **注:在开始前,如果有GPU设备,指定GPU设备号。** ```bash -export PYTHONPATH=$PYTHONPATH:. export CUDA_VISIBLE_DEVICES=0 ``` diff --git a/ppdet/data/tests/test_dataset.py b/ppdet/data/tests/test_dataset.py index 2e490551e..86e87257d 100644 --- a/ppdet/data/tests/test_dataset.py +++ b/ppdet/data/tests/test_dataset.py @@ -18,6 +18,10 @@ import sys import logging import random import copy +# add python path of PadleDetection to sys.path +parent_path = os.path.abspath(os.path.join(__file__, *(['..'] * 4))) +if parent_path not in sys.path: + sys.path.append(parent_path) from ppdet.data.parallel_map import ParallelMap diff --git a/ppdet/data/tests/test_loader.py b/ppdet/data/tests/test_loader.py index 9f373ad99..26c990a16 100644 --- a/ppdet/data/tests/test_loader.py +++ b/ppdet/data/tests/test_loader.py @@ -14,6 +14,11 @@ import unittest import os +import sys +# add python path of PadleDetection to sys.path +parent_path = os.path.abspath(os.path.join(__file__, *(['..'] * 4))) +if parent_path not in sys.path: + sys.path.append(parent_path) from ppdet.data.source.coco import COCODataSet from ppdet.data.reader import Reader diff --git a/ppdet/data/tests/test_loader_yaml.py b/ppdet/data/tests/test_loader_yaml.py index 121f52729..b0b526586 100644 --- a/ppdet/data/tests/test_loader_yaml.py +++ b/ppdet/data/tests/test_loader_yaml.py @@ -16,6 +16,11 @@ import unittest import os import yaml import logging +import sys +# add python path of PadleDetection to sys.path +parent_path = os.path.abspath(os.path.join(__file__, *(['..'] * 4))) +if parent_path not in sys.path: + sys.path.append(parent_path) from ppdet.utils.download import get_path from ppdet.utils.download import DATASET_HOME diff --git a/ppdet/modeling/tests/test_architectures.py b/ppdet/modeling/tests/test_architectures.py index fcfd87908..ef8ebfc16 100644 --- a/ppdet/modeling/tests/test_architectures.py +++ b/ppdet/modeling/tests/test_architectures.py @@ -20,6 +20,12 @@ import unittest import numpy as np import paddle.fluid as fluid +import os +import sys +# add python path of PadleDetection to sys.path +parent_path = os.path.abspath(os.path.join(__file__, *(['..'] * 4))) +if parent_path not in sys.path: + sys.path.append(parent_path) from ppdet.modeling.tests.decorator_helper import prog_scope from ppdet.core.workspace import load_config, merge_config, create diff --git a/tools/configure.py b/tools/configure.py index 098f48f84..a3d5bb013 100644 --- a/tools/configure.py +++ b/tools/configure.py @@ -14,8 +14,14 @@ from __future__ import print_function +import os import sys from argparse import ArgumentParser, RawDescriptionHelpFormatter +import sys +# add python path of PadleDetection to sys.path +parent_path = os.path.abspath(os.path.join(__file__, *(['..'] * 2))) +if parent_path not in sys.path: + sys.path.append(parent_path) import yaml diff --git a/tools/eval.py b/tools/eval.py index ef9a8ac57..bdf93180e 100644 --- a/tools/eval.py +++ b/tools/eval.py @@ -17,6 +17,11 @@ from __future__ import division from __future__ import print_function import os +import sys +# add python path of PadleDetection to sys.path +parent_path = os.path.abspath(os.path.join(__file__, *(['..'] * 2))) +if parent_path not in sys.path: + sys.path.append(parent_path) def set_paddle_flags(**kwargs): diff --git a/tools/export_model.py b/tools/export_model.py index 554840a0b..20b43e8c1 100644 --- a/tools/export_model.py +++ b/tools/export_model.py @@ -17,6 +17,11 @@ from __future__ import division from __future__ import print_function import os +import sys +# add python path of PadleDetection to sys.path +parent_path = os.path.abspath(os.path.join(__file__, *(['..'] * 2))) +if parent_path not in sys.path: + sys.path.append(parent_path) from paddle import fluid diff --git a/tools/face_eval.py b/tools/face_eval.py index ff1ef0f01..41545760a 100644 --- a/tools/face_eval.py +++ b/tools/face_eval.py @@ -17,6 +17,11 @@ from __future__ import division from __future__ import print_function import os +import sys +# add python path of PadleDetection to sys.path +parent_path = os.path.abspath(os.path.join(__file__, *(['..'] * 2))) +if parent_path not in sys.path: + sys.path.append(parent_path) import paddle.fluid as fluid import numpy as np diff --git a/tools/infer.py b/tools/infer.py index fdc370e53..699c48bcd 100644 --- a/tools/infer.py +++ b/tools/infer.py @@ -18,6 +18,11 @@ from __future__ import print_function import os import glob +import sys +# add python path of PadleDetection to sys.path +parent_path = os.path.abspath(os.path.join(__file__, *(['..'] * 2))) +if parent_path not in sys.path: + sys.path.append(parent_path) import numpy as np from PIL import Image diff --git a/tools/train.py b/tools/train.py index a1129a568..7fc8b4b87 100644 --- a/tools/train.py +++ b/tools/train.py @@ -23,6 +23,11 @@ import random import datetime from collections import deque from paddle.fluid import profiler +import sys +# add python path of PadleDetection to sys.path +parent_path = os.path.abspath(os.path.join(__file__, *(['..'] * 2))) +if parent_path not in sys.path: + sys.path.append(parent_path) def set_paddle_flags(**kwargs): -- GitLab