Skip to content

P
PaddleDetection

PaddlePaddle / PaddleDetection
1 年多 前同步成功

通知 696
Star 11112
Fork 2696
P
PaddleDetection
提交 23d3745d 编写于 作者: R root
浏览文件
操作

Merge branch 'ppdet_split' of /paddle/work/paddle-fork/models into init_ppdet

上级 db2d30dd 27545a84
隐藏空白更改
内联 并排
Showing with 466 addition and 207 deletion
configs/dcn/cascade_mask_rcnn_dcnv2_se154_vd_fpn_gn_s1x.yml
浏览文件 @ 23d3745d
......@@ -233,6 +233,24 @@ MaskRCNNTestFeed:
batch_size: 1
dataset:
annotation: dataset/coco/annotations/instances_val2017.json
sample_transforms:
- !DecodeImage
to_rgb: False
with_mixup: False
- !NormalizeImage
is_channel_first: false
is_scale: False
mean:
- 102.9801
- 115.9465
- 122.7717
std:
- 1.0
- 1.0
- 1.0
- !Permute
channel_first: true
to_bgr: false
batch_transforms:
- !PadBatch
pad_to_stride: 32
......
configs/dcn/cascade_mask_rcnn_dcnv2_se154_vd_fpn_gn_s1x_ms_test.yml
浏览文件 @ 23d3745d
......@@ -249,6 +249,23 @@ MaskRCNNTestFeed:
batch_size: 1
dataset:
annotation: dataset/coco/annotations/instances_val2017.json
sample_transforms:
- !DecodeImage
to_rgb: False
- !NormalizeImage
is_channel_first: false
is_scale: False
mean:
- 102.9801
- 115.9465
- 122.7717
std:
- 1.0
- 1.0
- 1.0
- !Permute
channel_first: true
to_bgr: false
batch_transforms:
- !PadBatch
pad_to_stride: 32
......
configs/obj365/cascade_rcnn_dcnv2_se154_vd_fpn_gn_cas.yml
浏览文件 @ 23d3745d
......@@ -192,6 +192,28 @@ FasterRCNNEvalFeed:
dataset_dir: dataset/objects365
annotation: annotations/val.json
image_dir: val
sample_transforms:
- !DecodeImage
to_rgb: False
with_mixup: False
- !NormalizeImage
is_channel_first: false
is_scale: False
mean:
- 102.9801
- 115.9465
- 122.7717
std:
- 1.0
- 1.0
- 1.0
- !ResizeImage
target_size: 800
max_size: 1333
interp: 1
- !Permute
channel_first: true
to_bgr: false
batch_transforms:
- !PadBatch
pad_to_stride: 32
......@@ -200,6 +222,24 @@ FasterRCNNTestFeed:
batch_size: 1
dataset:
annotation: dataset/obj365/annotations/val.json
sample_transforms:
- !DecodeImage
to_rgb: False
with_mixup: False
- !NormalizeImage
is_channel_first: false
is_scale: False
mean:
- 102.9801
- 115.9465
- 122.7717
std:
- 1.0
- 1.0
- 1.0
- !Permute
channel_first: true
to_bgr: false
batch_transforms:
- !PadBatch
pad_to_stride: 32
......
configs/ssd/ssd_mobilenet_v1_voc.yml
浏览文件 @ 23d3745d
......@@ -61,8 +61,7 @@ SSDTrainFeed:
use_process: true
dataset:
dataset_dir: dataset/voc
annotation: VOCdevkit/VOC_all/ImageSets/Main/train.txt
image_dir: VOCdevkit/VOC_all/JPEGImages
annotation: trainval.txt
use_default_label: true
SSDEvalFeed:
......@@ -70,8 +69,7 @@ SSDEvalFeed:
use_process: true
dataset:
dataset_dir: dataset/voc
annotation: VOCdevkit/VOC_all/ImageSets/Main/val.txt
image_dir: VOCdevkit/VOC_all/JPEGImages
annotation: test.txt
use_default_label: true
drop_last: false
......
configs/ssd/ssd_vgg16_300_voc.yml
浏览文件 @ 23d3745d
......@@ -64,8 +64,7 @@ SSDTrainFeed:
batch_size: 8
dataset:
dataset_dir: dataset/voc
annotation: VOCdevkit/VOC_all/ImageSets/Main/train.txt
image_dir: VOCdevkit/VOC_all/JPEGImages
annotation: trainval.txt
use_default_label: true
image_shape: [3, 300, 300]
sample_transforms:
......@@ -109,8 +108,7 @@ SSDEvalFeed:
batch_size: 32
dataset:
dataset_dir: dataset/voc
annotation: VOCdevkit/VOC_all/ImageSets/Main/val.txt
image_dir: VOCdevkit/VOC_all/JPEGImages
annotation: test.txt
use_default_label: true
drop_last: false
image_shape: [3, 300, 300]
......
configs/ssd/ssd_vgg16_512_voc.yml
浏览文件 @ 23d3745d
......@@ -68,8 +68,7 @@ SSDTrainFeed:
batch_size: 8
dataset:
dataset_dir: dataset/voc
annotation: VOCdevkit/VOC_all/ImageSets/Main/train.txt
image_dir: VOCdevkit/VOC_all/JPEGImages
annotation: trainval.txt
use_default_label: true
image_shape: [3, 512, 512]
sample_transforms:
......@@ -113,8 +112,7 @@ SSDEvalFeed:
batch_size: 32
dataset:
dataset_dir: dataset/voc
annotation: VOCdevkit/VOC_all/ImageSets/Main/val.txt
image_dir: VOCdevkit/VOC_all/JPEGImages
annotation: test.txt
use_default_label: true
drop_last: false
image_shape: [3, 512, 512]
......
configs/yolov3_darknet_voc.yml
浏览文件 @ 23d3745d
......@@ -62,8 +62,7 @@ YoloTrainFeed:
batch_size: 8
dataset:
dataset_dir: dataset/voc
annotation: VOCdevkit/VOC_all/ImageSets/Main/train.txt
image_dir: VOCdevkit/VOC_all/JPEGImages
annotation: trainval.txt
use_default_label: true
num_workers: 8
bufsize: 128
......@@ -75,8 +74,7 @@ YoloEvalFeed:
image_shape: [3, 608, 608]
dataset:
dataset_dir: dataset/voc
annotation: VOCdevkit/VOC_all/ImageSets/Main/val.txt
image_dir: VOCdevkit/VOC_all/JPEGImages
annotation: test.txt
use_default_label: true
YoloTestFeed:
......
configs/yolov3_mobilenet_v1_fruit.yml
浏览文件 @ 23d3745d
......@@ -64,8 +64,7 @@ YoloTrainFeed:
batch_size: 1
dataset:
dataset_dir: dataset/fruit/fruit-detection
annotation: ./ImageSets/Main/train.txt
image_dir: ./JPEGImages
annotation: train.txt
use_default_label: false
num_workers: 16
bufsize: 128
......@@ -111,8 +110,7 @@ YoloEvalFeed:
image_shape: [3, 608, 608]
dataset:
dataset_dir: dataset/fruit/fruit-detection
annotation: ./ImageSets/Main/val.txt
image_dir: ./JPEGImages
annotation: val.txt
use_default_label: false
......@@ -121,5 +119,4 @@ YoloTestFeed:
image_shape: [3, 608, 608]
dataset:
dataset_dir: dataset/fruit/fruit-detection
annotation: ./ImageSets/Main/label_list.txt
use_default_label: false
configs/yolov3_mobilenet_v1_voc.yml
浏览文件 @ 23d3745d
......@@ -63,8 +63,7 @@ YoloTrainFeed:
batch_size: 8
dataset:
dataset_dir: dataset/voc
annotation: VOCdevkit/VOC_all/ImageSets/Main/train.txt
image_dir: VOCdevkit/VOC_all/JPEGImages
annotation: trainval.txt
use_default_label: true
num_workers: 8
bufsize: 128
......@@ -76,8 +75,7 @@ YoloEvalFeed:
image_shape: [3, 608, 608]
dataset:
dataset_dir: dataset/voc
annotation: VOCdevkit/VOC_all/ImageSets/Main/val.txt
image_dir: VOCdevkit/VOC_all/JPEGImages
annotation: test.txt
use_default_label: true
YoloTestFeed:
......
configs/yolov3_r34_voc.yml
浏览文件 @ 23d3745d
......@@ -65,8 +65,7 @@ YoloTrainFeed:
batch_size: 8
dataset:
dataset_dir: dataset/voc
annotation: VOCdevkit/VOC_all/ImageSets/Main/train.txt
image_dir: VOCdevkit/VOC_all/JPEGImages
annotation: trainval.txt
use_default_label: true
num_workers: 8
bufsize: 128
......@@ -78,8 +77,7 @@ YoloEvalFeed:
image_shape: [3, 608, 608]
dataset:
dataset_dir: dataset/voc
annotation: VOCdevkit/VOC_all/ImageSets/Main/val.txt
image_dir: VOCdevkit/VOC_all/JPEGImages
annotation: test.txt
use_default_label: true
YoloTestFeed:
......
dataset/voc/create_list.py 0 → 100644
浏览文件 @ 23d3745d
# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import sys
import os.path as osp
import logging
from ppdet.utils.download import create_voc_list
logging.basicConfig(level=logging.INFO)
voc_path = osp.split(osp.realpath(sys.argv[0]))[0]
create_voc_list(voc_path)
dataset/voc/label_list.txt 0 → 100644
浏览文件 @ 23d3745d
aeroplane
bicycle
bird
boat
bottle
bus
car
cat
chair
cow
diningtable
dog
horse
motorbike
person
pottedplant
sheep
sofa
train
tvmonitor
docs/DATA.md
浏览文件 @ 23d3745d
......@@ -27,6 +27,7 @@ Parses various data sources and creates `data.Dataset` instances. Currently,
following data sources are supported:
- COCO data source
Loads `COCO` type datasets with directory structures like this:
```
......@@ -36,46 +37,54 @@ Loads `COCO` type datasets with directory structures like this:
│ ├── instances_train2017.json
│ ├── instances_val2014.json
│ ├── instances_val2017.json
| ...
│ | ...
├── train2017
│ ├── 000000000009.jpg
│ ├── 000000580008.jpg
| ...
│ | ...
├── val2017
│ ├── 000000000139.jpg
│ ├── 000000000285.jpg
│ | ...
| ...
```
- Pascal VOC data source
Loads `Pascal VOC` like datasets with directory structure like this:
```
data/pascalvoc/
├──Annotations
│ ├── i000050.jpg
│ ├── 003876.xml
| ...
├── ImageSets
│ ├──Main
└── train.txt
└── val.txt
└── test.txt
└── dog_train.txt
└── dog_trainval.txt
└── dog_val.txt
└── dog_test.txt
└── ...
│ ├──Layout
└──...
│ ├── Segmentation
└──...
├── JPEGImages
│ ├── 000050.jpg
│ ├── 003876.jpg
dataset/voc/
├── train.txt
├── val.txt
├── test.txt
├── label_list.txt (optional)
├── VOCdevkit/VOC2007
│ ├── Annotations
│ ├── 001789.xml
│ | ...
│ ├── JPEGImages
│ ├── 001789.xml
│ | ...
│ ├── ImageSets
│ | ...
├── VOCdevkit/VOC2012
│ ├── Annotations
│ ├── 003876.xml
│ | ...
│ ├── JPEGImages
│ ├── 003876.xml
│ | ...
│ ├── ImageSets
│ | ...
| ...
```
**NOTE:** If you set `use_default_label=False` in yaml configs, the `label_list.txt`
of Pascal VOC dataset will be read, otherwise, `label_list.txt` is unnecessary and
the default Pascal VOC label list which defined in
[voc\_loader.py](../ppdet/data/source/voc_loader.py) will be used.
- Roidb data source
A generalized data source serialized as pickle files, which have the following
structure:
......@@ -181,16 +190,18 @@ whole data pipeline is fully customizable through the yaml configuration files.
#### Custom Datasets
- Option 1: Convert the dataset to COCO or VOC format.
- Option 1: Convert the dataset to COCO format.
```sh
# a small utility (`tools/labelme2coco.py`) is provided to convert
# Labelme-annotated dataset to COCO format.
python ./ppdet/data/tools/labelme2coco.py --json_input_dir ./labelme_annos/
# a small utility (`tools/x2coco.py`) is provided to convert
# Labelme-annotated dataset or cityscape dataset to COCO format.
python ./ppdet/data/tools/x2coco.py --dataset_type labelme
--json_input_dir ./labelme_annos/
--image_input_dir ./labelme_imgs/
--output_dir ./cocome/
--train_proportion 0.8
--val_proportion 0.2
--test_proportion 0.0
# --dataset_type: The data format which is need to be converted. Currently supported are: 'labelme' and 'cityscape'
# --json_input_dir：The path of json files which are annotated by Labelme.
# --image_input_dir：The path of images.
# --output_dir：The path of coverted COCO dataset.
......
docs/DATA_cn.md
浏览文件 @ 23d3745d
......@@ -11,9 +11,12 @@
子功能介绍：
1. 数据解析
数据解析得到的是`data.Dataset`,实现逻辑位于`data.source`中。通过它可以实现解析不同格式的数据集，已支持的数据源包括：
数据解析得到的是`data.Dataset`,实现逻辑位于`data.source`中。通过它可以实现解析不同格式的数据集，已支持的数据源包括：
- COCO数据源
该数据集目前分为COCO2012和COCO2017，主要由json文件和image文件组成，其组织结构如下所示：
该数据集目前分为COCO2014和COCO2017，主要由json文件和image文件组成，其组织结构如下所示：
```
dataset/coco/
......@@ -22,49 +25,53 @@
│ ├── instances_train2017.json
│ ├── instances_val2014.json
│ ├── instances_val2017.json
| ...
│ | ...
├── train2017
│ ├── 000000000009.jpg
│ ├── 000000580008.jpg
| ...
│ | ...
├── val2017
│ ├── 000000000139.jpg
│ ├── 000000000285.jpg
│ | ...
| ...
```
- Pascal VOC数据源
该数据集目前分为VOC2007和VOC2012，主要由xml文件和image文件组成，其组织结构如下所示：
该数据集目前分为VOC2007和VOC2012，主要由xml文件和image文件组成，其组织结构如下所示：
```
data/pascalvoc/
├──Annotations
│ ├── i000050.jpg
│ ├── 003876.xml
| ...
├── ImageSets
│ ├──Main
└── train.txt
└── val.txt
└── test.txt
└── dog_train.txt
└── dog_trainval.txt
└── dog_val.txt
└── dog_test.txt
└── ...
│ ├──Layout
└──...
│ ├── Segmentation
└──...
├── JPEGImages
│ ├── 000050.jpg
│ ├── 003876.jpg
dataset/voc/
├── train.txt
├── val.txt
├── test.txt
├── label_list.txt (optional)
├── VOCdevkit/VOC2007
│ ├── Annotations
│ ├── 001789.xml
│ | ...
│ ├── JPEGImages
│ ├── 001789.xml
│ | ...
│ ├── ImageSets
│ | ...
├── VOCdevkit/VOC2012
│ ├── Annotations
│ ├── 003876.xml
│ | ...
│ ├── JPEGImages
│ ├── 003876.xml
│ | ...
│ ├── ImageSets
│ | ...
| ...
```
**说明：** 如果你在yaml配置文件中设置`use_default_label=False`, 将从`label_list.txt`
中读取类别列表，反之则可以没有`label_list.txt`文件，检测库会使用Pascal VOC数据集的默
认类别列表，默认类别列表定义在[voc\_loader.py](../ppdet/data/source/voc_loader.py)
- Roidb数据源
该数据集主要由COCO数据集和Pascal VOC数据集转换而成的pickle文件，包含一个dict，而dict中只包含一个命名为‘records’的list（可能还有一个命名为‘cname2cid’的字典），其内容如下所示：
......@@ -165,15 +172,17 @@ coco = Reader(ccfg.DATA, ccfg.TRANSFORM, maxiter=-1)
```
#### 如何使用自定义数据集？
- 选择1：将数据集转换为VOC格式或者COCO格式。
- 选择1：将数据集转换为COCO格式。
```
# 在./tools/中提供了labelme2coco.py用于将labelme标注的数据集转换为COCO数据集
python ./ppdet/data/tools/labelme2coco.py --json_input_dir ./labelme_annos/
# 在./tools/中提供了x2coco.py用于将labelme标注的数据集或cityscape数据集转换为COCO数据集
python ./ppdet/data/tools/x2coco.py --dataset_type labelme
--json_input_dir ./labelme_annos/
--image_input_dir ./labelme_imgs/
--output_dir ./cocome/
--train_proportion 0.8
--val_proportion 0.2
--test_proportion 0.0
# --dataset_type：需要转换的数据格式，目前支持：’labelme‘和’cityscape‘
# --json_input_dir：使用labelme标注的json文件所在文件夹
# --image_input_dir：图像文件所在文件夹
# --output_dir：转换后的COCO格式数据集存放位置
......
docs/INSTALL.md
浏览文件 @ 23d3745d
......@@ -111,6 +111,13 @@ ln -sf <path/to/coco> <path/to/paddle_detection>/dataset/coco
ln -sf <path/to/voc> <path/to/paddle_detection>/dataset/voc
```
For Pascal VOC dataset, you should create file list by:
```
export PYTHONPATH=$PYTHONPATH:.
python dataset/voc/create_list.py
```
**Download datasets manually:**
On the other hand, to download the datasets, run the following commands:
......@@ -122,13 +129,69 @@ export PYTHONPATH=$PYTHONPATH:.
python dataset/coco/download_coco.py
```
`COCO` dataset with directory structures like this:
```
dataset/coco/
├── annotations
│ ├── instances_train2014.json
│ ├── instances_train2017.json
│ ├── instances_val2014.json
│ ├── instances_val2017.json
│ | ...
├── train2017
│ ├── 000000000009.jpg
│ ├── 000000580008.jpg
│ | ...
├── val2017
│ ├── 000000000139.jpg
│ ├── 000000000285.jpg
│ | ...
| ...
```
- Pascal VOC
```
export PYTHONPATH=$PYTHONPATH:.
python dataset/voc/download_voc.py
python dataset/voc/create_list.py
```
`Pascal VOC` dataset with directory structure like this:
```
dataset/voc/
├── train.txt
├── val.txt
├── test.txt
├── label_list.txt (optional)
├── VOCdevkit/VOC2007
│ ├── Annotations
│ ├── 001789.xml
│ | ...
│ ├── JPEGImages
│ ├── 001789.xml
│ | ...
│ ├── ImageSets
│ | ...
├── VOCdevkit/VOC2012
│ ├── Annotations
│ ├── 003876.xml
│ | ...
│ ├── JPEGImages
│ ├── 003876.xml
│ | ...
│ ├── ImageSets
│ | ...
| ...
```
**NOTE:** If you set `use_default_label=False` in yaml configs, the `label_list.txt`
of Pascal VOC dataset will be read, otherwise, `label_list.txt` is unnecessary and
the default Pascal VOC label list which defined in
[voc\_loader.py](../ppdet/data/source/voc_loader.py) will be used.
**Download datasets automatically:**
If a training session is started but the dataset is not setup properly (e.g,
......
docs/INSTALL_cn.md
浏览文件 @ 23d3745d
......@@ -108,6 +108,13 @@ ln -sf <path/to/coco> <path/to/paddle_detection>/dataset/coco
ln -sf <path/to/voc> <path/to/paddle_detection>/dataset/voc
```
对于Pascal VOC数据集，需通过如下命令创建文件列表：
```
export PYTHONPATH=$PYTHONPATH:.
python dataset/voc/create_list.py
```
**手动下载数据集：**
若您本地没有数据集，可通过如下命令下载：
......@@ -119,13 +126,68 @@ export PYTHONPATH=$PYTHONPATH:.
python dataset/coco/download_coco.py
```
`COCO` 数据集目录结构如下：
```
dataset/coco/
├── annotations
│ ├── instances_train2014.json
│ ├── instances_train2017.json
│ ├── instances_val2014.json
│ ├── instances_val2017.json
│ | ...
├── train2017
│ ├── 000000000009.jpg
│ ├── 000000580008.jpg
│ | ...
├── val2017
│ ├── 000000000139.jpg
│ ├── 000000000285.jpg
│ | ...
| ...
```
- Pascal VOC
```
export PYTHONPATH=$PYTHONPATH:.
python dataset/voc/download_voc.py
python dataset/voc/create_list.py
```
`Pascal VOC` 数据集目录结构如下：
```
dataset/voc/
├── train.txt
├── val.txt
├── test.txt
├── label_list.txt (optional)
├── VOCdevkit/VOC2007
│ ├── Annotations
│ ├── 001789.xml
│ | ...
│ ├── JPEGImages
│ ├── 001789.xml
│ | ...
│ ├── ImageSets
│ | ...
├── VOCdevkit/VOC2012
│ ├── Annotations
│ ├── 003876.xml
│ | ...
│ ├── JPEGImages
│ ├── 003876.xml
│ | ...
│ ├── ImageSets
│ | ...
| ...
```
**说明：** 如果你在yaml配置文件中设置`use_default_label=False`, 将从`label_list.txt`
中读取类别列表，反之则可以没有`label_list.txt`文件，检测库会使用Pascal VOC数据集的默
认类别列表，默认类别列表定义在[voc\_loader.py](../ppdet/data/source/voc_loader.py)
**自动下载数据集：**
若您在数据集未成功设置（例如，在`dataset/coco`或`dataset/voc`中找不到）的情况下开始运行，
......
ppdet/data/data_feed.py
浏览文件 @ 23d3745d
......@@ -219,7 +219,7 @@ class DataSet(object):
def __init__(self,
annotation,
image_dir,
image_dir=None,
dataset_dir=None,
use_default_label=None):
super(DataSet, self).__init__()
......@@ -229,7 +229,7 @@ class DataSet(object):
self.use_default_label = use_default_label
COCO_DATASET_DIR = 'coco'
COCO_DATASET_DIR = 'dataset/coco'
COCO_TRAIN_ANNOTATION = 'annotations/instances_train2017.json'
COCO_TRAIN_IMAGE_DIR = 'train2017'
COCO_VAL_ANNOTATION = 'annotations/instances_val2017.json'
......@@ -246,12 +246,11 @@ class CocoDataSet(DataSet):
dataset_dir=dataset_dir, annotation=annotation, image_dir=image_dir)
VOC_DATASET_DIR = 'pascalvoc'
VOC_TRAIN_ANNOTATION = 'VOCdevkit/VOC_all/ImageSets/Main/train.txt'
VOC_VAL_ANNOTATION = 'VOCdevkit/VOC_all/ImageSets/Main/val.txt'
VOC_TEST_ANNOTATION = 'VOCdevkit/VOC_all/ImageSets/Main/test.txt'
VOC_IMAGE_DIR = 'VOCdevkit/VOC_all/JPEGImages'
VOC_USE_DEFAULT_LABEL = None
VOC_DATASET_DIR = 'dataset/voc'
VOC_TRAIN_ANNOTATION = 'train.txt'
VOC_VAL_ANNOTATION = 'val.txt'
VOC_IMAGE_DIR = None
VOC_USE_DEFAULT_LABEL = True
@serializable
......@@ -843,7 +842,7 @@ class SSDTestFeed(DataFeed):
__doc__ = DataFeed.__doc__
def __init__(self,
dataset=SimpleDataSet(VOC_TEST_ANNOTATION).__dict__,
dataset=SimpleDataSet(VOC_VAL_ANNOTATION).__dict__,
fields=['image', 'im_id', 'im_shape'],
image_shape=[3, 300, 300],
sample_transforms=[
......
ppdet/data/source/roidb_source.py
浏览文件 @ 23d3745d
......@@ -62,7 +62,7 @@ class RoiDbSource(Dataset):
assert os.path.isfile(anno_file) or os.path.isdir(anno_file), \
'anno_file {} is not a file or a directory'.format(anno_file)
self._fname = anno_file
self._image_dir = image_dir
self._image_dir = image_dir if image_dir is not None else ''
if image_dir is not None:
assert os.path.isdir(image_dir), \
'image_dir {} is not a directory'.format(image_dir)
......
ppdet/data/source/voc_loader.py
浏览文件 @ 23d3745d
......@@ -26,8 +26,7 @@ def get_roidb(anno_path,
Load VOC records with annotations in xml directory 'anno_path'
Notes:
${anno_path}/ImageSets/Main/train.txt must contains xml file names for annotations
${anno_path}/Annotations/xxx.xml must contain annotation info for one record
${anno_path} must contains xml file and image file path for annotations
Args:
anno_path (str): root directory for voc annotation data
......@@ -53,11 +52,7 @@ def get_roidb(anno_path,
'cname2id' is a dict to map category name to class id
"""
txt_file = anno_path
part = txt_file.split('ImageSets')
xml_path = os.path.join(part[0], 'Annotations')
assert os.path.isfile(txt_file) and \
os.path.isdir(xml_path), 'invalid xml path'
data_dir = os.path.dirname(anno_path)
records = []
ct = 0
......@@ -67,17 +62,16 @@ def get_roidb(anno_path,
# mapping category name to class id
# background:0, first_class:1, second_class:2, ...
with open(txt_file, 'r') as fr:
with open(anno_path, 'r') as fr:
while True:
line = fr.readline()
if not line:
break
fname = line.strip() + '.xml'
xml_file = os.path.join(xml_path, fname)
img_file, xml_file = [os.path.join(data_dir, x) \
for x in line.strip().split()[:2]]
if not os.path.isfile(xml_file):
continue
tree = ET.parse(xml_file)
im_fname = tree.find('filename').text
if tree.find('id') is None:
im_id = np.array([ct])
else:
......@@ -114,7 +108,7 @@ def get_roidb(anno_path,
is_crowd[i][0] = 0
difficult[i][0] = _difficult
voc_rec = {
'im_file': im_fname,
'im_file': img_file,
'im_id': im_id,
'h': im_h,
'w': im_w,
......@@ -144,8 +138,7 @@ def load(anno_path,
xml directory 'anno_path'
Notes:
${anno_path}/ImageSets/Main/train.txt must contains xml file names for annotations
${anno_path}/Annotations/xxx.xml must contain annotation info for one record
${anno_path} must contains xml file and image file path for annotations
Args:
@anno_path (str): root directory for voc annotation data
......@@ -171,11 +164,7 @@ def load(anno_path,
'cname2id' is a dict to map category name to class id
"""
txt_file = anno_path
part = txt_file.split('ImageSets')
xml_path = os.path.join(part[0], 'Annotations')
assert os.path.isfile(txt_file) and \
os.path.isdir(xml_path), 'invalid xml path'
data_dir = os.path.dirname(anno_path)
# mapping category name to class id
# if with_background is True:
......@@ -186,7 +175,7 @@ def load(anno_path,
ct = 0
cname2cid = {}
if not use_default_label:
label_path = os.path.join(part[0], 'ImageSets/Main/label_list.txt')
label_path = os.path.join(data_dir, 'label_list.txt')
with open(label_path, 'r') as fr:
label_id = int(with_background)
for line in fr.readlines():
......@@ -195,17 +184,16 @@ def load(anno_path,
else:
cname2cid = pascalvoc_label(with_background)
with open(txt_file, 'r') as fr:
with open(anno_path, 'r') as fr:
while True:
line = fr.readline()
if not line:
break
fname = line.strip() + '.xml'
xml_file = os.path.join(xml_path, fname)
img_file, xml_file = [os.path.join(data_dir, x) \
for x in line.strip().split()[:2]]
if not os.path.isfile(xml_file):
continue
tree = ET.parse(xml_file)
im_fname = tree.find('filename').text
if tree.find('id') is None:
im_id = np.array([ct])
else:
......@@ -235,7 +223,7 @@ def load(anno_path,
is_crowd[i][0] = 0
difficult[i][0] = _difficult
voc_rec = {
'im_file': im_fname,
'im_file': img_file,
'im_id': im_id,
'h': im_h,
'w': im_w,
......
ppdet/data/tools/labelme2coco.py → ppdet/data/tools/x2coco.py
浏览文件 @ 23d3745d
......@@ -44,7 +44,7 @@ def getbbox(self, points):
return self.mask2box(mask)
def images(data, num):
def images_labelme(data, num):
image = {}
image['height'] = data['imageHeight']
image['width'] = data['imageWidth']
......@@ -52,6 +52,14 @@ def images(data, num):
image['file_name'] = data['imagePath'].split('/')[-1]
return image
def images_cityscape(data, num, img_file):
image = {}
image['height'] = data['imgHeight']
image['width'] = data['imgWidth']
image['id'] = num + 1
image['file_name'] = img_file
return image
def categories(label, labels_list):
category = {}
......@@ -112,7 +120,7 @@ def get_bbox(height, width, points):
]
def deal_json(img_path, json_path):
def deal_json(ds_type, img_path, json_path):
data_coco = {}
label_to_num = {}
images_list = []
......@@ -120,34 +128,52 @@ def deal_json(img_path, json_path):
annotations_list = []
labels_list = []
image_num = -1
object_num = -1
for img_file in os.listdir(img_path):
img_label = img_file.split('.')[0]
if img_file.split('.')[-1] not in ['bmp', 'jpg', 'jpeg', 'png', 'JPEG', 'JPG', 'PNG']:
continue
label_file = osp.join(json_path, img_label + '.json')
print('Generating dataset from:', label_file)
image_num = image_num + 1
with open(label_file) as f:
data = json.load(f)
images_list.append(images(data, image_num))
object_num = -1
for shapes in data['shapes']:
object_num = object_num + 1
label = shapes['label']
if label not in labels_list:
categories_list.append(categories(label, labels_list))
labels_list.append(label)
label_to_num[label] = len(labels_list)
points = shapes['points']
p_type = shapes['shape_type']
if p_type == 'polygon':
annotations_list.append(
annotations_polygon(data['imageHeight'], data[
'imageWidth'], points, label, image_num, object_num, label_to_num))
if ds_type == 'labelme':
images_list.append(images_labelme(data, image_num))
elif ds_type == 'cityscape':
images_list.append(images_cityscape(data, image_num, img_file))
if ds_type == 'labelme':
for shapes in data['shapes']:
object_num = object_num + 1
label = shapes['label']
if label not in labels_list:
categories_list.append(categories(label, labels_list))
labels_list.append(label)
label_to_num[label] = len(labels_list)
points = shapes['points']
p_type = shapes['shape_type']
if p_type == 'polygon':
annotations_list.append(
annotations_polygon(data['imageHeight'], data[
'imageWidth'], points, label, image_num, object_num, label_to_num))
if p_type == 'rectangle':
points.append([points[0][0], points[1][1]])
points.append([points[1][0], points[0][1]])
if p_type == 'rectangle':
points.append([points[0][0], points[1][1]])
points.append([points[1][0], points[0][1]])
annotations_list.append(
annotations_rectangle(points, label, image_num, object_num, label_to_num))
elif ds_type == 'cityscape':
for shapes in data['objects']:
object_num = object_num + 1
label = shapes['label']
if label not in labels_list:
categories_list.append(categories(label, labels_list))
labels_list.append(label)
label_to_num[label] = len(labels_list)
points = shapes['polygon']
annotations_list.append(
annotations_rectangle(points, label, image_num, object_num, label_to_num))
annotations_polygon(data['imgHeight'], data[
'imgWidth'], points, label, image_num, object_num, label_to_num))
data_coco['images'] = images_list
data_coco['categories'] = categories_list
data_coco['annotations'] = annotations_list
......@@ -157,6 +183,7 @@ def deal_json(img_path, json_path):
def main():
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('--dataset_type', help='the type of dataset')
parser.add_argument('--json_input_dir', help='input annotated directory')
parser.add_argument('--image_input_dir', help='image directory')
parser.add_argument(
......@@ -177,6 +204,11 @@ def main():
type=float,
default=0.0)
args = parser.parse_args()
try:
assert args.dataset_type in ['labelme', 'cityscape']
except AssertionError as e:
print('Now only support the cityscape dataset and labelme dataset!!')
os._exit(0)
try:
assert os.path.exists(args.json_input_dir)
except AssertionError as e:
......@@ -234,7 +266,8 @@ def main():
if not os.path.exists(args.output_dir + '/annotations'):
os.makedirs(args.output_dir + '/annotations')
if args.train_proportion != 0:
train_data_coco = deal_json(args.output_dir + '/train',
train_data_coco = deal_json(args.dataset_type,
args.output_dir + '/train',
args.json_input_dir)
train_json_path = osp.join(args.output_dir + '/annotations',
'instance_train.json')
......
ppdet/utils/download.py
浏览文件 @ 23d3745d
......@@ -25,7 +25,7 @@ import hashlib
import tarfile
import zipfile
from .voc_utils import merge_and_create_list
from .voc_utils import create_list
import logging
logger = logging.getLogger(__name__)
......@@ -59,7 +59,7 @@ DATASETS = {
(
'http://host.robots.ox.ac.uk/pascal/VOC/voc2007/VOCtest_06-Nov-2007.tar',
'b6e924de25625d8de591ea690078ad9f', ),
], ["VOCdevkit/VOC_all"]),
], ["VOCdevkit/VOC2012", "VOCdevkit/VOC2007"]),
'wider_face': ([
(
'https://dataset.bj.bcebos.com/wider_face/WIDER_train.zip',
......@@ -85,7 +85,8 @@ def get_weights_path(url):
"""Get weights path from WEIGHT_HOME, if not exists,
download it from url.
"""
return get_path(url, WEIGHTS_HOME)
path, _ = get_path(url, WEIGHTS_HOME)
return path
def get_dataset_path(path, annotation, image_dir):
......@@ -107,19 +108,26 @@ def get_dataset_path(path, annotation, image_dir):
"{}".format(path, name))
data_dir = osp.join(DATASET_HOME, name)
# For voc, only check merged dir VOC_all
# For voc, only check dir VOCdevkit/VOC2012, VOCdevkit/VOC2007
if name == 'voc':
check_dir = osp.join(data_dir, dataset[1][0])
if osp.exists(check_dir):
logger.info("Found {}".format(check_dir))
exists = True
for sub_dir in dataset[1]:
check_dir = osp.join(data_dir, sub_dir)
if osp.exists(check_dir):
logger.info("Found {}".format(check_dir))
else:
exists = False
if exists:
return data_dir
# voc exist is checked above, voc is not exist here
check_exist = name != 'voc'
for url, md5sum in dataset[0]:
get_path(url, data_dir, md5sum)
get_path(url, data_dir, md5sum, check_exist)
# voc should merge dir and create list after download
# voc should create list after download
if name == 'voc':
_merge_voc_dir(data_dir, dataset[1][0])
create_voc_list(data_dir)
return data_dir
# not match any dataset in DATASETS
......@@ -129,26 +137,17 @@ def get_dataset_path(path, annotation, image_dir):
osp.split(path)[-1]))
def _merge_voc_dir(data_dir, output_subdir):
logger.info("Download voc dataset successed, merge "
"VOC2007 and VOC2012 to VOC_all...")
output_dir = osp.join(data_dir, output_subdir)
devkit_dir = "/".join(output_dir.split('/')[:-1])
def create_voc_list(data_dir, devkit_subdir='VOCdevkit'):
logger.info("Create voc file list...")
devkit_dir = osp.join(data_dir, devkit_subdir)
years = ['2007', '2012']
# merge dir in output_tmp_dir at first, move to
# output_dir after merge sucessed.
output_tmp_dir = osp.join(data_dir, 'tmp')
if osp.isdir(output_tmp_dir):
shutil.rmtree(output_tmp_dir)
# NOTE: since using auto download VOC
# dataset, VOC default label list should be used,
# do not generate label_list.txt here. For default
# label, see ../data/source/voc_loader.py
merge_and_create_list(devkit_dir, years, output_tmp_dir)
shutil.move(output_tmp_dir, output_dir)
# remove source directory VOC2007 and VOC2012
shutil.rmtree(osp.join(devkit_dir, "VOC2007"))
shutil.rmtree(osp.join(devkit_dir, "VOC2012"))
create_list(devkit_dir, years, data_dir)
logger.info("Create voc file list finished")
def map_path(url, root_dir):
......@@ -161,7 +160,7 @@ def map_path(url, root_dir):
return osp.join(root_dir, fpath)
def get_path(url, root_dir, md5sum=None):
def get_path(url, root_dir, md5sum=None, check_exist=True):
""" Download from given url to root_dir.
if file or directory specified by url is exists under
root_dir, return the path directly, otherwise download
......@@ -178,20 +177,25 @@ def get_path(url, root_dir, md5sum=None):
# For same zip file, decompressed directory name different
# from zip file name, rename by following map
decompress_name_map = {
"VOC": "VOCdevkit/VOC_all",
"VOCtrainval_11-May-2012": "VOCdevkit/VOC2012",
"VOCtrainval_06-Nov-2007": "VOCdevkit/VOC2007",
"VOCtest_06-Nov-2007": "VOCdevkit/VOC2007",
"annotations_trainval": "annotations"
}
for k, v in decompress_name_map.items():
if fullpath.find(k) >= 0:
fullpath = '/'.join(fullpath.split('/')[:-1] + [v])
if osp.exists(fullpath):
exist_flag = False
if osp.exists(fullpath) and check_exist:
exist_flag = True
logger.info("Found {}".format(fullpath))
else:
exist_flag = False
fullname = _download(url, root_dir, md5sum)
_decompress(fullname)
return fullpath
return fullpath, exist_flag
def download_dataset(path, dataset=None):
......@@ -201,9 +205,7 @@ def download_dataset(path, dataset=None):
return
dataset_info = DATASETS[dataset][0]
for info in dataset_info:
get_path(info[0], path, info[1])
if dataset == 'voc':
_merge_voc_dir(path, DATASETS[dataset][1][0])
get_path(info[0], path, info[1], False)
logger.info("Download dataset {} finished.".format(dataset))
......
ppdet/utils/voc_utils.py
浏览文件 @ 23d3745d
......@@ -22,20 +22,15 @@ import re
import random
import shutil
__all__ = ['merge_and_create_list']
__all__ = ['create_list']
def merge_and_create_list(devkit_dir, years, output_dir):
def create_list(devkit_dir, years, output_dir):
"""
Merge VOC2007 and VOC2012 to output_dir and create following list:
1. train.txt
2. val.txt
3. test.txt
create following list:
1. trainval.txt
2. test.txt
"""
os.makedirs(osp.join(output_dir, 'Annotations/'))
os.makedirs(osp.join(output_dir, 'ImageSets/Main/'))
os.makedirs(osp.join(output_dir, 'JPEGImages/'))
trainval_list = []
test_list = []
for year in years:
......@@ -43,20 +38,16 @@ def merge_and_create_list(devkit_dir, years, output_dir):
trainval_list.extend(trainval)
test_list.extend(test)
main_dir = osp.join(output_dir, 'ImageSets/Main/')
random.shuffle(trainval_list)
with open(osp.join(main_dir, 'train.txt'), 'w') as ftrainval:
with open(osp.join(output_dir, 'trainval.txt'), 'w') as ftrainval:
for item in trainval_list:
ftrainval.write(item + '\n')
ftrainval.write(item[0] + ' ' + item[1] + '\n')
with open(osp.join(main_dir, 'val.txt'), 'w') as fval:
with open(osp.join(main_dir, 'test.txt'), 'w') as ftest:
ct = 0
for item in test_list:
ct += 1
fval.write(item + '\n')
if ct <= 1000:
ftest.write(item + '\n')
with open(osp.join(output_dir, 'test.txt'), 'w') as fval:
ct = 0
for item in test_list:
ct += 1
fval.write(item[0] + ' ' + item[1] + '\n')
def _get_voc_dir(devkit_dir, year, type):
......@@ -86,14 +77,10 @@ def _walk_voc_dir(devkit_dir, year, output_dir):
if name_prefix in added:
continue
added.add(name_prefix)
ann_path = osp.join(annotation_dir, name_prefix + '.xml')
img_path = osp.join(img_dir, name_prefix + '.jpg')
new_ann_path = osp.join(output_dir, 'Annotations/',
name_prefix + '.xml')
new_img_path = osp.join(output_dir, 'JPEGImages/',
name_prefix + '.jpg')
shutil.copy(ann_path, new_ann_path)
shutil.copy(img_path, new_img_path)
img_ann_list.append(name_prefix)
ann_path = osp.join(osp.relpath(annotation_dir, output_dir),
name_prefix + '.xml')
img_path = osp.join(osp.relpath(img_dir, output_dir),
name_prefix + '.jpg')
img_ann_list.append((img_path, ann_path))
return trainval_list, test_list
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或