diff --git a/new_tutorials/train/README.md b/new_tutorials/train/README.md new file mode 100644 index 0000000000000000000000000000000000000000..1900143bceb3435da8ffa04a7fed7b0205e04477 --- /dev/null +++ b/new_tutorials/train/README.md @@ -0,0 +1,18 @@ +# 使用教程——训练模型 + +本目录下整理了使用PaddleX训练模型的示例代码,代码中均提供了示例数据的自动下载,并均使用单张GPU卡进行训练。 + +|代码 | 模型任务 | 数据 | +|------|--------|---------| +|classification/mobilenetv2.py | 图像分类MobileNetV2 | 蔬菜分类 | +|classification/resnet50.py | 图像分类ResNet50 | 蔬菜分类 | +|detection/faster_rcnn_r50_fpn.py | 目标检测FasterRCNN | 昆虫检测 | +|detection/mask_rcnn_f50_fpn.py | 实例分割MaskRCNN | 垃圾分拣 | +|segmentation/deeplabv3p.py | 语义分割DeepLabV3| 视盘分割 | +|segmentation/unet.py | 语义分割UNet | 视盘分割 | + +## 开始训练 +在安装PaddleX后,使用如下命令开始训练 +``` +python classification/mobilenetv2.py +``` diff --git a/new_tutorials/train/classification/mobilenetv2.py b/new_tutorials/train/classification/mobilenetv2.py new file mode 100644 index 0000000000000000000000000000000000000000..3f637125b760de6d992d6a062e4d456bf5038426 --- /dev/null +++ b/new_tutorials/train/classification/mobilenetv2.py @@ -0,0 +1,51 @@ +import os +# 选择使用0号卡 +os.environ['CUDA_VISIBLE_DEVICES'] = '0' + +from paddlex.cls import transforms +import paddlex as pdx + +# 下载和解压蔬菜分类数据集 +veg_dataset = 'https://bj.bcebos.com/paddlex/datasets/vegetables_cls.tar.gz' +pdx.utils.download_and_decompress(veg_dataset, path='./') + +# 定义训练和验证时的transforms +train_transforms = transforms.Compose([ + transforms.RandomCrop(crop_size=224), + transforms.RandomHorizontalFlip(), + transforms.Normalize() +]) +eval_transforms = transforms.Compose([ + transforms.ResizeByShort(short_size=256), + transforms.CenterCrop(crop_size=224), + transforms.Normalize() +]) + +# 定义训练和验证所用的数据集 +train_dataset = pdx.datasets.ImageNet( + data_dir='vegetables_cls', + file_list='vegetables_cls/train_list.txt', + label_list='vegetables_cls/labels.txt', + transforms=train_transforms, + shuffle=True) +eval_dataset = pdx.datasets.ImageNet( + data_dir='vegetables_cls', + file_list='vegetables_cls/val_list.txt', + label_list='vegetables_cls/labels.txt', + transforms=eval_transforms) + +# 初始化模型,并进行训练 +# 可使用VisualDL查看训练指标 +# VisualDL启动方式: visualdl --logdir output/mobilenetv2/vdl_log --port 8001 +# 浏览器打开 https://0.0.0.0:8001即可 +# 其中0.0.0.0为本机访问,如为远程服务, 改成相应机器IP +model = pdx.cls.MobileNetV2(num_classes=len(train_dataset.labels)) +model.train( + num_epochs=10, + train_dataset=train_dataset, + train_batch_size=32, + eval_dataset=eval_dataset, + lr_decay_epochs=[4, 6, 8], + learning_rate=0.025, + save_dir='output/mobilenetv2', + use_vdl=True) diff --git a/new_tutorials/train/classification/resnet50.py b/new_tutorials/train/classification/resnet50.py new file mode 100644 index 0000000000000000000000000000000000000000..2e5a9b4820c7e66a83abaca0b13e057b15ceb830 --- /dev/null +++ b/new_tutorials/train/classification/resnet50.py @@ -0,0 +1,58 @@ +import os +# 选择使用0号卡 +os.environ['CUDA_VISIBLE_DEVICES'] = '0' + +import paddle.fluid as fluid +from paddlex.cls import transforms +import paddlex as pdx + +# 下载和解压蔬菜分类数据集 +veg_dataset = 'https://bj.bcebos.com/paddlex/datasets/vegetables_cls.tar.gz' +pdx.utils.download_and_decompress(veg_dataset, path='./') + +# 定义训练和验证时的transforms +train_transforms = transforms.Compose( + [transforms.RandomCrop(crop_size=224), + transforms.Normalize()]) +eval_transforms = transforms.Compose([ + transforms.ResizeByShort(short_size=256), + transforms.CenterCrop(crop_size=224), + transforms.Normalize() +]) + +# 定义训练和验证所用的数据集 +train_dataset = pdx.datasets.ImageNet( + data_dir='vegetables_cls', + file_list='vegetables_cls/train_list.txt', + label_list='vegetables_cls/labels.txt', + transforms=train_transforms, + shuffle=True) +eval_dataset = pdx.datasets.ImageNet( + data_dir='vegetables_cls', + file_list='vegetables_cls/val_list.txt', + label_list='vegetables_cls/labels.txt', + transforms=eval_transforms) + +# PaddleX支持自定义构建优化器 +step_each_epoch = train_dataset.num_samples // 32 +learning_rate = fluid.layers.cosine_decay( + learning_rate=0.025, step_each_epoch=step_each_epoch, epochs=10) +optimizer = fluid.optimizer.Momentum( + learning_rate=learning_rate, + momentum=0.9, + regularization=fluid.regularizer.L2Decay(4e-5)) + +# 初始化模型,并进行训练 +# 可使用VisualDL查看训练指标 +# VisualDL启动方式: visualdl --logdir output/resnet50/vdl_log --port 8001 +# 浏览器打开 https://0.0.0.0:8001即可 +# 其中0.0.0.0为本机访问,如为远程服务, 改成相应机器IP +model = pdx.cls.ResNet50(num_classes=len(train_dataset.labels)) +model.train( + num_epochs=10, + train_dataset=train_dataset, + train_batch_size=32, + eval_dataset=eval_dataset, + optimizer=optimizer, + save_dir='output/resnet50', + use_vdl=True) diff --git a/new_tutorials/train/detection/faster_rcnn_r50_fpn.py b/new_tutorials/train/detection/faster_rcnn_r50_fpn.py new file mode 100644 index 0000000000000000000000000000000000000000..cbe6dabe535b5972418349ac31576b344652e69d --- /dev/null +++ b/new_tutorials/train/detection/faster_rcnn_r50_fpn.py @@ -0,0 +1,55 @@ +import os +# 选择使用0号卡 +os.environ['CUDA_VISIBLE_DEVICES'] = '0' + +from paddlex.det import transforms +import paddlex as pdx + +# 下载和解压昆虫检测数据集 +insect_dataset = 'https://bj.bcebos.com/paddlex/datasets/insect_det.tar.gz' +pdx.utils.download_and_decompress(insect_dataset, path='./') + +# 定义训练和验证时的transforms +train_transforms = transforms.Compose([ + transforms.RandomHorizontalFlip(), + transforms.Normalize(), + transforms.ResizeByShort(short_size=800, max_size=1333), + transforms.Padding(coarsest_stride=32) +]) + +eval_transforms = transforms.Compose([ + transforms.Normalize(), + transforms.ResizeByShort(short_size=800, max_size=1333), + transforms.Padding(coarsest_stride=32), +]) + +# 定义训练和验证所用的数据集 +train_dataset = pdx.datasets.VOCDetection( + data_dir='insect_det', + file_list='insect_det/train_list.txt', + label_list='insect_det/labels.txt', + transforms=train_transforms, + shuffle=True) +eval_dataset = pdx.datasets.VOCDetection( + data_dir='insect_det', + file_list='insect_det/val_list.txt', + label_list='insect_det/labels.txt', + transforms=eval_transforms) + +# 初始化模型,并进行训练 +# 可使用VisualDL查看训练指标 +# VisualDL启动方式: visualdl --logdir output/faster_rcnn_r50_fpn/vdl_log --port 8001 +# 浏览器打开 https://0.0.0.0:8001即可 +# 其中0.0.0.0为本机访问,如为远程服务, 改成相应机器IP +# num_classes 需要设置为包含背景类的类别数,即: 目标类别数量 + 1 +num_classes = len(train_dataset.labels) + 1 +model = pdx.det.FasterRCNN(num_classes=num_classes) +model.train( + num_epochs=12, + train_dataset=train_dataset, + train_batch_size=2, + eval_dataset=eval_dataset, + learning_rate=0.0025, + lr_decay_epochs=[8, 11], + save_dir='output/faster_rcnn_r50_fpn', + use_vdl=True) diff --git a/new_tutorials/train/detection/mask_rcnn_r50_fpn.py b/new_tutorials/train/detection/mask_rcnn_r50_fpn.py new file mode 100644 index 0000000000000000000000000000000000000000..15a6b840528fe7948c80f4cf605498cf55b5c918 --- /dev/null +++ b/new_tutorials/train/detection/mask_rcnn_r50_fpn.py @@ -0,0 +1,54 @@ +import os +# 选择使用0号卡 +os.environ['CUDA_VISIBLE_DEVICES'] = '0' + +from paddlex.det import transforms +import paddlex as pdx + +# 下载和解压小度熊分拣数据集 +xiaoduxiong_dataset = 'https://bj.bcebos.com/paddlex/datasets/xiaoduxiong_ins_det.tar.gz' +pdx.utils.download_and_decompress(xiaoduxiong_dataset, path='./') + +# 定义训练和验证时的transforms +train_transforms = transforms.Compose([ + transforms.RandomHorizontalFlip(), + transforms.Normalize(), + transforms.ResizeByShort(short_size=800, max_size=1333), + transforms.Padding(coarsest_stride=32) +]) + +eval_transforms = transforms.Compose([ + transforms.Normalize(), + transforms.ResizeByShort(short_size=800, max_size=1333), + transforms.Padding(coarsest_stride=32) +]) + +# 定义训练和验证所用的数据集 +train_dataset = pdx.datasets.CocoDetection( + data_dir='xiaoduxiong_ins_det/JPEGImages', + ann_file='xiaoduxiong_ins_det/train.json', + transforms=train_transforms, + shuffle=True) +eval_dataset = pdx.datasets.CocoDetection( + data_dir='xiaoduxiong_ins_det/JPEGImages', + ann_file='xiaoduxiong_ins_det/val.json', + transforms=eval_transforms) + +# 初始化模型,并进行训练 +# 可使用VisualDL查看训练指标 +# VisualDL启动方式: visualdl --logdir output/mask_rcnn_r50_fpn/vdl_log --port 8001 +# 浏览器打开 https://0.0.0.0:8001即可 +# 其中0.0.0.0为本机访问,如为远程服务, 改成相应机器IP +# num_classes 需要设置为包含背景类的类别数,即: 目标类别数量 + 1 +num_classes = len(train_dataset.labels) + 1 +model = pdx.det.MaskRCNN(num_classes=num_classes) +model.train( + num_epochs=12, + train_dataset=train_dataset, + train_batch_size=1, + eval_dataset=eval_dataset, + learning_rate=0.00125, + warmup_steps=10, + lr_decay_epochs=[8, 11], + save_dir='output/mask_rcnn_r50_fpn', + use_vdl=True) diff --git a/new_tutorials/train/detection/yolov3_darknet53.py b/new_tutorials/train/detection/yolov3_darknet53.py new file mode 100644 index 0000000000000000000000000000000000000000..c38656b04e9a35cd033dc583811c58aa8baafba2 --- /dev/null +++ b/new_tutorials/train/detection/yolov3_darknet53.py @@ -0,0 +1,56 @@ +import os +# 选择使用0号卡 +os.environ['CUDA_VISIBLE_DEVICES'] = '0' + +from paddlex.det import transforms +import paddlex as pdx + +# 下载和解压昆虫检测数据集 +insect_dataset = 'https://bj.bcebos.com/paddlex/datasets/insect_det.tar.gz' +pdx.utils.download_and_decompress(insect_dataset, path='./') + +# 定义训练和验证时的transforms +train_transforms = transforms.Compose([ + transforms.MixupImage(mixup_epoch=250), + transforms.RandomDistort(), + transforms.RandomExpand(), + transforms.RandomCrop(), + transforms.Resize(target_size=608, interp='RANDOM'), + transforms.RandomHorizontalFlip(), + transforms.Normalize(), +]) + +eval_transforms = transforms.Compose([ + transforms.Resize(target_size=608, interp='CUBIC'), + transforms.Normalize(), +]) + +# 定义训练和验证所用的数据集 +train_dataset = pdx.datasets.VOCDetection( + data_dir='insect_det', + file_list='insect_det/train_list.txt', + label_list='insect_det/labels.txt', + transforms=train_transforms, + shuffle=True) +eval_dataset = pdx.datasets.VOCDetection( + data_dir='insect_det', + file_list='insect_det/val_list.txt', + label_list='insect_det/labels.txt', + transforms=eval_transforms) + +# 初始化模型,并进行训练 +# 可使用VisualDL查看训练指标 +# VisualDL启动方式: visualdl --logdir output/yolov3_darknet/vdl_log --port 8001 +# 浏览器打开 https://0.0.0.0:8001即可 +# 其中0.0.0.0为本机访问,如为远程服务, 改成相应机器IP +num_classes = len(train_dataset.labels) +model = pdx.det.YOLOv3(num_classes=num_classes, backbone='DarkNet53') +model.train( + num_epochs=270, + train_dataset=train_dataset, + train_batch_size=8, + eval_dataset=eval_dataset, + learning_rate=0.000125, + lr_decay_epochs=[210, 240], + save_dir='output/yolov3_darknet53', + use_vdl=True) diff --git a/new_tutorials/train/segmentation/deeplabv3p.py b/new_tutorials/train/segmentation/deeplabv3p.py new file mode 100644 index 0000000000000000000000000000000000000000..346a229a358a76830112acfd596740c070822874 --- /dev/null +++ b/new_tutorials/train/segmentation/deeplabv3p.py @@ -0,0 +1,50 @@ +import os +# 选择使用0号卡 +os.environ['CUDA_VISIBLE_DEVICES'] = '0' + +import paddlex as pdx +from paddlex.seg import transforms + +# 下载和解压视盘分割数据集 +optic_dataset = 'https://bj.bcebos.com/paddlex/datasets/optic_disc_seg.tar.gz' +pdx.utils.download_and_decompress(optic_dataset, path='./') + +# 定义训练和验证时的transforms +train_transforms = transforms.Compose([ + transforms.RandomHorizontalFlip(), + transforms.Resize(target_size=512), + transforms.RandomPaddingCrop(crop_size=500), + transforms.Normalize() +]) + +eval_transforms = transforms.Compose( + [transforms.Resize(512), transforms.Normalize()]) + +# 定义训练和验证所用的数据集 +train_dataset = pdx.datasets.SegDataset( + data_dir='optic_disc_seg', + file_list='optic_disc_seg/train_list.txt', + label_list='optic_disc_seg/labels.txt', + transforms=train_transforms, + shuffle=True) +eval_dataset = pdx.datasets.SegDataset( + data_dir='optic_disc_seg', + file_list='optic_disc_seg/val_list.txt', + label_list='optic_disc_seg/labels.txt', + transforms=eval_transforms) + +# 初始化模型,并进行训练 +# 可使用VisualDL查看训练指标 +# VisualDL启动方式: visualdl --logdir output/deeplab/vdl_log --port 8001 +# 浏览器打开 https://0.0.0.0:8001即可 +# 其中0.0.0.0为本机访问,如为远程服务, 改成相应机器IP +num_classes = len(train_dataset.labels) +model = pdx.seg.DeepLabv3p(num_classes=num_classes) +model.train( + num_epochs=40, + train_dataset=train_dataset, + train_batch_size=4, + eval_dataset=eval_dataset, + learning_rate=0.01, + save_dir='output/deeplab', + use_vdl=True) diff --git a/new_tutorials/train/segmentation/hrnet.py b/new_tutorials/train/segmentation/hrnet.py new file mode 100644 index 0000000000000000000000000000000000000000..f887b78c3ae16ae66235f1965ada8bd2355d62c6 --- /dev/null +++ b/new_tutorials/train/segmentation/hrnet.py @@ -0,0 +1,50 @@ +import os +# 选择使用0号卡 +os.environ['CUDA_VISIBLE_DEVICES'] = '0' + +import paddlex as pdx +from paddlex.seg import transforms + +# 下载和解压视盘分割数据集 +optic_dataset = 'https://bj.bcebos.com/paddlex/datasets/optic_disc_seg.tar.gz' +pdx.utils.download_and_decompress(optic_dataset, path='./') + +# 定义训练和验证时的transforms +train_transforms = transforms.Compose([ + transforms.RandomHorizontalFlip(), transforms.ResizeRangeScaling(), + transforms.RandomPaddingCrop(crop_size=512), transforms.Normalize() +]) + +eval_transforms = transforms.Compose([ + transforms.ResizeByLong(long_size=512), + transforms.Padding(target_size=512), transforms.Normalize() +]) + +# 定义训练和验证所用的数据集 +train_dataset = pdx.datasets.SegDataset( + data_dir='optic_disc_seg', + file_list='optic_disc_seg/train_list.txt', + label_list='optic_disc_seg/labels.txt', + transforms=train_transforms, + shuffle=True) +eval_dataset = pdx.datasets.SegDataset( + data_dir='optic_disc_seg', + file_list='optic_disc_seg/val_list.txt', + label_list='optic_disc_seg/labels.txt', + transforms=eval_transforms) + +# 初始化模型,并进行训练 +# 可使用VisualDL查看训练指标 +# VisualDL启动方式: visualdl --logdir output/unet/vdl_log --port 8001 +# 浏览器打开 https://0.0.0.0:8001即可 +# 其中0.0.0.0为本机访问,如为远程服务, 改成相应机器IP +num_classes = len(train_dataset.labels) +model = pdx.seg.HRNet(num_classes=num_classes) +model.train( + num_epochs=20, + train_dataset=train_dataset, + train_batch_size=4, + eval_dataset=eval_dataset, + learning_rate=0.01, + save_dir='output/hrnet', + use_vdl=True) diff --git a/new_tutorials/train/segmentation/unet.py b/new_tutorials/train/segmentation/unet.py new file mode 100644 index 0000000000000000000000000000000000000000..a683af98322eacb9d0775b3a5256d900f5743bb2 --- /dev/null +++ b/new_tutorials/train/segmentation/unet.py @@ -0,0 +1,53 @@ +import os +# 选择使用0号卡 +os.environ['CUDA_VISIBLE_DEVICES'] = '0' + +import paddlex as pdx +from paddlex.seg import transforms + +# 下载和解压视盘分割数据集 +optic_dataset = 'https://bj.bcebos.com/paddlex/datasets/optic_disc_seg.tar.gz' +pdx.utils.download_and_decompress(optic_dataset, path='./') + +# 定义训练和验证时的transforms +train_transforms = transforms.Compose([ + transforms.RandomHorizontalFlip(), + transforms.ResizeRangeScaling(), + transforms.RandomPaddingCrop(crop_size=512), + transforms.Normalize() +]) + +eval_transforms = transforms.Compose([ + transforms.ResizeByLong(long_size=512), + transforms.Padding(target_size=512), + transforms.Normalize() +]) + +# 定义训练和验证所用的数据集 +train_dataset = pdx.datasets.SegDataset( + data_dir='optic_disc_seg', + file_list='optic_disc_seg/train_list.txt', + label_list='optic_disc_seg/labels.txt', + transforms=train_transforms, + shuffle=True) +eval_dataset = pdx.datasets.SegDataset( + data_dir='optic_disc_seg', + file_list='optic_disc_seg/val_list.txt', + label_list='optic_disc_seg/labels.txt', + transforms=eval_transforms) + +# 初始化模型,并进行训练 +# 可使用VisualDL查看训练指标 +# VisualDL启动方式: visualdl --logdir output/unet/vdl_log --port 8001 +# 浏览器打开 https://0.0.0.0:8001即可 +# 其中0.0.0.0为本机访问,如为远程服务, 改成相应机器IP +num_classes = len(train_dataset.labels) +model = pdx.seg.UNet(num_classes=num_classes) +model.train( + num_epochs=20, + train_dataset=train_dataset, + train_batch_size=4, + eval_dataset=eval_dataset, + learning_rate=0.01, + save_dir='output/unet', + use_vdl=True) diff --git a/paddlex/__init__.py b/paddlex/__init__.py index 972210bdb80c445e59d4a8ed10418ee988bd353c..d1656161a0d764c0a7fbd125f246d0e43125bcda 100644 --- a/paddlex/__init__.py +++ b/paddlex/__init__.py @@ -53,4 +53,4 @@ log_level = 2 from . import interpret -__version__ = '1.0.4' +__version__ = '1.0.5' diff --git a/paddlex/cv/transforms/cls_transforms.py b/paddlex/cv/transforms/cls_transforms.py index 3b2ff789f280d1006c55bf8a0ed8ab4e5ebd5967..dbcd34222daf71c05c8f26a2a38c94faacb526f2 100644 --- a/paddlex/cv/transforms/cls_transforms.py +++ b/paddlex/cv/transforms/cls_transforms.py @@ -18,6 +18,7 @@ import random import os.path as osp import numpy as np from PIL import Image, ImageEnhance +import paddlex.utils.logging as logging class ClsTransform: @@ -96,6 +97,10 @@ class Compose(ClsTransform): if not isinstance(augmenters, list): raise Exception( "augmenters should be list type in func add_augmenters()") + transform_names = [type(x).__name__ for x in self.transforms] + for aug in augmenters: + if type(aug).__name__ in transform_names: + logging.error("{} is already in ComposedTransforms, need to remove it from add_augmenters().".format(type(aug).__name__)) self.transforms = augmenters + self.transforms diff --git a/paddlex/cv/transforms/det_transforms.py b/paddlex/cv/transforms/det_transforms.py index 841a980ecd621cc8d1ac494f41e8bebf1d40c8f7..6ec3570c192fd3fa6bc174a9f601b250c3b2b651 100644 --- a/paddlex/cv/transforms/det_transforms.py +++ b/paddlex/cv/transforms/det_transforms.py @@ -27,6 +27,7 @@ from PIL import Image, ImageEnhance from .imgaug_support import execute_imgaug from .ops import * from .box_utils import * +import paddlex.utils.logging as logging class DetTransform: @@ -156,6 +157,10 @@ class Compose(DetTransform): if not isinstance(augmenters, list): raise Exception( "augmenters should be list type in func add_augmenters()") + transform_names = [type(x).__name__ for x in self.transforms] + for aug in augmenters: + if type(aug).__name__ in transform_names: + logging.error("{} is already in ComposedTransforms, need to remove it from add_augmenters().".format(type(aug).__name__)) self.transforms = augmenters + self.transforms diff --git a/paddlex/cv/transforms/seg_transforms.py b/paddlex/cv/transforms/seg_transforms.py index 34ba681b5be197549ce680e8bd762e2cab06f00b..9ea1c3bdc2159dbc1f33ac5f15dc710e12ccb83c 100644 --- a/paddlex/cv/transforms/seg_transforms.py +++ b/paddlex/cv/transforms/seg_transforms.py @@ -21,6 +21,7 @@ import numpy as np from PIL import Image import cv2 from collections import OrderedDict +import paddlex.utils.logging as logging class SegTransform: @@ -112,6 +113,10 @@ class Compose(SegTransform): if not isinstance(augmenters, list): raise Exception( "augmenters should be list type in func add_augmenters()") + transform_names = [type(x).__name__ for x in self.transforms] + for aug in augmenters: + if type(aug).__name__ in transform_names: + logging.error("{} is already in ComposedTransforms, need to remove it from add_augmenters().".format(type(aug).__name__)) self.transforms = augmenters + self.transforms diff --git a/paddlex/utils/logging.py b/paddlex/utils/logging.py index e5deb7388459f1052fd90d758d09aad759592ec8..c118a28782528b727bf9af4591d07714cddae6ae 100644 --- a/paddlex/utils/logging.py +++ b/paddlex/utils/logging.py @@ -47,9 +47,10 @@ def info(message="", use_color=False): log(level=2, message=message, use_color=use_color) -def warning(message="", use_color=False): +def warning(message="", use_color=True): log(level=1, message=message, use_color=use_color) -def error(message="", use_color=False): +def error(message="", use_color=True): log(level=0, message=message, use_color=use_color) + sys.exit(-1) diff --git a/setup.py b/setup.py index bba199719ce65075f8a61b965bca49f026406c91..db62ca5e9e8107f2f32e804a0e92fb48766d3c27 100644 --- a/setup.py +++ b/setup.py @@ -19,7 +19,7 @@ long_description = "PaddleX. A end-to-end deeplearning model development toolkit setuptools.setup( name="paddlex", - version='1.0.4', + version='1.0.5', author="paddlex", author_email="paddlex@baidu.com", description=long_description,