Skip to content

M
models

PaddlePaddle / models
大约 2 年 前同步成功

通知 232
Star 6828
Fork 2962
M
models
提交 2bb4e4a6 编写于 作者: Y Yancey1989
浏览文件
操作

polish code

上级 2e110d79
隐藏空白更改
内联 并排
Showing with 173 addition and 236 deletion
fluid/PaddleCV/image_classification/fast_resnet/torchvision_reader.py
浏览文件 @ 2bb4e4a6
...@@ -3,33 +3,30 @@ import os ...@@ -3,33 +3,30 @@ import os
import numpy as np import numpy as np
import math import math
import random import random
import torchvision import torch
import torch.utils.data
from torch.utils.data.distributed import DistributedSampler
import torchvision.transforms as transforms import torchvision.transforms as transforms
import torchvision.datasets as datasets import torchvision.datasets as datasets
from torch.utils.data.sampler import Sampler
import torchvision
import pickle import pickle
from tqdm import tqdm from tqdm import tqdm
import time import time
import multiprocessing import multiprocessing
TRAINER_NUMS = int(os.getenv("PADDLE_TRAINERS_NUM", "1"))
TRAINER_ID = int(os.getenv("PADDLE_TRAINER_ID", "0"))
FINISH_EVENT = "FINISH_EVENT" FINISH_EVENT = "FINISH_EVENT"
class PaddleDataLoader(object): class PaddleDataLoader(object):
def __init__(self, torch_dataset, indices=None, concurrent=16, queue_size=3072, shuffle=True, batch_size=224, is_distributed=True): def __init__(self, torch_dataset, indices=None, concurrent=24, queue_size=3072, shuffle=True):
self.torch_dataset = torch_dataset self.torch_dataset = torch_dataset
self.data_queue = multiprocessing.Queue(queue_size) self.data_queue = multiprocessing.Queue(queue_size)
self.indices = indices self.indices = indices
self.concurrent = concurrent self.concurrent = concurrent
self.shuffle_seed = 0
self.shuffle = shuffle self.shuffle = shuffle
self.is_distributed = is_distributed
self.batch_size = batch_size
def _worker_loop(self, dataset, worker_indices, worker_id): def _worker_loop(self, dataset, worker_indices, worker_id):
cnt = 0 cnt = 0
print("worker [%d], len: [%d], indices: [%s]"%(worker_id, len(worker_indices), worker_indices[:10]))
for idx in worker_indices: for idx in worker_indices:
cnt += 1 cnt += 1
img, label = self.torch_dataset[idx] img, label = self.torch_dataset[idx]
...@@ -43,28 +40,21 @@ class PaddleDataLoader(object): ...@@ -43,28 +40,21 @@ class PaddleDataLoader(object):
worker_processes = [] worker_processes = []
total_img = len(self.torch_dataset) total_img = len(self.torch_dataset)
print("total image: ", total_img) print("total image: ", total_img)
if self.indices is None: #if self.indices is None:
self.indices = [i for i in xrange(total_img)]
if self.shuffle: if self.shuffle:
random.seed(self.shuffle_seed) self.indices = [i for i in xrange(total_img)]
random.seed(time.time())
random.shuffle(self.indices) random.shuffle(self.indices)
worker_indices = self.indices print("shuffle indices: %s ..." % self.indices[:10])
if self.is_distributed:
cnt_per_node = len(self.indices) / TRAINER_NUMS imgs_per_worker = int(math.ceil(total_img / self.concurrent))
offset = TRAINER_ID * cnt_per_node
worker_indices = self.indices[offset: (offset + cnt_per_node)]
if len(worker_indices) % self.batch_size != 0:
worker_indices += worker_indices[-(self.batch_size - (len(worker_indices) % self.batch_size)):]
print("shuffle: [%d], shuffle seed: [%d], worker indices len: [%d], %s" % (self.shuffle, self.shuffle_seed, len(worker_indices), worker_indices[:10]))
cnt_per_thread = int(math.ceil(len(worker_indices) / self.concurrent))
for i in xrange(self.concurrent): for i in xrange(self.concurrent):
offset = i * cnt_per_thread start = i * imgs_per_worker
thread_incides = worker_indices[offset: (offset + cnt_per_thread)] end = (i + 1) * imgs_per_worker if i != self.concurrent - 1 else None
print("loader thread: [%d] start idx: [%d], end idx: [%d], len: [%d]" % (i, offset, (offset + cnt_per_thread), len(thread_incides))) sliced_indices = self.indices[start:end]
w = multiprocessing.Process( w = multiprocessing.Process(
target=self._worker_loop, target=self._worker_loop,
args=(self.torch_dataset, thread_incides, i) args=(self.torch_dataset, sliced_indices, i)
) )
w.daemon = True w.daemon = True
w.start() w.start()
...@@ -80,13 +70,13 @@ class PaddleDataLoader(object): ...@@ -80,13 +70,13 @@ class PaddleDataLoader(object):
return _reader_creator return _reader_creator
def train(traindir, bs, sz, min_scale=0.08): def train(traindir, sz, min_scale=0.08):
train_tfms = [ train_tfms = [
transforms.RandomResizedCrop(sz, scale=(min_scale, 1.0)), transforms.RandomResizedCrop(sz, scale=(min_scale, 1.0)),
transforms.RandomHorizontalFlip() transforms.RandomHorizontalFlip()
] ]
train_dataset = datasets.ImageFolder(traindir, transforms.Compose(train_tfms)) train_dataset = datasets.ImageFolder(traindir, transforms.Compose(train_tfms))
return PaddleDataLoader(train_dataset, batch_size=bs) return PaddleDataLoader(train_dataset).reader()
def test(valdir, bs, sz, rect_val=False): def test(valdir, bs, sz, rect_val=False):
if rect_val: if rect_val:
...@@ -96,12 +86,12 @@ def test(valdir, bs, sz, rect_val=False): ...@@ -96,12 +86,12 @@ def test(valdir, bs, sz, rect_val=False):
ar_tfms = [transforms.Resize(int(sz* 1.14)), CropArTfm(idx2ar, sz)] ar_tfms = [transforms.Resize(int(sz* 1.14)), CropArTfm(idx2ar, sz)]
val_dataset = ValDataset(valdir, transform=ar_tfms) val_dataset = ValDataset(valdir, transform=ar_tfms)
return PaddleDataLoader(val_dataset, concurrent=1, indices=idx_sorted, shuffle=False, is_distributed=False) return PaddleDataLoader(val_dataset, concurrent=1, indices=idx_sorted, shuffle=False).reader()
val_tfms = [transforms.Resize(int(sz* 1.14)), transforms.CenterCrop(sz)] val_tfms = [transforms.Resize(int(sz* 1.14)), transforms.CenterCrop(sz)]
val_dataset = datasets.ImageFolder(valdir, transforms.Compose(val_tfms)) val_dataset = datasets.ImageFolder(valdir, transforms.Compose(val_tfms))
return PaddleDataLoader(val_dataset, is_distributed=False) return PaddleDataLoader(val_dataset).reader()
class ValDataset(datasets.ImageFolder): class ValDataset(datasets.ImageFolder):
...@@ -122,6 +112,7 @@ class ValDataset(datasets.ImageFolder): ...@@ -122,6 +112,7 @@ class ValDataset(datasets.ImageFolder):
return sample, target return sample, target
class CropArTfm(object): class CropArTfm(object):
def __init__(self, idx2ar, target_size): def __init__(self, idx2ar, target_size):
self.idx2ar, self.target_size = idx2ar, target_size self.idx2ar, self.target_size = idx2ar, target_size
...@@ -134,7 +125,7 @@ class CropArTfm(object): ...@@ -134,7 +125,7 @@ class CropArTfm(object):
else: else:
h = int(self.target_size * target_ar) h = int(self.target_size * target_ar)
size = (self.target_size, h // 8 * 8) size = (self.target_size, h // 8 * 8)
return transforms.functional.center_crop(img, size) return torchvision.transforms.functional.center_crop(img, size)
def sort_ar(valdir): def sort_ar(valdir):
...@@ -166,5 +157,15 @@ def map_idx2ar(idx_ar_sorted, batch_size): ...@@ -166,5 +157,15 @@ def map_idx2ar(idx_ar_sorted, batch_size):
return idx2ar return idx2ar
if __name__ == "__main__": if __name__ == "__main__":
reader = test("/work/fast_resnet_data", 64, 128).reader() #ds, sampler = create_validation_set("/data/imagenet/validation", 128, 288, True, True)
print(next(reader())) #for item in sampler:
\ No newline at end of file # for idx in item:
# ds[idx]
import time
test_reader = test(valdir="/data/imagenet/validation", bs=64, sz=288, rect_val=True)
start_ts = time.time()
for idx, data in enumerate(test_reader()):
print(idx, data[0], data[0].shape, data[1])
if idx == 2:
break
\ No newline at end of file
fluid/PaddleCV/image_classification/fast_resnet/train.py
浏览文件 @ 2bb4e4a6
...@@ -19,9 +19,8 @@ import os ...@@ -19,9 +19,8 @@ import os
import traceback import traceback
import numpy as np import numpy as np
import torch
import torchvision_reader import torchvision_reader
import torch
import paddle import paddle
import paddle.fluid as fluid import paddle.fluid as fluid
import paddle.fluid.core as core import paddle.fluid.core as core
...@@ -32,25 +31,9 @@ import sys ...@@ -32,25 +31,9 @@ import sys
sys.path.append("..") sys.path.append("..")
from utility import add_arguments, print_arguments from utility import add_arguments, print_arguments
import functools import functools
import models from models.fast_resnet import FastResNet, lr_decay
import utils import utils
from env import dist_env from env import dist_env
import reader as imagenet_reader
def is_mp_mode():
return True if os.getenv("FLAGS_selected_gpus") else False
def nccl2_prepare(args, startup_prog):
config = fluid.DistributeTranspilerConfig()
config.mode = "nccl2"
t = fluid.DistributeTranspiler(config=config)
envs = args.dist_env
t.transpile(envs["trainer_id"],
trainers=','.join(envs["trainer_endpoints"]),
current_endpoint=envs["current_endpoint"],
startup_program=startup_prog)
def parse_args(): def parse_args():
parser = argparse.ArgumentParser(description=__doc__) parser = argparse.ArgumentParser(description=__doc__)
...@@ -62,7 +45,7 @@ def parse_args(): ...@@ -62,7 +45,7 @@ def parse_args():
add_arg('model_save_dir', str, "output", "model save directory") add_arg('model_save_dir', str, "output", "model save directory")
add_arg('pretrained_model', str, None, "Whether to use pretrained model.") add_arg('pretrained_model', str, None, "Whether to use pretrained model.")
add_arg('checkpoint', str, None, "Whether to resume checkpoint.") add_arg('checkpoint', str, None, "Whether to resume checkpoint.")
add_arg('lr', float, 0.1, "set learning rate.") add_arg('lr', float, 1.0, "set learning rate.")
add_arg('lr_strategy', str, "piecewise_decay", "Set the learning rate decay strategy.") add_arg('lr_strategy', str, "piecewise_decay", "Set the learning rate decay strategy.")
add_arg('model', str, "FastResNet", "Set the network to use.") add_arg('model', str, "FastResNet", "Set the network to use.")
add_arg('data_dir', str, "./data/ILSVRC2012", "The ImageNet dataset root dir.") add_arg('data_dir', str, "./data/ILSVRC2012", "The ImageNet dataset root dir.")
...@@ -89,83 +72,7 @@ def get_device_num(): ...@@ -89,83 +72,7 @@ def get_device_num():
['nvidia-smi', '-L']).decode().count('\n') ['nvidia-smi', '-L']).decode().count('\n')
return device_num return device_num
def linear_lr_decay(lr_values, epochs, bs_values, total_images): DEVICE_NUM = get_device_num()
from paddle.fluid.layers.learning_rate_scheduler import _decay_step_counter
import paddle.fluid.layers.tensor as tensor
import math
with paddle.fluid.default_main_program()._lr_schedule_guard():
global_step = _decay_step_counter()
lr = tensor.create_global_var(
shape=[1],
value=0.0,
dtype='float32',
persistable=True,
name="learning_rate")
with fluid.layers.control_flow.Switch() as switch:
last_steps = 0
for idx, epoch_bound in enumerate(epochs):
start_epoch, end_epoch = epoch_bound
linear_epoch = end_epoch - start_epoch
start_lr, end_lr = lr_values[idx]
linear_lr = end_lr - start_lr
steps = last_steps + linear_epoch * total_images / bs_values[idx] + 1
with switch.case(global_step < steps):
decayed_lr = start_lr + linear_lr * ((global_step - last_steps)* 1.0/(steps - last_steps))
last_steps = steps
fluid.layers.tensor.assign(decayed_lr, lr)
last_value_var = tensor.fill_constant(
shape=[1],
dtype='float32',
value=float(lr_values[-1]))
with switch.default():
fluid.layers.tensor.assign(last_value_var, lr)
return lr
def linear_lr_decay_by_epoch(lr_values, epochs, bs_values, total_images):
from paddle.fluid.layers.learning_rate_scheduler import _decay_step_counter
import paddle.fluid.layers.tensor as tensor
import math
with paddle.fluid.default_main_program()._lr_schedule_guard():
global_step = _decay_step_counter()
lr = tensor.create_global_var(
shape=[1],
value=0.0,
dtype='float32',
persistable=True,
name="learning_rate")
with fluid.layers.control_flow.Switch() as switch:
last_steps = 0
for idx, epoch_bound in enumerate(epochs):
start_epoch, end_epoch = epoch_bound
linear_epoch = end_epoch - start_epoch
start_lr, end_lr = lr_values[idx]
linear_lr = end_lr - start_lr
for epoch_step in xrange(linear_epoch):
steps = last_steps + (1 + epoch_step) * total_images / bs_values[idx]
boundary_val = tensor.fill_constant(
shape=[1],
dtype='float32',
value=float(steps),
force_cpu=True)
decayed_lr = start_lr + epoch_step * linear_lr * 1.0 / linear_epoch
with switch.case(global_step < boundary_val):
value_var = tensor.fill_constant(shape=[1], dtype='float32', value=float(decayed_lr))
print("steps: [%d], epoch : [%d], decayed_lr: [%f]" % (steps, start_epoch + epoch_step, decayed_lr))
fluid.layers.tensor.assign(value_var, lr)
last_steps = steps
last_value_var = tensor.fill_constant(
shape=[1],
dtype='float32',
value=float(lr_values[-1]))
with switch.default():
fluid.layers.tensor.assign(last_value_var, lr)
return lr
def test_parallel(exe, test_args, args, test_prog, feeder, bs): def test_parallel(exe, test_args, args, test_prog, feeder, bs):
acc_evaluators = [] acc_evaluators = []
...@@ -183,69 +90,51 @@ def test_parallel(exe, test_args, args, test_prog, feeder, bs): ...@@ -183,69 +90,51 @@ def test_parallel(exe, test_args, args, test_prog, feeder, bs):
for i, e in enumerate(acc_evaluators): for i, e in enumerate(acc_evaluators):
e.update( e.update(
value=np.array(acc_rets[i]), weight=bs) value=np.array(acc_rets[i]), weight=bs)
num_samples = batch_id * bs * get_device_num() num_samples = batch_id * bs * DEVICE_NUM
print_train_time(start_ts, time.time(), num_samples, "Test") print_train_time(start_ts, time.time(), num_samples)
return [e.eval() for e in acc_evaluators] return [e.eval() for e in acc_evaluators]
def test_single(exe, test_args, args, test_prog, feeder, bs):
test_reader = test_args[3] def build_program(args, is_train, main_prog, startup_prog, py_reader_startup_prog, sz, trn_dir, bs, min_scale, rect_val=False):
to_fetch = [v.name for v in test_args[2]]
acc1 = fluid.metrics.Accuracy() dshape=[3, sz, sz]
acc5 = fluid.metrics.Accuracy() class_dim=1000
start_ts = time.time()
for batch_id, data in enumerate(test_reader()):
batch_size = len(data[0])
acc_rets = exe.run(test_prog, fetch_list=to_fetch, feed=feeder.feed(data))
acc1.update(value=np.array(acc_rets[0]), weight=batch_size)
acc5.update(value=np.array(acc_rets[1]), weight=batch_size)
if batch_id % 30 == 0:
print("Test batch: [%d], acc_rets: [%s]" % (batch_id, acc_rets))
num_samples = batch_id * bs
print_train_time(start_ts, time.time(), num_samples, "Test")
return np.mean(acc1.eval()), np.mean(acc5.eval())
def build_program(args, is_train, main_prog, startup_prog, py_reader_startup_prog, img_size, trn_dir, batch_size, min_scale, rect_val):
dataloader = None
if is_train: if is_train:
dataloader = torchvision_reader.train(traindir=os.path.join(args.data_dir, trn_dir, "train"), bs=batch_size if is_mp_mode() else batch_size * get_device_num(), sz=img_size, min_scale=min_scale) reader = torchvision_reader.train(
traindir="/data/imagenet/%strain" % trn_dir, sz=sz, min_scale=min_scale)
else: else:
dataloader = torchvision_reader.test(valdir=os.path.join(args.data_dir, trn_dir, "validation"), bs=batch_size if is_mp_mode() else batch_size * get_device_num(), sz=img_size, rect_val=rect_val) reader = torchvision_reader.test(
dshape = [3, img_size, img_size] valdir="/data/imagenet/%svalidation" % trn_dir, bs=bs*DEVICE_NUM, sz=sz, rect_val=rect_val)
class_dim = 1000
pyreader = None pyreader = None
batched_reader = None batched_reader = None
model_name = args.model
model_list = [m for m in dir(models) if "__" not in m]
assert model_name in model_list, "{} is not in lists: {}".format(args.model,
model_list)
model = models.__dict__[model_name]()
with fluid.program_guard(main_prog, startup_prog): with fluid.program_guard(main_prog, startup_prog):
with fluid.unique_name.guard(): with fluid.unique_name.guard():
if is_train: if is_train:
with fluid.program_guard(main_prog, py_reader_startup_prog): with fluid.program_guard(main_prog, py_reader_startup_prog):
with fluid.unique_name.guard(): with fluid.unique_name.guard():
pyreader = fluid.layers.py_reader( pyreader = fluid.layers.py_reader(
capacity=batch_size if is_mp_mode() else batch_size * get_device_num(), capacity=bs * DEVICE_NUM,
shapes=([-1] + dshape, (-1, 1)), shapes=([-1] + dshape, (-1, 1)),
dtypes=('uint8', 'int64'), dtypes=('uint8', 'int64'),
name="train_reader_" + str(img_size) if is_train else "test_reader_" + str(img_size), name="train_reader_" + str(sz) if is_train else "test_reader_" + str(sz),
use_double_buffer=True) use_double_buffer=True)
input, label = fluid.layers.read_file(pyreader) input, label = fluid.layers.read_file(pyreader)
pyreader.decorate_paddle_reader(paddle.batch(reader, batch_size=bs))
else: else:
input = fluid.layers.data(name="image", shape=[3, 244, 244], dtype="uint8") input = fluid.layers.data(name="image", shape=[3, 244, 244], dtype="uint8")
label = fluid.layers.data(name="label", shape=[1], dtype="int64") label = fluid.layers.data(name="label", shape=[1], dtype="int64")
batched_reader = paddle.batch(reader, batch_size=bs * DEVICE_NUM)
cast_img_type = "float16" if args.fp16 else "float32" cast_img_type = "float16" if args.fp16 else "float32"
cast = fluid.layers.cast(input, cast_img_type) cast = fluid.layers.cast(input, cast_img_type)
img_mean = fluid.layers.create_global_var([3, 1, 1], 0.0, cast_img_type, name="img_mean", persistable=True) img_mean = fluid.layers.create_global_var([3, 1, 1], 0.0, cast_img_type, name="img_mean", persistable=True)
img_std = fluid.layers.create_global_var([3, 1, 1], 0.0, cast_img_type, name="img_std", persistable=True) img_std = fluid.layers.create_global_var([3, 1, 1], 0.0, cast_img_type, name="img_std", persistable=True)
#image = (image - (mean * 255.0)) / (std * 255.0) # image = (image - (mean * 255.0)) / (std * 255.0)
t1 = fluid.layers.elementwise_sub(cast, img_mean, axis=1) t1 = fluid.layers.elementwise_sub(cast, img_mean, axis=1)
t2 = fluid.layers.elementwise_div(t1, img_std, axis=1) t2 = fluid.layers.elementwise_div(t1, img_std, axis=1)
predict = model.net(t2, class_dim=class_dim, img_size=img_size, is_train=is_train) model = FastResNet(is_train=is_train)
predict = model.net(t2, class_dim=class_dim, img_size=sz)
cost, pred = fluid.layers.softmax_with_cross_entropy(predict, label, return_softmax=True) cost, pred = fluid.layers.softmax_with_cross_entropy(predict, label, return_softmax=True)
if args.scale_loss > 1: if args.scale_loss > 1:
avg_cost = fluid.layers.mean(x=cost) * float(args.scale_loss) avg_cost = fluid.layers.mean(x=cost) * float(args.scale_loss)
...@@ -258,15 +147,19 @@ def build_program(args, is_train, main_prog, startup_prog, py_reader_startup_pro ...@@ -258,15 +147,19 @@ def build_program(args, is_train, main_prog, startup_prog, py_reader_startup_pro
# configure optimize # configure optimize
optimizer = None optimizer = None
if is_train: if is_train:
total_images = args.total_images
lr = args.lr
epochs = [(0,7), (7,13), (13, 22), (22, 25), (25, 28)] epochs = [(0,7), (7,13), (13, 22), (22, 25), (25, 28)]
bs_epoch = [x if is_mp_mode() else x * get_device_num() for x in [224, 224, 96, 96, 50]] bs_epoch = [bs*DEVICE_NUM for bs in [224, 224, 96, 96, 50]]
lrs = [(1.0, 2.0), (2.0, 0.25), (0.42857142857142855, 0.04285714285714286), (0.04285714285714286, 0.004285714285714286), (0.0022321428571428575, 0.00022321428571428573), 0.00022321428571428573] bs_scale = [bs*1.0 / bs_epoch[0] for bs in bs_epoch]
images_per_worker = args.total_images / get_device_num() if is_mp_mode() else args.total_images lrs = [(lr, lr*2), (lr*2, lr/4), (lr*bs_scale[2], lr/10*bs_scale[2]), (lr/10*bs_scale[2], lr/100*bs_scale[2]), (lr/100*bs_scale[4], lr/1000*bs_scale[4]), lr/1000*bs_scale[4]]
boundaries, values = lr_decay(lrs, epochs, bs_epoch, total_images)
optimizer = fluid.optimizer.Momentum( optimizer = fluid.optimizer.Momentum(
learning_rate=linear_lr_decay_by_epoch(lrs, epochs, bs_epoch, images_per_worker), learning_rate=fluid.layers.piecewise_decay(boundaries=boundaries, values=values),
momentum=0.9) momentum=0.9)
#regularization=fluid.regularizer.L2Decay(1e-4))
if args.fp16: if args.fp16:
params_grads = optimizer.backward(avg_cost) params_grads = optimizer.backward(avg_cost)
master_params_grads = utils.create_master_params_grads( master_params_grads = utils.create_master_params_grads(
...@@ -276,41 +169,32 @@ def build_program(args, is_train, main_prog, startup_prog, py_reader_startup_pro ...@@ -276,41 +169,32 @@ def build_program(args, is_train, main_prog, startup_prog, py_reader_startup_pro
else: else:
optimizer.minimize(avg_cost) optimizer.minimize(avg_cost)
if args.memory_optimize: if args.memory_optimize:
fluid.memory_optimize(main_prog, skip_grads=True) fluid.memory_optimize(main_prog, skip_grads=True)
if is_train:
pyreader.decorate_paddle_reader(paddle.batch(dataloader.reader(), batch_size=batch_size, drop_last=True))
else:
batched_reader = paddle.batch(dataloader.reader(), batch_size=batch_size if is_mp_mode() else batch_size * get_device_num(), drop_last=True)
return avg_cost, optimizer, [batch_acc1, return avg_cost, optimizer, [batch_acc1,
batch_acc5], batched_reader, pyreader, py_reader_startup_prog, dataloader batch_acc5], batched_reader, pyreader, py_reader_startup_prog
def refresh_program(args, epoch, sz, trn_dir, bs, val_bs, need_update_start_prog=False, min_scale=0.08, rect_val=False): def refresh_program(args, epoch, sz, trn_dir, bs, val_bs, need_update_start_prog=False, min_scale=0.08, rect_val=False):
print('program changed: epoch: [%d], image size: [%d], trn_dir: [%s], batch_size:[%d]' % (epoch, sz, trn_dir, bs)) print('refresh program: epoch: [%d], image size: [%d], trn_dir: [%s], batch_size:[%d]' % (epoch, sz, trn_dir, bs))
train_prog = fluid.Program() train_prog = fluid.Program()
test_prog = fluid.Program() test_prog = fluid.Program()
startup_prog = fluid.Program() startup_prog = fluid.Program()
py_reader_startup_prog = fluid.Program() py_reader_startup_prog = fluid.Program()
num_trainers = args.dist_env["num_trainers"]
trainer_id = args.dist_env["trainer_id"]
train_args = build_program(args, True, train_prog, startup_prog, py_reader_startup_prog, sz, trn_dir, bs, min_scale, False) train_args = build_program(args, True, train_prog, startup_prog, py_reader_startup_prog, sz, trn_dir, bs, min_scale)
test_args = build_program(args, False, test_prog, startup_prog, py_reader_startup_prog, sz, trn_dir, val_bs, min_scale, rect_val) test_args = build_program(args, False, test_prog, startup_prog, py_reader_startup_prog, sz, trn_dir, val_bs, min_scale, rect_val=rect_val)
gpu_id = int(os.getenv("FLAGS_selected_gpus")) if is_mp_mode() else 0
place = core.CUDAPlace(gpu_id) place = core.CUDAPlace(0)
startup_exe = fluid.Executor(place) startup_exe = fluid.Executor(place)
print("execute py_reader startup program")
startup_exe.run(py_reader_startup_prog) startup_exe.run(py_reader_startup_prog)
if need_update_start_prog: if need_update_start_prog:
print("execute startup program")
if is_mp_mode():
nccl2_prepare(args, startup_prog)
startup_exe.run(startup_prog) startup_exe.run(startup_prog)
conv2d_w_vars = [var for var in startup_prog.global_block().vars.values() if var.name.startswith('conv2d_')] conv2d_w_vars = [var for var in startup_prog.global_block().vars.values() if var.name.startswith('conv2d_')]
for var in conv2d_w_vars: for var in conv2d_w_vars:
torch_w = torch.empty(var.shape) torch_w = torch.empty(var.shape)
#print("initialize %s, shape: %s, with kaiming normalization." % (var.name, var.shape))
kaiming_np = torch.nn.init.kaiming_normal_(torch_w, mode='fan_out', nonlinearity='relu').numpy() kaiming_np = torch.nn.init.kaiming_normal_(torch_w, mode='fan_out', nonlinearity='relu').numpy()
tensor = fluid.global_scope().find_var(var.name).get_tensor() tensor = fluid.global_scope().find_var(var.name).get_tensor()
if args.fp16: if args.fp16:
...@@ -328,28 +212,24 @@ def refresh_program(args, epoch, sz, trn_dir, bs, val_bs, need_update_start_prog ...@@ -328,28 +212,24 @@ def refresh_program(args, epoch, sz, trn_dir, bs, val_bs, need_update_start_prog
else: else:
var.get_tensor().set(np_tensor, place) var.get_tensor().set(np_tensor, place)
strategy = fluid.ExecutionStrategy() strategy = fluid.ExecutionStrategy()
strategy.num_threads = args.num_threads strategy.num_threads = args.num_threads
strategy.allow_op_delay = False strategy.allow_op_delay = False
strategy.num_iteration_per_drop_scope = 1 strategy.num_iteration_per_drop_scope = 1
build_strategy = fluid.BuildStrategy() build_strategy = fluid.BuildStrategy()
build_strategy.reduce_strategy = fluid.BuildStrategy().ReduceStrategy.AllReduce build_strategy.reduce_strategy = fluid.BuildStrategy().ReduceStrategy.AllReduce
avg_loss = train_args[0] avg_loss = train_args[0]
train_exe = fluid.ParallelExecutor( train_exe = fluid.ParallelExecutor(
True, True,
avg_loss.name, avg_loss.name,
main_program=train_prog, main_program=train_prog,
exec_strategy=strategy, exec_strategy=strategy,
build_strategy=build_strategy, build_strategy=build_strategy)
num_trainers=num_trainers,
trainer_id=trainer_id)
test_exe = fluid.ParallelExecutor( test_exe = fluid.ParallelExecutor(
True, main_program=test_prog, share_vars_from=train_exe) True, main_program=test_prog, share_vars_from=train_exe)
#return train_args, test_args, test_prog, train_exe, test_exe
return train_args, test_args, test_prog, train_exe, test_exe return train_args, test_args, test_prog, train_exe, test_exe
# NOTE: only need to benchmark using parallelexe # NOTE: only need to benchmark using parallelexe
...@@ -363,18 +243,17 @@ def train_parallel(args): ...@@ -363,18 +243,17 @@ def train_parallel(args):
test_args = None test_args = None
bs = 224 bs = 224
val_bs = 64 val_bs = 64
for pass_id in range(args.num_epochs): for epoch_id in range(args.num_epochs):
# program changed # program changed
if pass_id == 0: if epoch_id == 0:
train_args, test_args, test_prog, exe, test_exe = refresh_program(args, pass_id, sz=128, trn_dir="sz/160/", bs=bs, val_bs=val_bs, need_update_start_prog=True) train_args, test_args, test_prog, exe, test_exe = refresh_program(args, epoch_id, sz=128, trn_dir="sz/160/", bs=bs, val_bs=val_bs, need_update_start_prog=True)
elif pass_id == 13: #13 elif epoch_id == 13: #13
bs = 96 bs = 96
val_bs = 32 train_args, test_args, test_prog, exe, test_exe = refresh_program(args, epoch_id, sz=224, trn_dir="sz/352/", bs=bs, val_bs=val_bs, min_scale=0.087)
train_args, test_args, test_prog, exe, test_exe = refresh_program(args, pass_id, sz=224, trn_dir="sz/352/", bs=bs, val_bs=val_bs, min_scale=0.087) elif epoch_id == 25: #25
elif pass_id == 25: #25
bs = 50 bs = 50
val_bs=4 val_bs=8
train_args, test_args, test_prog, exe, test_exe = refresh_program(args, pass_id, sz=288, trn_dir="", bs=bs, val_bs=val_bs, min_scale=0.5, rect_val=True) train_args, test_args, test_prog, exe, test_exe = refresh_program(args, epoch_id, sz=288, trn_dir="", bs=bs, val_bs=val_bs, min_scale=0.5, rect_val=True)
else: else:
pass pass
...@@ -382,17 +261,14 @@ def train_parallel(args): ...@@ -382,17 +261,14 @@ def train_parallel(args):
num_samples = 0 num_samples = 0
iters = 0 iters = 0
start_time = time.time() start_time = time.time()
train_dataloader = train_args[6] # Paddle DataLoader
train_dataloader.shuffle_seed = pass_id + 1
train_args[4].start() # start pyreader train_args[4].start() # start pyreader
batch_time_start = time.time() batch_start_time = time.time()
samples_per_step = bs if is_mp_mode() else bs * get_device_num()
while True: while True:
fetch_list = [avg_loss.name] fetch_list = [avg_loss.name]
acc_name_list = [v.name for v in train_args[2]] acc_name_list = [v.name for v in train_args[2]]
fetch_list.extend(acc_name_list) fetch_list.extend(acc_name_list)
fetch_list.append("learning_rate") fetch_list.append("learning_rate")
if iters > 0 and iters % args.log_period == 0: if iters % args.log_period == 0:
should_print = True should_print = True
else: else:
should_print = False should_print = False
...@@ -410,35 +286,36 @@ def train_parallel(args): ...@@ -410,35 +286,36 @@ def train_parallel(args):
except fluid.core.EnforceNotMet as ex: except fluid.core.EnforceNotMet as ex:
traceback.print_exc() traceback.print_exc()
exit(1) exit(1)
num_samples += samples_per_step
num_samples += bs * DEVICE_NUM
if should_print: if should_print:
fetched_data = [np.mean(np.array(d)) for d in fetch_ret] fetched_data = [np.mean(np.array(d)) for d in fetch_ret]
print("Pass %d, batch %d, loss %s, accucacys: %s, learning_rate %s, py_reader queue_size: %d, avg batch time: %0.4f secs" % print("Epoch %d, batch %d, loss %s, accucacys: %s, learning_rate %s, py_reader queue_size: %d, avg batch time: %0.4f secs" %
(pass_id, iters, fetched_data[0], fetched_data[1:-1], fetched_data[-1], train_args[4].queue.size(), (time.time() - batch_time_start) * 1.0 / args.log_period )) (epoch_id, iters, fetched_data[0], fetched_data[1:-1], fetched_data[-1], train_args[4].queue.size(), (time.time() - batch_start_time)*1.0/args.log_period))
batch_time_start = time.time() batch_start_time = time.time()
iters += 1 iters += 1
print_train_time(start_time, time.time(), num_samples, "Train") print_train_time(start_time, time.time(), num_samples)
feed_list = [test_prog.global_block().var(varname) for varname in ("image", "label")] feed_list = [test_prog.global_block().var(varname) for varname in ("image", "label")]
gpu_id = int(os.getenv("FLAGS_selected_gpus")) if is_mp_mode() else 0 test_feeder = fluid.DataFeeder(feed_list=feed_list, place=fluid.CUDAPlace(0))
test_feeder = fluid.DataFeeder(feed_list=feed_list, place=fluid.CUDAPlace(gpu_id))
#test_ret = test_single(test_exe, test_args, args, test_prog, test_feeder, val_bs)
test_ret = test_parallel(test_exe, test_args, args, test_prog, test_feeder, val_bs) test_ret = test_parallel(test_exe, test_args, args, test_prog, test_feeder, val_bs)
print("Pass: %d, Test Accuracy: %s, Spend %.2f hours\n" % print("Epoch: %d, Test Accuracy: %s, Spend %.2f hours\n" %
(pass_id, [np.mean(np.array(v)) for v in test_ret], (time.time() - over_all_start) / 3600)) (epoch_id, [np.mean(np.array(v)) for v in test_ret], (time.time() - over_all_start) / 3600))
print("total train time: ", time.time() - over_all_start) print("total train time: ", time.time() - over_all_start)
def print_train_time(start_time, end_time, num_samples, prefix_text=""):
def print_train_time(start_time, end_time, num_samples):
train_elapsed = end_time - start_time train_elapsed = end_time - start_time
examples_per_sec = num_samples / train_elapsed examples_per_sec = num_samples / train_elapsed
print('\n%s Total examples: %d, total time: %.5f, %.5f examples/sed\n' % print('\nTotal examples: %d, total time: %.5f, %.5f examples/sed\n' %
(prefix_text, num_samples, train_elapsed, examples_per_sec)) (num_samples, train_elapsed, examples_per_sec))
def print_paddle_envs(): def print_paddle_envs():
print('----------- Configuration envs -----------') print('----------- Configuration envs -----------')
print("DEVICE_NUM: %d" % DEVICE_NUM)
for k in os.environ: for k in os.environ:
if "PADDLE_" in k: if "PADDLE_" in k:
print "ENV %s:%s" % (k, os.environ[k]) print "ENV %s:%s" % (k, os.environ[k])
...@@ -447,7 +324,6 @@ def print_paddle_envs(): ...@@ -447,7 +324,6 @@ def print_paddle_envs():
def main(): def main():
args = parse_args() args = parse_args()
args.dist_env = dist_env()
print_arguments(args) print_arguments(args)
print_paddle_envs() print_paddle_envs()
train_parallel(args) train_parallel(args)
......
fluid/PaddleCV/image_classification/models/fast_resnet.py
浏览文件 @ 2bb4e4a6
...@@ -30,14 +30,12 @@ import paddle.fluid.core as core ...@@ -30,14 +30,12 @@ import paddle.fluid.core as core
import paddle.fluid.profiler as profiler import paddle.fluid.profiler as profiler
import utils import utils
## visreader for imagenet
import torchvision_reader
__all__ = ["FastResNet"] __all__ = ["FastResNet"]
class FastResNet(): class FastResNet():
def __init__(self, layers=50): def __init__(self, layers=50, is_train=True):
self.layers = layers self.layers = layers
self.is_train = is_train
def net(self, input, class_dim=1000, img_size=224, is_train=True): def net(self, input, class_dim=1000, img_size=224, is_train=True):
layers = self.layers layers = self.layers
...@@ -54,7 +52,7 @@ class FastResNet(): ...@@ -54,7 +52,7 @@ class FastResNet():
num_filters = [64, 128, 256, 512] num_filters = [64, 128, 256, 512]
conv = self.conv_bn_layer( conv = self.conv_bn_layer(
input=input, num_filters=64, filter_size=7, stride=2, act='relu', is_train=is_train) input=input, num_filters=64, filter_size=7, stride=2, act='relu')
conv = fluid.layers.pool2d( conv = fluid.layers.pool2d(
input=conv, input=conv,
pool_size=3, pool_size=3,
...@@ -73,6 +71,7 @@ class FastResNet(): ...@@ -73,6 +71,7 @@ class FastResNet():
input=conv, pool_size=pool_size, pool_type='avg', global_pooling=True) input=conv, pool_size=pool_size, pool_type='avg', global_pooling=True)
out = fluid.layers.fc(input=pool, out = fluid.layers.fc(input=pool,
size=class_dim, size=class_dim,
act=None,
param_attr=fluid.param_attr.ParamAttr( param_attr=fluid.param_attr.ParamAttr(
initializer=fluid.initializer.NormalInitializer(0.0, 0.01), initializer=fluid.initializer.NormalInitializer(0.0, 0.01),
regularizer=fluid.regularizer.L2Decay(1e-4)), regularizer=fluid.regularizer.L2Decay(1e-4)),
...@@ -87,8 +86,7 @@ class FastResNet(): ...@@ -87,8 +86,7 @@ class FastResNet():
stride=1, stride=1,
groups=1, groups=1,
act=None, act=None,
bn_init_value=1.0, bn_init_value=1.0):
is_train=True):
conv = fluid.layers.conv2d( conv = fluid.layers.conv2d(
input=input, input=input,
num_filters=num_filters, num_filters=num_filters,
...@@ -98,10 +96,8 @@ class FastResNet(): ...@@ -98,10 +96,8 @@ class FastResNet():
groups=groups, groups=groups,
act=None, act=None,
bias_attr=False, bias_attr=False,
param_attr=fluid.ParamAttr( param_attr=fluid.ParamAttr(regularizer=fluid.regularizer.L2Decay(1e-4)))
initializer=fluid.initializer.MSRAInitializer(), return fluid.layers.batch_norm(input=conv, act=act, is_test=not self.is_train,
regularizer=fluid.regularizer.L2Decay(1e-4)))
return fluid.layers.batch_norm(input=conv, act=act, is_test=not is_train,
param_attr=fluid.param_attr.ParamAttr( param_attr=fluid.param_attr.ParamAttr(
initializer=fluid.initializer.Constant(bn_init_value), initializer=fluid.initializer.Constant(bn_init_value),
regularizer=None)) regularizer=None))
...@@ -129,3 +125,67 @@ class FastResNet(): ...@@ -129,3 +125,67 @@ class FastResNet():
short = self.shortcut(input, num_filters * 4, stride) short = self.shortcut(input, num_filters * 4, stride)
return fluid.layers.elementwise_add(x=short, y=conv2, act='relu') return fluid.layers.elementwise_add(x=short, y=conv2, act='relu')
def lr_decay(lrs, epochs, bs, total_image):
boundaries = []
values = []
for idx, epoch in enumerate(epochs):
step = total_image // bs[idx]
if step * bs[idx] < total_image:
step += 1
ratio = (lrs[idx][1] - lrs[idx][0])*1.0 / (epoch[1] - epoch[0])
lr_base = lrs[idx][0]
for s in xrange(epoch[0], epoch[1]):
if boundaries:
boundaries.append(boundaries[-1] + step)
else:
boundaries = [step]
lr = lr_base + ratio * (s - epoch[0])
values.append(lr)
print("epoch: [%d], steps: [%d], lr: [%f]" % (s, boundaries[-1], values[-1]))
values.append(lrs[-1])
print("epoch: [%d:], steps: [%d:], lr:[%f]" % (epochs[-1][-1], boundaries[-1], values[-1]))
return boundaries, values
def linear_lr_decay_by_epoch(lr_values, epochs, bs_values, total_images):
from paddle.fluid.layers.learning_rate_scheduler import _decay_step_counter
import paddle.fluid.layers.tensor as tensor
import math
with paddle.fluid.default_main_program()._lr_schedule_guard():
global_step = _decay_step_counter()
lr = tensor.create_global_var(
shape=[1],
value=0.0,
dtype='float32',
persistable=True,
name="learning_rate")
with fluid.layers.control_flow.Switch() as switch:
last_steps = 0
for idx, epoch_bound in enumerate(epochs):
start_epoch, end_epoch = epoch_bound
linear_epoch = end_epoch - start_epoch
start_lr, end_lr = lr_values[idx]
linear_lr = end_lr - start_lr
for epoch_step in xrange(linear_epoch):
steps = last_steps + (1 + epoch_step) * total_images / bs_values[idx] + 1
boundary_val = tensor.fill_constant(
shape=[1],
dtype='float32',
value=float(steps),
force_cpu=True)
decayed_lr = start_lr + epoch_step * linear_lr * 1.0 / linear_epoch
with switch.case(global_step < boundary_val):
value_var = tensor.fill_constant(shape=[1], dtype='float32', value=float(decayed_lr))
print("steps: [%d], epoch : [%d], decayed_lr: [%f]" % (steps, start_epoch + epoch_step, decayed_lr))
fluid.layers.tensor.assign(value_var, lr)
last_steps = steps
last_value_var = tensor.fill_constant(
shape=[1],
dtype='float32',
value=float(lr_values[-1]))
with switch.default():
fluid.layers.tensor.assign(last_value_var, lr)
return lr
\ No newline at end of file
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或